diff options
425 files changed, 35759 insertions, 5396 deletions
diff --git a/application/pom.xml b/application/pom.xml index 61cea1a1826..af25bda0f07 100644 --- a/application/pom.xml +++ b/application/pom.xml @@ -97,17 +97,6 @@ <scope>compile</scope> </dependency> <dependency> - <groupId>com.optimaize.languagedetector</groupId> - <artifactId>language-detector</artifactId> - <exclusions> - <exclusion> - <!-- We want to get this via jdisc-core --> - <groupId>org.slf4j</groupId> - <artifactId>slf4j-api</artifactId> - </exclusion> - </exclusions> - </dependency> - <dependency> <groupId>org.antlr</groupId> <artifactId>antlr-runtime</artifactId> </dependency> diff --git a/bundle-plugin-test/integration-test/src/test/java/com/yahoo/container/plugin/BundleTest.java b/bundle-plugin-test/integration-test/src/test/java/com/yahoo/container/plugin/BundleTest.java index dbfac4e730f..faa33542f5c 100644 --- a/bundle-plugin-test/integration-test/src/test/java/com/yahoo/container/plugin/BundleTest.java +++ b/bundle-plugin-test/integration-test/src/test/java/com/yahoo/container/plugin/BundleTest.java @@ -18,10 +18,8 @@ import java.util.jar.Manifest; import java.util.regex.Pattern; import java.util.zip.ZipEntry; -import static org.hamcrest.CoreMatchers.containsString; -import static org.hamcrest.CoreMatchers.is; +import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNotNull; -import static org.junit.Assert.assertThat; import static org.junit.Assert.assertTrue; /** @@ -35,7 +33,7 @@ public class BundleTest { // If bundle-plugin-test is compiled in a mvn command that also built dependencies, e.g. jrt, // the artifact is jrt.jar, otherwise the installed and versioned artifact // is used: jrt-7-SNAPSHOT.jar or e.g. jrt-7.123.45.jar. - private static String snapshotOrVersionOrNone = "(-\\d+((-SNAPSHOT)|((\\.\\d+(\\.\\d+)?)?))?)?\\.jar"; + private static final String snapshotOrVersionOrNone = "(-\\d+((-SNAPSHOT)|((\\.\\d+(\\.\\d+)?)?))?)?\\.jar"; private JarFile jarFile; private Attributes mainAttributes; @@ -67,12 +65,12 @@ public class BundleTest { // Because of snapshot builds, we can only verify the major version. int majorBundleVersion = Integer.valueOf(bundleVersion.substring(0, bundleVersion.indexOf('.'))); - assertThat(majorBundleVersion, is(VespaVersion.major)); + assertEquals(VespaVersion.major, majorBundleVersion); } @Test public void require_that_bundle_symbolic_name_matches_pom_artifactId() { - assertThat(mainAttributes.getValue("Bundle-SymbolicName"), is("main")); + assertEquals("main", mainAttributes.getValue("Bundle-SymbolicName")); } @Test @@ -80,29 +78,29 @@ public class BundleTest { String importPackage = mainAttributes.getValue("Import-Package"); // From SimpleSearcher - assertThat(importPackage, containsString("com.yahoo.prelude.hitfield")); + assertTrue(importPackage.contains("com.yahoo.prelude.hitfield")); // From SimpleSearcher2 - assertThat(importPackage, containsString("com.yahoo.processing")); - assertThat(importPackage, containsString("com.yahoo.metrics.simple")); - assertThat(importPackage, containsString("com.google.inject")); + assertTrue(importPackage.contains("com.yahoo.processing")); + assertTrue(importPackage.contains("com.yahoo.metrics.simple")); + assertTrue(importPackage.contains("com.google.inject")); } @Test public void require_that_manifest_contains_manual_imports() { String importPackage = mainAttributes.getValue("Import-Package"); - assertThat(importPackage, containsString("manualImport.withoutVersion")); - assertThat(importPackage, containsString("manualImport.withVersion;version=\"12.3.4\"")); + assertTrue(importPackage.contains("manualImport.withoutVersion")); + assertTrue(importPackage.contains("manualImport.withVersion;version=\"12.3.4\"")); for (int i=1; i<=2; ++i) - assertThat(importPackage, containsString("multiple.packages.with.the.same.version" + i + ";version=\"[1,2)\"")); + assertTrue(importPackage.contains("multiple.packages.with.the.same.version" + i + ";version=\"[1,2)\"")); } @Test public void require_that_manifest_contains_exports() { String exportPackage = mainAttributes.getValue("Export-Package"); - assertThat(exportPackage, containsString("com.yahoo.test;version=1.2.3.RELEASE")); + assertTrue(exportPackage.contains("com.yahoo.test;version=1.2.3.RELEASE")); } @Test @@ -110,7 +108,7 @@ public class BundleTest { // generated bundle. (It's compile scoped in pom.xml to be added to the bundle-cp.) public void require_that_manifest_contains_bundle_class_path() { String bundleClassPath = mainAttributes.getValue("Bundle-ClassPath"); - assertThat(bundleClassPath, containsString(".,")); + assertTrue(bundleClassPath.contains(".,")); Pattern jrtPattern = Pattern.compile("dependencies/jrt" + snapshotOrVersionOrNone); assertTrue("Bundle class path did not contain jrt.", jrtPattern.matcher(bundleClassPath).find()); @@ -139,7 +137,7 @@ public class BundleTest { @Test public void require_that_web_inf_url_is_propagated_to_the_manifest() { String webInfUrl = mainAttributes.getValue("WebInfUrl"); - assertThat(webInfUrl, containsString("/WEB-INF/web.xml")); + assertTrue(webInfUrl.contains("/WEB-INF/web.xml")); } } diff --git a/bundle-plugin-test/integration-test/src/test/java/com/yahoo/container/plugin/ExportPackageVersionTest.java b/bundle-plugin-test/integration-test/src/test/java/com/yahoo/container/plugin/ExportPackageVersionTest.java index 36d3faba81f..f99f1583324 100644 --- a/bundle-plugin-test/integration-test/src/test/java/com/yahoo/container/plugin/ExportPackageVersionTest.java +++ b/bundle-plugin-test/integration-test/src/test/java/com/yahoo/container/plugin/ExportPackageVersionTest.java @@ -10,9 +10,8 @@ import java.util.jar.Attributes; import java.util.jar.JarFile; import static com.yahoo.container.plugin.BundleTest.findBundleJar; -import static org.hamcrest.CoreMatchers.containsString; -import static org.hamcrest.CoreMatchers.not; -import static org.junit.Assert.assertThat; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; /** * Verifies that the 'useArtifactVersionForExportPackages' setting for the bundle-plugin works as intended. @@ -42,16 +41,16 @@ public class ExportPackageVersionTest { String expectedExport = "ai.vespa.noversion;version=" + bundleVersion; String exportPackage = mainAttributes.getValue("Export-Package"); - assertThat(exportPackage, containsString(expectedExport)); + assertTrue(exportPackage.contains(expectedExport)); // Verify that there is no qualifier - assertThat(exportPackage, not(containsString(expectedExport + "."))); + assertFalse(exportPackage.contains(expectedExport + ".")); } @Test public void explicit_version_in_ExportPackage_annotation_overrides_artifact_version() { String exportPackage = mainAttributes.getValue("Export-Package"); - assertThat(exportPackage, containsString("ai.vespa.explicitversion;version=2.4.6.RELEASE")); + assertTrue(exportPackage.contains("ai.vespa.explicitversion;version=2.4.6.RELEASE")); } @Test @@ -60,13 +59,13 @@ public class ExportPackageVersionTest { // TODO: This test should have checked for a fixed version of the dependency bundle, different than the main bundle version. // See comment in the dependency bundle's pom.xml for why this is not the case. - assertThat(exportPackage, containsString("ai.vespa.noversion_dep;version=" + bundleVersion)); + assertTrue(exportPackage.contains("ai.vespa.noversion_dep;version=" + bundleVersion)); } @Test public void explicit_version_in_ExportPackage_annotation_overrides_artifact_version_of_compile_scoped_dependency() { String exportPackage = mainAttributes.getValue("Export-Package"); - assertThat(exportPackage, containsString("ai.vespa.explicitversion_dep;version=3.6.9.RELEASE")); + assertTrue(exportPackage.contains("ai.vespa.explicitversion_dep;version=3.6.9.RELEASE")); } } diff --git a/bundle-plugin/pom.xml b/bundle-plugin/pom.xml index 07811d23581..9ed4c47c023 100644 --- a/bundle-plugin/pom.xml +++ b/bundle-plugin/pom.xml @@ -39,15 +39,6 @@ <scope>test</scope> </dependency> <dependency> - <groupId>org.hamcrest</groupId> - <artifactId>hamcrest-library</artifactId> - <scope>test</scope> - </dependency> - <dependency> - <groupId>com.google.guava</groupId> - <artifactId>guava</artifactId> - </dependency> - <dependency> <groupId>org.ow2.asm</groupId> <artifactId>asm</artifactId> </dependency> diff --git a/bundle-plugin/src/main/java/com/yahoo/container/plugin/bundle/AnalyzeBundle.java b/bundle-plugin/src/main/java/com/yahoo/container/plugin/bundle/AnalyzeBundle.java index 2e44e69bac6..2b5941cc5aa 100644 --- a/bundle-plugin/src/main/java/com/yahoo/container/plugin/bundle/AnalyzeBundle.java +++ b/bundle-plugin/src/main/java/com/yahoo/container/plugin/bundle/AnalyzeBundle.java @@ -13,7 +13,6 @@ import java.util.List; import java.util.Optional; import java.util.Set; import java.util.jar.Manifest; -import java.util.stream.Collectors; /** * Static utilities for analyzing jar files. diff --git a/bundle-plugin/src/main/java/com/yahoo/container/plugin/osgi/ExportPackages.java b/bundle-plugin/src/main/java/com/yahoo/container/plugin/osgi/ExportPackages.java index 3b2aa2369ef..04fa18c9bc5 100644 --- a/bundle-plugin/src/main/java/com/yahoo/container/plugin/osgi/ExportPackages.java +++ b/bundle-plugin/src/main/java/com/yahoo/container/plugin/osgi/ExportPackages.java @@ -5,6 +5,7 @@ import java.util.Collection; import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.Objects; import java.util.Optional; import java.util.Set; import java.util.stream.Collectors; @@ -58,6 +59,19 @@ public class ExportPackages { public String getValue() { return value; } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + Parameter parameter = (Parameter) o; + return Objects.equals(name, parameter.name) && Objects.equals(value, parameter.value); + } + + @Override + public int hashCode() { + return Objects.hash(name, value); + } } public static Set<String> packageNames(Collection<Export> exports) { diff --git a/bundle-plugin/src/test/java/com/yahoo/container/plugin/bundle/AnalyzeBundleTest.java b/bundle-plugin/src/test/java/com/yahoo/container/plugin/bundle/AnalyzeBundleTest.java index 8fba5983049..62ecd780d7e 100644 --- a/bundle-plugin/src/test/java/com/yahoo/container/plugin/bundle/AnalyzeBundleTest.java +++ b/bundle-plugin/src/test/java/com/yahoo/container/plugin/bundle/AnalyzeBundleTest.java @@ -3,25 +3,17 @@ package com.yahoo.container.plugin.bundle; import com.yahoo.container.plugin.osgi.ExportPackages; import com.yahoo.container.plugin.osgi.ExportPackages.Export; -import org.hamcrest.Description; -import org.hamcrest.TypeSafeMatcher; -import org.junit.Rule; import org.junit.Test; -import org.junit.rules.ExpectedException; import java.io.File; -import java.util.Arrays; -import java.util.HashSet; import java.util.List; import java.util.Map; +import java.util.stream.Collectors; -import static com.yahoo.container.plugin.classanalysis.TestUtilities.throwableMessage; -import static org.hamcrest.CoreMatchers.containsString; -import static org.hamcrest.CoreMatchers.hasItem; -import static org.hamcrest.CoreMatchers.is; -import static org.hamcrest.CoreMatchers.not; -import static org.hamcrest.Matchers.startsWith; -import static org.junit.Assert.assertThat; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.fail; /** * @author Tony Vaagenes @@ -36,7 +28,7 @@ public class AnalyzeBundleTest { public AnalyzeBundleTest() { File notOsgi = new File(jarDir, "notAOsgiBundle.jar"); File simple = new File(jarDir, "simple1.jar"); - exports = AnalyzeBundle.exportedPackagesAggregated(Arrays.asList(notOsgi, simple)); + exports = AnalyzeBundle.exportedPackagesAggregated(List.of(notOsgi, simple)); exportsByPackageName = ExportPackages.exportsByPackageName(this.exports); } @@ -46,45 +38,28 @@ public class AnalyzeBundleTest { @Test public void require_that_non_osgi_bundles_are_ignored() { - assertThat(exportsByPackageName.keySet(), not(hasItem("com.yahoo.sample.exported.package.ignored"))); + assertFalse(exportsByPackageName.containsKey("com.yahoo.sample.exported.package.ignored")); } @Test public void require_that_exports_are_retrieved_from_manifest_in_jars() { - assertThat(exportsByPackageName.keySet().size(), is(1)); - assertThat(exportsByPackageName.keySet(), hasItem("com.yahoo.sample.exported.package")); + assertEquals(1, exportsByPackageName.keySet().size()); + assertTrue(exportsByPackageName.containsKey("com.yahoo.sample.exported.package")); } @Test public void exported_class_names_can_be_retrieved() { - assertThat(ExportPackages.packageNames(exports), is(new HashSet<>(exports.get(0).getPackageNames()))); + assertEquals(ExportPackages.packageNames(exports), exports.get(0).getPackageNames().stream().collect(Collectors.toSet())); } - @Rule - public ExpectedException exception = ExpectedException.none(); - @Test public void require_that_invalid_exports_throws_exception() { - exception.expect(Exception.class); - - exception.expectMessage(containsString("Invalid manifest in bundle")); - exception.expectMessage(matchesPattern("Invalid manifest in bundle '.*errorExport.jar'")); - exception.expectCause(throwableMessage(startsWith("Failed parsing Export-Package"))); - - AnalyzeBundle.exportedPackages(jarFile("errorExport.jar")); - } - - private TypeSafeMatcher<String> matchesPattern(String pattern) { - return new TypeSafeMatcher<String>() { - @Override - protected boolean matchesSafely(String s) { - return s.matches(pattern); - } - - @Override - public void describeTo(Description description) { - description.appendText("expects String that matches the pattern ").appendValue(pattern); - } - }; + try { + AnalyzeBundle.exportedPackages(jarFile("errorExport.jar")); + fail(); + } catch (RuntimeException e) { + assertTrue(e.getMessage().contains("Invalid manifest in bundle 'src/test/resources/jar/errorExport.jar'")); + assertTrue(e.getCause().getMessage(), e.getCause().getMessage().startsWith("Failed parsing Export-Package")); + } } } diff --git a/bundle-plugin/src/test/java/com/yahoo/container/plugin/classanalysis/AnalyzeClassTest.java b/bundle-plugin/src/test/java/com/yahoo/container/plugin/classanalysis/AnalyzeClassTest.java index 44ef26646f0..518f65ded17 100644 --- a/bundle-plugin/src/test/java/com/yahoo/container/plugin/classanalysis/AnalyzeClassTest.java +++ b/bundle-plugin/src/test/java/com/yahoo/container/plugin/classanalysis/AnalyzeClassTest.java @@ -12,28 +12,22 @@ import com.yahoo.container.plugin.classanalysis.sampleclasses.MethodAnnotation; import com.yahoo.container.plugin.classanalysis.sampleclasses.MethodInvocation; import com.yahoo.osgi.annotation.ExportPackage; import com.yahoo.osgi.annotation.Version; -import org.hamcrest.Matcher; -import org.hamcrest.Matchers; -import org.junit.Rule; import org.junit.Test; -import org.junit.rules.ExpectedException; import javax.security.auth.login.LoginException; import java.awt.Image; import java.awt.image.ImagingOpException; import java.awt.image.Kernel; +import java.util.List; import java.util.Optional; import static com.yahoo.container.plugin.classanalysis.TestUtilities.analyzeClass; import static com.yahoo.container.plugin.classanalysis.TestUtilities.classFile; import static com.yahoo.container.plugin.classanalysis.TestUtilities.name; -import static com.yahoo.container.plugin.classanalysis.TestUtilities.throwableMessage; -import static org.hamcrest.Matchers.allOf; -import static org.hamcrest.Matchers.containsString; -import static org.hamcrest.Matchers.hasItem; -import static org.hamcrest.Matchers.is; -import static org.hamcrest.Matchers.not; -import static org.junit.Assert.assertThat; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; /** * Tests that analysis of class files works. @@ -45,128 +39,127 @@ public class AnalyzeClassTest { @Test public void require_that_full_class_name_is_returned() { - assertThat(analyzeClass(Base.class).getName(), is(name(Base.class))); + assertEquals(name(Base.class), analyzeClass(Base.class).getName()); } @Test public void require_that_base_class_is_included() { - assertThat(analyzeClass(Derived.class).getReferencedClasses(), hasItem(name(Base.class))); + assertTrue(analyzeClass(Derived.class).getReferencedClasses().contains(name(Base.class))); } @Test public void require_that_implemented_interfaces_are_included() { - assertThat(analyzeClass(Base.class).getReferencedClasses(), - allOf(hasItem(name(Interface1.class)), hasItem(name(Interface2.class)))); + assertTrue(analyzeClass(Base.class).getReferencedClasses().containsAll( + List.of(name(Interface1.class), name(Interface2.class)))); } @Test public void require_that_interface_can_be_analyzed() { ClassFileMetaData classMetaData = analyzeClass(Interface1.class); - assertThat(classMetaData.getName(), is(name(Interface1.class))); - assertThat(classMetaData.getReferencedClasses(), hasItem(name(Interface2.class))); + assertEquals(name(Interface1.class), classMetaData.getName()); + assertTrue(classMetaData.getReferencedClasses().contains(name(Interface2.class))); } @Test public void require_that_return_type_is_included() { - assertThat(analyzeClass(Interface1.class).getReferencedClasses(), hasItem(name(Image.class))); + assertTrue(analyzeClass(Interface1.class).getReferencedClasses().contains(name(Image.class))); } @Test public void require_that_parameters_are_included() { - assertThat(analyzeClass(Interface1.class).getReferencedClasses(), hasItem(name(Kernel.class))); + assertTrue(analyzeClass(Interface1.class).getReferencedClasses().contains(name(Kernel.class))); } @Test public void require_that_exceptions_are_included() { - assertThat(analyzeClass(Interface1.class).getReferencedClasses(), hasItem(name(ImagingOpException.class))); + assertTrue(analyzeClass(Interface1.class).getReferencedClasses().contains(name(ImagingOpException.class))); } @Test public void require_that_basic_types_ignored() { - assertThat(analyzeClass(Interface1.class).getReferencedClasses(), - not(Matchers.<Iterable<? super String>>anyOf(hasItem("int"), hasItem("float")))); + List.of("int", "float").forEach(type -> + assertFalse(analyzeClass(Interface1.class).getReferencedClasses().contains(type))); } @Test public void require_that_arrays_of_basic_types_ignored() { - assertThat(analyzeClass(Interface1.class).getReferencedClasses(), - not(Matchers.<Iterable<? super String>>anyOf(hasItem("int[]"), hasItem("int[][]")))); + List.of("int[]", "int[][]").forEach(type -> + assertFalse(analyzeClass(Interface1.class).getReferencedClasses().contains(type))); } @Test public void require_that_instance_field_types_are_included() { - assertThat(analyzeClass(Fields.class).getReferencedClasses(), hasItem(name(String.class))); + assertTrue(analyzeClass(Fields.class).getReferencedClasses().contains(name(String.class))); } @Test public void require_that_static_field_types_are_included() { - assertThat(analyzeClass(Fields.class).getReferencedClasses(), hasItem(name(java.util.List.class))); + assertTrue(analyzeClass(Fields.class).getReferencedClasses().contains(name(java.util.List.class))); } @Test public void require_that_field_annotation_is_included() { - assertThat(analyzeClass(Fields.class).getReferencedClasses(), hasItem(name(DummyAnnotation.class))); + assertTrue(analyzeClass(Fields.class).getReferencedClasses().contains(name(DummyAnnotation.class))); } @Test public void require_that_class_annotation_is_included() { - assertThat(analyzeClass(ClassAnnotation.class).getReferencedClasses(), hasItem(name(DummyAnnotation.class))); + assertTrue(analyzeClass(ClassAnnotation.class).getReferencedClasses().contains(name(DummyAnnotation.class))); } @Test public void require_that_method_annotation_is_included() { - assertThat(analyzeClass(MethodAnnotation.class).getReferencedClasses(), hasItem(name(DummyAnnotation.class))); + assertTrue(analyzeClass(MethodAnnotation.class).getReferencedClasses().contains(name(DummyAnnotation.class))); } @Test public void require_that_export_package_annotations_are_ignored() { - assertThat(Analyze.analyzeClass(classFile("com.yahoo.container.plugin.classanalysis.sampleclasses.package-info")) - .getReferencedClasses(), not(Matchers.<Iterable<? super String>>anyOf( - hasItem(name(ExportPackage.class)), hasItem(name(Version.class))))); + List.of(ExportPackage.class, Version.class).forEach(type -> + assertFalse(Analyze.analyzeClass(classFile("com.yahoo.container.plugin.classanalysis.sampleclasses.package-info")) + .getReferencedClasses().contains(type))); } @Test public void require_that_export_annotations_are_processed() { - assertThat( - Analyze.analyzeClass(classFile("com.yahoo.container.plugin.classanalysis.sampleclasses.package-info")).getExportPackage(), - is(Optional.of(new ExportPackageAnnotation(3, 1, 4, "TEST_QUALIFIER-2")))); + assertEquals(Optional.of(new ExportPackageAnnotation(3, 1, 4, "TEST_QUALIFIER-2")), + Analyze.analyzeClass(classFile("com.yahoo.container.plugin.classanalysis.sampleclasses.package-info")).getExportPackage()); } - @Rule - public ExpectedException expectedException = ExpectedException.none(); - @Test public void require_that_export_annotations_are_validated() { - expectedException.expect(RuntimeException.class); - expectedException.expectMessage(containsString("invalid/package-info")); - expectedException.expectCause(throwableMessage(containsString("qualifier must follow the format"))); - expectedException.expectCause(throwableMessage(containsString("'EXAMPLE INVALID QUALIFIER'"))); - Analyze.analyzeClass(classFile("com.yahoo.container.plugin.classanalysis.sampleclasses.invalid.package-info")); + try { + Analyze.analyzeClass(classFile("com.yahoo.container.plugin.classanalysis.sampleclasses.invalid.package-info")); + fail(); + } catch (RuntimeException e) { + assertTrue(e.getMessage().contains("invalid/package-info")); + assertTrue(e.getCause().getMessage().contains("qualifier must follow the format")); + assertTrue(e.getCause().getMessage().contains("'EXAMPLE INVALID QUALIFIER'")); + } } @Test public void require_that_catch_clauses_are_included() { - assertThat(Analyze.analyzeClass(classFile("com.yahoo.container.plugin.classanalysis.sampleclasses.CatchException")) - .getReferencedClasses(), hasItem(name(LoginException.class))); + assertTrue(Analyze.analyzeClass(classFile("com.yahoo.container.plugin.classanalysis.sampleclasses.CatchException")) + .getReferencedClasses().contains(name(LoginException.class))); } @Test public void require_that_class_references_are_included() { - assertThat(Analyze.analyzeClass(classFile("com.yahoo.container.plugin.classanalysis.sampleclasses.ClassReference")) - .getReferencedClasses(), hasItem(name(Interface1.class))); + assertTrue(Analyze.analyzeClass(classFile("com.yahoo.container.plugin.classanalysis.sampleclasses.ClassReference")) + .getReferencedClasses().contains(name(Interface1.class))); } @Test public void require_that_return_type_of_method_invocations_are_included() { - assertThat(analyzeClass(MethodInvocation.class).getReferencedClasses(), hasItem(name(Image.class))); + assertTrue(analyzeClass(MethodInvocation.class).getReferencedClasses().contains(name(Image.class))); } @Test public void require_that_attributes_are_included() { //Uses com/coremedia/iso/Utf8.class from com.googlecode.mp4parser:isoparser:1.0-RC-1 - assertThat(Analyze.analyzeClass(classFile("class/Utf8")).getReferencedClasses(), - hasItem("org.aspectj.weaver.MethodDeclarationLineNumber")); + assertTrue(Analyze.analyzeClass(classFile("class/Utf8")).getReferencedClasses() + .contains("org.aspectj.weaver.MethodDeclarationLineNumber")); } } diff --git a/bundle-plugin/src/test/java/com/yahoo/container/plugin/classanalysis/AnalyzeMethodBodyTest.java b/bundle-plugin/src/test/java/com/yahoo/container/plugin/classanalysis/AnalyzeMethodBodyTest.java index b7565a4dad8..4502f88d158 100644 --- a/bundle-plugin/src/test/java/com/yahoo/container/plugin/classanalysis/AnalyzeMethodBodyTest.java +++ b/bundle-plugin/src/test/java/com/yahoo/container/plugin/classanalysis/AnalyzeMethodBodyTest.java @@ -11,13 +11,12 @@ import com.yahoo.container.plugin.classanalysis.sampleclasses.Methods; import org.junit.Test; import java.io.PrintStream; +import java.util.List; import static com.yahoo.container.plugin.classanalysis.TestUtilities.analyzeClass; import static com.yahoo.container.plugin.classanalysis.TestUtilities.name; -import static org.hamcrest.Matchers.allOf; -import static org.hamcrest.Matchers.hasItem; -import static org.hamcrest.Matchers.not; -import static org.junit.Assert.assertThat; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; /** * Tests that classes used in method bodies are included in the imports list. @@ -27,47 +26,46 @@ import static org.junit.Assert.assertThat; public class AnalyzeMethodBodyTest { @Test public void require_that_class_of_locals_are_included() { - assertThat(analyzeClass(Methods.class).getReferencedClasses(), hasItem(name(Base.class))); + assertTrue(analyzeClass(Methods.class).getReferencedClasses().contains(name(Base.class))); } @Test public void require_that_class_of_locals_in_static_method_are_included() { - assertThat(analyzeClass(Methods.class).getReferencedClasses(), hasItem(name(Derived.class))); + assertTrue(analyzeClass(Methods.class).getReferencedClasses().contains(name(Derived.class))); } @Test public void require_that_field_references_are_included() { - assertThat(analyzeClass(Methods.class).getReferencedClasses(), - allOf(hasItem(name(java.util.List.class)), hasItem(name(Fields.class)))); + assertTrue(analyzeClass(Methods.class).getReferencedClasses().containsAll(List.of(name(java.util.List.class), name(Fields.class)))); } @Test public void require_that_class_owning_field_is_included() { - assertThat(analyzeClass(Methods.class).getReferencedClasses(), hasItem(name(System.class))); + assertTrue(analyzeClass(Methods.class).getReferencedClasses().contains(name(System.class))); } @Test public void require_that_class_containing_method_is_included() { - assertThat(analyzeClass(Methods.class).getReferencedClasses(), hasItem(name(PrintStream.class))); + assertTrue(analyzeClass(Methods.class).getReferencedClasses().contains(name(PrintStream.class))); } @Test public void require_that_element_of_new_multidimensional_array_is_included() { - assertThat(analyzeClass(Methods.class).getReferencedClasses(), hasItem(name(Interface1.class))); + assertTrue(analyzeClass(Methods.class).getReferencedClasses().contains(name(Interface1.class))); } @Test public void require_that_basic_arrays_are_not_included() { - assertThat(analyzeClass(Methods.class).getReferencedClasses(), not(hasItem("int[]"))); + assertFalse(analyzeClass(Methods.class).getReferencedClasses().contains("int[]")); } @Test public void require_that_container_generic_parameters_are_included() { - assertThat(analyzeClass(Methods.class).getReferencedClasses(), hasItem(name(Dummy.class))); + assertTrue(analyzeClass(Methods.class).getReferencedClasses().contains(name(Dummy.class))); } @Test public void require_that_class_owning_method_handler_is_included() { - assertThat(analyzeClass(Methods.class).getReferencedClasses(), hasItem(name(ClassWithMethod.class))); + assertTrue(analyzeClass(Methods.class).getReferencedClasses().contains(name(ClassWithMethod.class))); } } diff --git a/bundle-plugin/src/test/java/com/yahoo/container/plugin/classanalysis/PackageTallyTest.java b/bundle-plugin/src/test/java/com/yahoo/container/plugin/classanalysis/PackageTallyTest.java index cfee980cb91..5bef1457c5f 100644 --- a/bundle-plugin/src/test/java/com/yahoo/container/plugin/classanalysis/PackageTallyTest.java +++ b/bundle-plugin/src/test/java/com/yahoo/container/plugin/classanalysis/PackageTallyTest.java @@ -3,11 +3,10 @@ package com.yahoo.container.plugin.classanalysis; import org.junit.Test; +import java.util.Map; import java.util.Set; -import static java.util.Collections.emptyMap; -import static org.hamcrest.Matchers.is; -import static org.junit.Assert.assertThat; +import static org.junit.Assert.assertEquals; /** * @author gjoranv @@ -16,9 +15,9 @@ public class PackageTallyTest { @Test public void referenced_packages_missing_from_are_detected() { - PackageTally tally = new PackageTally(emptyMap(), Set.of("p1", "java.util", "com.yahoo.api.annotations", "missing")); + PackageTally tally = new PackageTally(Map.of(), Set.of("p1", "java.util", "com.yahoo.api.annotations", "missing")); Set<String> missingPackages = tally.referencedPackagesMissingFrom(Set.of("p1")); - assertThat(missingPackages, is(Set.of("missing"))); + assertEquals(Set.of("missing"), missingPackages); } } diff --git a/bundle-plugin/src/test/java/com/yahoo/container/plugin/classanalysis/TestUtilities.java b/bundle-plugin/src/test/java/com/yahoo/container/plugin/classanalysis/TestUtilities.java index f0784cf80bc..9eacc0625b5 100644 --- a/bundle-plugin/src/test/java/com/yahoo/container/plugin/classanalysis/TestUtilities.java +++ b/bundle-plugin/src/test/java/com/yahoo/container/plugin/classanalysis/TestUtilities.java @@ -1,10 +1,6 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.container.plugin.classanalysis; -import org.hamcrest.Description; -import org.hamcrest.Matcher; -import org.hamcrest.TypeSafeMatcher; - import java.io.File; /** @@ -23,18 +19,4 @@ public class TestUtilities { public static String name(Class<?> clazz) { return clazz.getName(); } - - public static TypeSafeMatcher<Throwable> throwableMessage(final Matcher<String> matcher) { - return new TypeSafeMatcher<Throwable>() { - @Override - protected boolean matchesSafely(Throwable throwable) { - return matcher.matches(throwable.getMessage()); - } - - @Override - public void describeTo(Description description) { - description.appendText("expects Throwable and a message matching ").appendDescriptionOf(matcher); - } - }; - } } diff --git a/bundle-plugin/src/test/java/com/yahoo/container/plugin/osgi/ExportPackageParserTest.java b/bundle-plugin/src/test/java/com/yahoo/container/plugin/osgi/ExportPackageParserTest.java index cb976e46655..2c9b32a325f 100644 --- a/bundle-plugin/src/test/java/com/yahoo/container/plugin/osgi/ExportPackageParserTest.java +++ b/bundle-plugin/src/test/java/com/yahoo/container/plugin/osgi/ExportPackageParserTest.java @@ -3,17 +3,13 @@ package com.yahoo.container.plugin.osgi; import com.yahoo.container.plugin.osgi.ExportPackages.Export; import com.yahoo.container.plugin.osgi.ExportPackages.Parameter; -import org.hamcrest.Description; -import org.hamcrest.TypeSafeMatcher; import org.junit.Ignore; import org.junit.Test; import java.util.List; -import static org.hamcrest.CoreMatchers.is; -import static org.hamcrest.Matchers.contains; -import static org.hamcrest.Matchers.empty; -import static org.junit.Assert.assertThat; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.assertEquals; /** * @author Tony Vaagenes @@ -26,41 +22,41 @@ public class ExportPackageParserTest { public void require_that_package_is_parsed_correctly() { List<Export> exports = ExportPackageParser.parseExports("sample.exported.package"); - assertThat(exports.size(), is(1)); - assertThat(exports.get(0).getParameters(), empty()); - assertThat(exports.get(0).getPackageNames(), contains("sample.exported.package")); + assertEquals(1, exports.size()); + assertTrue(exports.get(0).getParameters().isEmpty()); + assertTrue(exports.get(0).getPackageNames().contains("sample.exported.package")); } @Test public void require_that_version_is_parsed_correctly() { List<Export> exports = ExportPackageParser.parseExports("com.yahoo.sample.exported.package;version=\"1.2.3.sample\""); - assertThat(exports.size(), is(1)); + assertEquals(1, exports.size()); Export export = exports.get(0); - assertThat(export.getPackageNames(), contains("com.yahoo.sample.exported.package")); - assertThat(export.getParameters(), contains(parameterMatching(versionParameter))); + assertTrue(export.getPackageNames().contains("com.yahoo.sample.exported.package")); + assertTrue(export.getParameters().contains(versionParameter)); } @Test public void require_that_multiple_packages_with_same_parameters_is_parsed_correctly() { List<Export> exports = ExportPackageParser.parseExports("exported.package1;exported.package2;version=\"1.2.3.sample\""); - assertThat(exports.size(), is(1)); + assertEquals(1, exports.size()); Export export = exports.get(0); - assertThat(export.getPackageNames(), contains("exported.package1", "exported.package2")); - assertThat(export.getParameters(), contains(parameterMatching(versionParameter))); + assertTrue(export.getPackageNames().containsAll(List.of("exported.package1", "exported.package2"))); + assertTrue(export.getParameters().contains(versionParameter)); } @Test public void require_that_spaces_between_separators_are_allowed() { List<Export> exports = ExportPackageParser.parseExports("exported.package1 , exported.package2 ; version = \"1.2.3.sample\" "); - assertThat(exports.size(), is(2)); + assertEquals(2, exports.size()); Export export = exports.get(0); - assertThat(export.getPackageNames(), contains("exported.package1")); + assertTrue(export.getPackageNames().contains("exported.package1")); export = exports.get(1); - assertThat(export.getPackageNames(), contains("exported.package2")); - assertThat(export.getParameters(), contains(parameterMatching(versionParameter))); + assertTrue(export.getPackageNames().contains("exported.package2")); + assertTrue(export.getParameters().contains(versionParameter)); } @SuppressWarnings("unchecked") @@ -68,39 +64,39 @@ public class ExportPackageParserTest { public void require_that_multiple_parameters_for_a_package_is_parsed_correctly() { List<Export> exports = ExportPackageParser.parseExports("exported.package;version=\"1.2.3.sample\";param2=true"); - assertThat(exports.size(), is(1)); + assertEquals(1, exports.size()); Export export = exports.get(0); - assertThat(export.getParameters(), contains(parameterMatching(versionParameter), parameterMatching("param2", "true"))); + assertTrue(export.getParameters().containsAll(List.of(versionParameter, new Parameter("param2", "true")))); } @Test public void require_that_multiple_exports_are_parsed_correctly() { List<Export> exports = ExportPackageParser.parseExports("exported.package1,exported.package2"); - assertThat(exports.size(), is(2)); + assertEquals(2, exports.size()); Export export = exports.get(0); - assertThat(export.getPackageNames(), contains("exported.package1")); - assertThat(export.getParameters(), empty()); + assertTrue(export.getPackageNames().contains("exported.package1")); + assertTrue(export.getParameters().isEmpty()); export = exports.get(1); - assertThat(export.getPackageNames(), contains("exported.package2")); - assertThat(export.getParameters(), empty()); + assertTrue(export.getPackageNames().contains("exported.package2")); + assertTrue(export.getParameters().isEmpty()); exports = ExportPackageParser.parseExports("exported.package1;version=\"1.2.3.sample\",exported.package2"); - assertThat(exports.size(), is(2)); + assertEquals(2, exports.size()); export = exports.get(0); - assertThat(export.getPackageNames(), contains("exported.package1")); - assertThat(export.getParameters(), contains(parameterMatching(versionParameter))); + assertTrue(export.getPackageNames().contains("exported.package1")); + assertTrue(export.getParameters().contains(versionParameter)); export = exports.get(1); - assertThat(export.getPackageNames(), contains("exported.package2")); - assertThat(export.getParameters(), empty()); + assertTrue(export.getPackageNames().contains("exported.package2")); + assertTrue(export.getParameters().isEmpty()); exports = ExportPackageParser.parseExports("exported.package1,exported.package2;version=\"1.2.3.sample\""); - assertThat(exports.size(), is(2)); + assertEquals(2, exports.size()); export = exports.get(0); - assertThat(export.getPackageNames(), contains("exported.package1")); - assertThat(export.getParameters(), empty()); + assertTrue(export.getPackageNames().contains("exported.package1")); + assertTrue(export.getParameters().isEmpty()); export = exports.get(1); - assertThat(export.getPackageNames(), contains("exported.package2")); - assertThat(export.getParameters(), contains(parameterMatching(versionParameter))); + assertTrue(export.getPackageNames().contains("exported.package2")); + assertTrue(export.getParameters().contains(versionParameter)); } // TODO: MAVEN_OPTS are not propagated by the maven-surefire-plugin. Either try to fix the underlying problem or set -Xss in plugin config. @@ -278,21 +274,4 @@ public class ExportPackageParserTest { ExportPackageParser.parseExports(exportHeader); } - private static TypeSafeMatcher<Parameter> parameterMatching(final String name, final String value) { - return new TypeSafeMatcher<Parameter>() { - @Override - protected boolean matchesSafely(Parameter parameter) { - return parameter.getName().equals(name) && parameter.getValue().equals(value); - } - - @Override - public void describeTo(Description description) { - description.appendText("Parameter with name ").appendValue(name).appendText(" with value ").appendValue(value); - } - }; - } - - private static TypeSafeMatcher<Parameter> parameterMatching(final Parameter param) { - return parameterMatching(param.getName(), param.getValue()); - } } diff --git a/bundle-plugin/src/test/java/com/yahoo/container/plugin/osgi/ImportPackageTest.java b/bundle-plugin/src/test/java/com/yahoo/container/plugin/osgi/ImportPackageTest.java index d6b5b0fe9e0..ee7b6c9c2f1 100644 --- a/bundle-plugin/src/test/java/com/yahoo/container/plugin/osgi/ImportPackageTest.java +++ b/bundle-plugin/src/test/java/com/yahoo/container/plugin/osgi/ImportPackageTest.java @@ -4,25 +4,19 @@ package com.yahoo.container.plugin.osgi; import com.yahoo.container.plugin.osgi.ExportPackages.Export; import com.yahoo.container.plugin.osgi.ExportPackages.Parameter; import com.yahoo.container.plugin.osgi.ImportPackages.Import; -import org.hamcrest.Description; -import org.hamcrest.TypeSafeMatcher; import org.junit.Rule; import org.junit.Test; import org.junit.rules.ExpectedException; import java.util.Collections; import java.util.HashSet; +import java.util.List; import java.util.Map; import java.util.Optional; import java.util.Set; -import static java.util.Collections.emptyMap; -import static java.util.Collections.emptySet; -import static java.util.Collections.singletonList; -import static org.hamcrest.Matchers.contains; -import static org.hamcrest.Matchers.empty; -import static org.hamcrest.Matchers.is; -import static org.junit.Assert.assertThat; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; /** * @author Tony Vaagenes @@ -30,58 +24,56 @@ import static org.junit.Assert.assertThat; */ public class ImportPackageTest { private static final String PACKAGE_NAME = "com.yahoo.exported"; - private Set<String> referencedPackages = Collections.singleton(PACKAGE_NAME); - private Map<String, Export> exports = exportByPackageName(new Export(singletonList(PACKAGE_NAME), Collections.emptyList())); - private Map<String, Export> exportsWithVersion = exportByPackageName( - new Export(singletonList(PACKAGE_NAME), singletonList(new Parameter("version", "1.3")))); + private final Set<String> referencedPackages = Collections.singleton(PACKAGE_NAME); + private final Map<String, Export> exports = exportByPackageName(new Export(List.of(PACKAGE_NAME), Collections.emptyList())); + private final Map<String, Export> exportsWithVersion = exportByPackageName( + new Export(List.of(PACKAGE_NAME), List.of(new Parameter("version", "1.3")))); private static Map<String, Export> exportByPackageName(Export export) { - return ExportPackages.exportsByPackageName(singletonList(export)); + return ExportPackages.exportsByPackageName(List.of(export)); } @Test public void require_that_non_implemented_import_with_matching_export_is_included() { - Set<Import> imports = calculateImports(referencedPackages, emptySet(), exports); - assertThat(imports, contains(importMatching(PACKAGE_NAME, ""))); + Set<Import> imports = calculateImports(referencedPackages, Set.of(), exports); + assertEquals(1, imports.stream().filter(imp -> imp.packageName().equals(PACKAGE_NAME) && imp.version().isEmpty()).count()); } @Test public void require_that_non_implemented_import_without_matching_export_is_excluded() { - Set<Import> imports = calculateImports(referencedPackages, emptySet(), emptyMap()); - assertThat(imports, empty()); + Set<Import> imports = calculateImports(referencedPackages, Set.of(), Map.of()); + assertTrue(imports.isEmpty()); } @Test public void require_that_implemented_import_with_matching_export_is_excluded() { Set<Import> imports = calculateImports(referencedPackages, referencedPackages, exports); - - assertThat(imports, empty()); + assertTrue(imports.isEmpty()); } @Test public void require_that_version_is_included() { - Set<Import> imports = calculateImports(referencedPackages, emptySet(), exportsWithVersion); - - assertThat(imports, contains(importMatching(PACKAGE_NAME, "1.3"))); + Set<Import> imports = calculateImports(referencedPackages, Set.of(), exportsWithVersion); + assertEquals(1, imports.stream().filter(imp -> imp.packageName().equals(PACKAGE_NAME) && imp.version().equals("1.3")).count()); } @Test public void require_that_all_versions_up_to_the_next_major_version_is_in_range() { - assertThat(new Import("foo", Optional.of("1.2")).importVersionRange().get(), is("[1.2,2)")); + assertEquals("[1.2,2)", new Import("foo", Optional.of("1.2")).importVersionRange().get()); } // TODO: Detecting guava packages should be based on bundle-symbolicName, not package name. @Test public void require_that_for_guava_all_future_major_versions_are_in_range() { Optional<String> rangeWithInfiniteUpperLimit = Optional.of("[18.1," + ImportPackages.INFINITE_VERSION + ")"); - assertThat(new Import("com.google.common", Optional.of("18.1")).importVersionRange(), is(rangeWithInfiniteUpperLimit)); - assertThat(new Import("com.google.common.foo", Optional.of("18.1")).importVersionRange(), is(rangeWithInfiniteUpperLimit)); - assertThat(new Import("com.google.commonality", Optional.of("18.1")).importVersionRange(), is(Optional.of("[18.1,19)"))); + assertEquals(rangeWithInfiniteUpperLimit, new Import("com.google.common", Optional.of("18.1")).importVersionRange()); + assertEquals(rangeWithInfiniteUpperLimit, new Import("com.google.common.foo", Optional.of("18.1")).importVersionRange()); + assertEquals(Optional.of("[18.1,19)"), new Import("com.google.commonality", Optional.of("18.1")).importVersionRange()); } @Test public void require_that_none_version_gives_non_version_range() { - assertThat(new Import("foo", Optional.empty()).importVersionRange(), is(Optional.empty())); + assertTrue(new Import("foo", Optional.empty()).importVersionRange().isEmpty()); } @Rule @@ -95,18 +87,17 @@ public class ImportPackageTest { @Test public void require_that_osgi_import_supports_missing_version() { - assertThat(new Import("com.yahoo.exported", Optional.empty()).asOsgiImport(), is("com.yahoo.exported")); + assertEquals("com.yahoo.exported", new Import("com.yahoo.exported", Optional.empty()).asOsgiImport()); } @Test public void require_that_osgi_import_version_range_includes_all_versions_from_the_current_up_to_the_next_major_version() { - assertThat(new Import("com.yahoo.exported", Optional.of("1.2")).asOsgiImport(), is("com.yahoo.exported;version=\"[1.2,2)\"")); + assertEquals("com.yahoo.exported;version=\"[1.2,2)\"", new Import("com.yahoo.exported", Optional.of("1.2")).asOsgiImport()); } @Test public void require_that_osgi_import_version_range_ignores_qualifier() { - assertThat(new Import("com.yahoo.exported", Optional.of("1.2.3.qualifier")).asOsgiImport(), - is("com.yahoo.exported;version=\"[1.2.3,2)\"")); + assertEquals("com.yahoo.exported;version=\"[1.2.3,2)\"", new Import("com.yahoo.exported", Optional.of("1.2.3.qualifier")).asOsgiImport()); } private static Set<Import> calculateImports(Set<String> referencedPackages, @@ -114,18 +105,4 @@ public class ImportPackageTest { Map<String, Export> exportedPackages) { return new HashSet<>(ImportPackages.calculateImports(referencedPackages, implementedPackages, exportedPackages).values()); } - - private static TypeSafeMatcher<Import> importMatching(String packageName, String version) { - return new TypeSafeMatcher<Import>() { - @Override - protected boolean matchesSafely(Import anImport) { - return anImport.packageName().equals(packageName) && anImport.version().equals(version); - } - - @Override - public void describeTo(Description description) { - description.appendText("an Import of package ").appendValue(packageName).appendText(" with version ").appendValue(version); - } - }; - } } diff --git a/cloud-tenant-base-dependencies-enforcer/pom.xml b/cloud-tenant-base-dependencies-enforcer/pom.xml index da50fadbd16..83434209615 100644 --- a/cloud-tenant-base-dependencies-enforcer/pom.xml +++ b/cloud-tenant-base-dependencies-enforcer/pom.xml @@ -43,7 +43,7 @@ <hk2.version>2.5.0-b32</hk2.version> <hk2.osgi-resource-locator.version>1.0.1</hk2.osgi-resource-locator.version> <httpclient5.version>5.1.2</httpclient5.version> - <jackson2.version>2.12.1</jackson2.version> + <jackson2.version>2.12.6</jackson2.version> <jackson-databind.version>${jackson2.version}</jackson-databind.version> <javassist.version>3.20.0-GA</javassist.version> <javax.annotation-api.version>1.2</javax.annotation-api.version> @@ -219,7 +219,6 @@ <include>com.ibm.icu:icu4j:57.1:jar:test</include> <include>com.intellij:annotations:12.0:jar:test</include> <include>com.microsoft.onnxruntime:onnxruntime:[${onnxruntime.version}]:jar:test</include> - <include>com.optimaize.languagedetector:language-detector:0.6:jar:test</include> <include>com.thaiopensource:jing:20091111:jar:test</include> <include>com.yahoo.athenz:athenz-auth-core:[${athenz.version}]:jar:test</include> <include>com.yahoo.athenz:athenz-client-common:[${athenz.version}]:jar:test</include> @@ -249,7 +248,7 @@ <include>org.apache.httpcomponents.core5:httpcore5-h2:${httpclient5.version}:jar:test</include> <include>org.apache.httpcomponents:httpclient:4.5.12:jar:test</include> <include>org.apache.httpcomponents:httpcore:4.4.13:jar:test</include> - <include>org.apache.opennlp:opennlp-tools:1.8.4:jar:test</include> + <include>org.apache.opennlp:opennlp-tools:1.9.3:jar:test</include> <include>org.apiguardian:apiguardian-api:1.1.0:jar:test</include> <include>org.codehaus.woodstox:stax2-api:3.1.4:jar:test</include> <include>org.eclipse.jetty.alpn:alpn-api:[${jetty-alpn.version}]:jar:test</include> @@ -275,7 +274,7 @@ <include>org.kohsuke:libpam4j:1.11:jar:test</include> <include>org.opentest4j:opentest4j:1.2.0:jar:test</include> <include>software.amazon.ion:ion-java:1.0.2:jar:test</include> - <include>xerces:xercesImpl:2.12.0:jar:test</include> + <include>xerces:xercesImpl:2.12.1:jar:test</include> </includes> </bannedDependencies> </rules> diff --git a/clustercontroller-reindexer/src/main/java/ai/vespa/reindexing/Reindexer.java b/clustercontroller-reindexer/src/main/java/ai/vespa/reindexing/Reindexer.java index fb59cb73ba2..4fe7ee74d6d 100644 --- a/clustercontroller-reindexer/src/main/java/ai/vespa/reindexing/Reindexer.java +++ b/clustercontroller-reindexer/src/main/java/ai/vespa/reindexing/Reindexer.java @@ -206,9 +206,9 @@ public class Reindexer { VisitorParameters createParameters(DocumentType type, double speed, ProgressToken progress) { VisitorParameters parameters = new VisitorParameters(type.getName()); parameters.setThrottlePolicy(new DynamicThrottlePolicy().setWindowSizeIncrement(speed) - .setWindowSizeDecrementFactor(5) - .setResizeRate(10) - .setMinWindowSize((int) (5 * speed))); + .setWindowSizeDecrementFactor(3) + .setResizeRate(5) + .setMinWindowSize(3 + (int) (5 * speed))); parameters.setRemoteDataHandler(cluster.name()); parameters.setMaxPending(8); parameters.setResumeToken(progress); diff --git a/config-application-package/src/test/java/com/yahoo/config/model/application/provider/FilesApplicationPackageTest.java b/config-application-package/src/test/java/com/yahoo/config/model/application/provider/FilesApplicationPackageTest.java index 2a70799ab3d..ec68ed73864 100644 --- a/config-application-package/src/test/java/com/yahoo/config/model/application/provider/FilesApplicationPackageTest.java +++ b/config-application-package/src/test/java/com/yahoo/config/model/application/provider/FilesApplicationPackageTest.java @@ -9,7 +9,6 @@ import com.yahoo.config.provision.Zone; import com.yahoo.io.IOUtils; import org.junit.Rule; import org.junit.Test; -import org.junit.rules.ExpectedException; import org.junit.rules.TemporaryFolder; import java.io.File; @@ -17,10 +16,10 @@ import java.io.FileReader; import java.io.IOException; import java.nio.file.Files; -import static org.hamcrest.CoreMatchers.containsString; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; /** * @author Ulf Lilleengen @@ -30,9 +29,6 @@ public class FilesApplicationPackageTest { @Rule public TemporaryFolder temporaryFolder = new TemporaryFolder(); - @Rule - public ExpectedException expectedException = ExpectedException.none(); - @Test public void testPreprocessing() throws IOException { File appDir = temporaryFolder.newFolder(); @@ -112,9 +108,12 @@ public class FilesApplicationPackageTest { IOUtils.copyDirectory(new File("src/test/resources/multienvapp"), appDir); Files.delete(new File(appDir, "services.xml").toPath()); FilesApplicationPackage app = FilesApplicationPackage.fromFile(appDir); - expectedException.expect(IllegalArgumentException.class); - expectedException.expectMessage(containsString("services.xml in application package is empty")); - app.preprocess(new Zone(Environment.dev, RegionName.defaultName()), new BaseDeployLogger()); + try { + app.preprocess(new Zone(Environment.dev, RegionName.defaultName()), new BaseDeployLogger()); + fail(); + } catch (IllegalArgumentException e) { + assertTrue(e.getMessage().contains("services.xml in application package is empty")); + } } } diff --git a/config-lib/src/test/java/com/yahoo/config/ConfigInstanceBuilderTest.java b/config-lib/src/test/java/com/yahoo/config/ConfigInstanceBuilderTest.java index 7b2c2757b56..613cfe1cced 100644 --- a/config-lib/src/test/java/com/yahoo/config/ConfigInstanceBuilderTest.java +++ b/config-lib/src/test/java/com/yahoo/config/ConfigInstanceBuilderTest.java @@ -26,14 +26,10 @@ import static com.yahoo.test.FunctionTestConfig.RootStruct; import static com.yahoo.test.FunctionTestConfig.MyStructMap; import static com.yahoo.foo.MaptypesConfig.Innermap; -import static org.hamcrest.CoreMatchers.containsString; -import static org.hamcrest.CoreMatchers.is; -import static org.hamcrest.CoreMatchers.startsWith; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertNull; -import static org.junit.Assert.assertThat; import static org.junit.Assert.assertTrue; /** @@ -51,8 +47,8 @@ public class ConfigInstanceBuilderTest .gender(FEMALE); StructtypesConfig config = builder.build(); - assertThat(config.simple().name(), is("myname")); - assertThat(config.simple().gender(), is(FEMALE)); + assertEquals("myname", config.simple().name()); + assertEquals(FEMALE, config.simple().gender()); } @Test @@ -65,8 +61,8 @@ public class ConfigInstanceBuilderTest builder.intmap(newMap); MaptypesConfig config = new MaptypesConfig(builder); - assertThat(config.intmap("one"), is(1)); - assertThat(config.intmap("two"), is(2)); + assertEquals(1, config.intmap("one")); + assertEquals(2, config.intmap("two")); } @Test @@ -80,8 +76,8 @@ public class ConfigInstanceBuilderTest builder.innermap(newMap); MaptypesConfig config = new MaptypesConfig(builder); - assertThat(config.innermap("one").foo(), is(1)); - assertThat(config.innermap("two").foo(), is(2)); + assertEquals(1, config.innermap("one").foo()); + assertEquals(2, config.innermap("two").foo()); } @Test @@ -262,12 +258,12 @@ public class ConfigInstanceBuilderTest assertEquals("etc", config.fileVal().value()); assertEquals(1, config.boolarr().size()); assertEquals(1, config.boolarr().size()); // new api with accessor for a List of the original Java type - assertEquals(false, config.boolarr().get(0)); // new List api - assertEquals(false, config.boolarr(0)); // short-hand + assertFalse(config.boolarr().get(0)); // new List api + assertFalse(config.boolarr(0)); // short-hand assertEquals(0, config.intarr().size()); assertEquals(2, config.longarr().size()); assertEquals(Long.MAX_VALUE, config.longarr(0)); - assertThat(config.longarr().get(1), is(Long.MIN_VALUE)); + assertEquals(Long.MIN_VALUE, config.longarr().get(1).longValue()); assertEquals(2, config.doublearr().size()); assertEquals(1, config.stringarr().size()); assertEquals(1, config.enumarr().size()); @@ -279,25 +275,25 @@ public class ConfigInstanceBuilderTest assertEquals("parent:", config.refarr(2)); assertEquals("bin", config.fileArr(0).value()); - assertThat(config.intMap("one"), is(1)); - assertThat(config.intMap("two"), is(2)); - assertThat(config.stringMap("one"), is("first")); - assertThat(config.filemap("f1").value(), is("/var")); - assertThat(config.filemap("f2").value(), is("/store")); + assertEquals(1, config.intMap("one")); + assertEquals(2, config.intMap("two")); + assertEquals("first", config.stringMap("one")); + assertEquals("/var", config.filemap("f1").value()); + assertEquals("/store", config.filemap("f2").value()); assertEquals("basicFoo", config.basicStruct().foo()); assertEquals(3, config.basicStruct().bar()); // new List api assertEquals(2, config.basicStruct().intArr().size()); - assertThat(config.basicStruct().intArr().get(0), is(310)); // new List api - assertThat(config.basicStruct().intArr().get(1), is(311)); // new List api + assertEquals(310, config.basicStruct().intArr().get(0).intValue()); // new List api + assertEquals(311, config.basicStruct().intArr().get(1).intValue()); // new List api assertEquals(310, config.basicStruct().intArr(0)); // short-hand assertEquals("inner0", config.rootStruct().inner0().name()); // new List api assertEquals(11, config.rootStruct().inner0().index()); assertEquals("inner1", config.rootStruct().inner1().name()); assertEquals(12, config.rootStruct().inner1().index()); assertEquals(2, config.rootStruct().innerArr().size()); - assertEquals(true, config.rootStruct().innerArr(0).boolVal()); + assertTrue(config.rootStruct().innerArr(0).boolVal()); assertEquals("deep", config.rootStruct().innerArr(0).stringVal()); - assertEquals(false, config.rootStruct().innerArr(1).boolVal()); + assertFalse(config.rootStruct().innerArr(1).boolVal()); assertEquals("blue a=\"escaped\"", config.rootStruct().innerArr(1).stringVal()); assertEquals(2, config.myarray().size()); // new List api @@ -311,12 +307,12 @@ public class ConfigInstanceBuilderTest assertEquals(-1, config.myarray(1).myStruct().a()); assertEquals(-2, config.myarray(1).myStruct().b()); - assertThat(config.myStructMap("one").myInt(), is(1)); - assertThat(config.myStructMap("one").myString(), is("bull")); - assertThat(config.myStructMap("one").myIntDef(), is(2)); - assertThat(config.myStructMap("one").myStringDef(), is("bear")); - assertThat(config.myStructMap("one").anotherMap("anotherOne").anInt(), is(3)); - assertThat(config.myStructMap("one").anotherMap("anotherOne").anIntDef(), is(4)); + assertEquals(1, config.myStructMap("one").myInt()); + assertEquals("bull", config.myStructMap("one").myString()); + assertEquals(2, config.myStructMap("one").myIntDef()); + assertEquals("bear", config.myStructMap("one").myStringDef()); + assertEquals(3, config.myStructMap("one").anotherMap("anotherOne").anInt()); + assertEquals(4, config.myStructMap("one").anotherMap("anotherOne").anIntDef()); } private boolean callContainsFieldsFlaggedWithRestart(Class<?> configClass) @@ -372,9 +368,8 @@ public class ConfigInstanceBuilderTest report = callGetChangesRequiringRestart(function1, function2); assertTrue(report.needsRestart()); assertEquals("function-test", report.getName()); - assertThat( - report.toString(), - startsWith( + assertTrue( + report.toString().startsWith( "# An int value\n" + "# Also test that multiline comments\n" + "# work.\n" + @@ -398,9 +393,8 @@ public class ConfigInstanceBuilderTest ) ); - assertThat( - report.toString(), - containsString( + assertTrue( + report.toString().contains( "function-test.myStructMap{one}.myInt has changed from 1 to 42\n" + "function-test.myStructMap{new} was added with value \n" ) @@ -410,9 +404,8 @@ public class ConfigInstanceBuilderTest FunctionTestConfig function3 = new FunctionTestConfig(funcBuilder); report = callGetChangesRequiringRestart(function2, function3); assertEquals(1, report.getReportLines().size()); - assertThat( - report.toString(), - containsString("function-test.myStructMap{one} with value \n") + assertTrue( + report.toString().contains("function-test.myStructMap{one} with value \n") ); } } diff --git a/config-lib/src/test/java/com/yahoo/config/ConfigInstanceEqualsTest.java b/config-lib/src/test/java/com/yahoo/config/ConfigInstanceEqualsTest.java index 1ca33b8628a..536c18786da 100644 --- a/config-lib/src/test/java/com/yahoo/config/ConfigInstanceEqualsTest.java +++ b/config-lib/src/test/java/com/yahoo/config/ConfigInstanceEqualsTest.java @@ -14,10 +14,8 @@ import static com.yahoo.test.FunctionTestConfig.Enum_val; import static com.yahoo.test.FunctionTestConfig.Enumarr; import static com.yahoo.test.FunctionTestConfig.Myarray; import static com.yahoo.test.FunctionTestConfig.RootStruct; -import static org.hamcrest.CoreMatchers.is; -import static org.hamcrest.CoreMatchers.not; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertThat; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotEquals; public class ConfigInstanceEqualsTest { FunctionTestConfig config1; @@ -33,90 +31,90 @@ public class ConfigInstanceEqualsTest { @Test public void require_same_hashCode_for_equal_instances() { - assertThat(config1.hashCode(), is(config2.hashCode())); + assertEquals(config1.hashCode(), config2.hashCode()); } @Test public void require_true_for_equal_instances() { - assertThat(config1, is(config2)); + assertEquals(config1, config2); } @Test public void require_false_for_null() { - assertThat(config1, not((FunctionTestConfig) null)); + assertNotEquals(null, config1); } @Test public void require_false_for_different_subclass() { - assertFalse(config1.equals(new AppConfig(new AppConfig.Builder()))); + assertNotEquals(config1, new AppConfig(new AppConfig.Builder())); } @Test public void require_false_for_different_scalars_at_root_node() { - assertThat(config1, not(new FunctionTestConfig(newBuilder().bool_val(true)))); - assertThat(config1, not(new FunctionTestConfig(newBuilder().int_val(0)))); - assertThat(config1, not(new FunctionTestConfig(newBuilder().long_val(0L)))); - assertThat(config1, not(new FunctionTestConfig(newBuilder().double_val(0.0)))); - assertThat(config1, not(new FunctionTestConfig(newBuilder().string_val("")))); - assertThat(config1, not(new FunctionTestConfig(newBuilder().enum_val(Enum_val.FOO)))); - assertThat(config1, not(new FunctionTestConfig(newBuilder().refval("")))); - assertThat(config1, not(new FunctionTestConfig(newBuilder().fileVal("")))); + assertNotEquals(config1, new FunctionTestConfig(newBuilder().bool_val(true))); + assertNotEquals(config1, new FunctionTestConfig(newBuilder().int_val(0))); + assertNotEquals(config1, new FunctionTestConfig(newBuilder().long_val(0L))); + assertNotEquals(config1, new FunctionTestConfig(newBuilder().double_val(0.0))); + assertNotEquals(config1, new FunctionTestConfig(newBuilder().string_val(""))); + assertNotEquals(config1, new FunctionTestConfig(newBuilder().enum_val(Enum_val.FOO))); + assertNotEquals(config1, new FunctionTestConfig(newBuilder().refval(""))); + assertNotEquals(config1, new FunctionTestConfig(newBuilder().fileVal(""))); } @Test public void require_false_for_different_leaf_array_at_root_node() { builder2.longarr.set(0, 0L); - assertThat(config1, not(new FunctionTestConfig(builder2))); + assertNotEquals(config1, new FunctionTestConfig(builder2)); } @Test public void require_false_for_different_scalar_in_struct() { builder2.basicStruct(new BasicStruct.Builder(config1.basicStruct()).bar(0)); - assertThat(config1, not(new FunctionTestConfig(builder2))); + assertNotEquals(config1, new FunctionTestConfig(builder2)); } @Test public void require_false_for_different_scalar_in_inner_array() { builder2.myarray.get(0).intval(0); - assertThat(config1, not(new FunctionTestConfig(builder2))); + assertNotEquals(config1, new FunctionTestConfig(builder2)); } @Test public void require_false_for_different_leaf_array_in_inner_array() { builder2.myarray.get(0).stringval.set(0, ""); - assertThat(config1, not(new FunctionTestConfig(builder2))); + assertNotEquals(config1, new FunctionTestConfig(builder2)); } @Test public void require_equal_structs_for_equal_configs() { - assertThat(config1.basicStruct(), is(config2.basicStruct())); - assertThat(config1.rootStruct(), is(config2.rootStruct())); - assertThat(config1.rootStruct().inner0(), is(config2.rootStruct().inner0())); + assertEquals(config1.basicStruct(), config2.basicStruct()); + assertEquals(config1.rootStruct(), config2.rootStruct()); + assertEquals(config1.rootStruct().inner0(), config2.rootStruct().inner0()); } @Test public void require_equal_inner_arrays_for_equal_configs() { - assertThat(config1.myarray(), is(config2.myarray())); - assertThat(config1.myarray(0).anotherarray(), is(config2.myarray(0).anotherarray())); + assertEquals(config1.myarray(), config2.myarray()); + assertEquals(config1.myarray(0).anotherarray(), config2.myarray(0).anotherarray()); } @Test public void require_equal_inner_array_elements_for_equal_configs() { - assertThat(config1.myarray(0), is(config2.myarray(0))); - assertThat(config1.myarray(0).anotherarray(0), is(config2.myarray(0).anotherarray(0))); + assertEquals(config1.myarray(0), config2.myarray(0)); + assertEquals(config1.myarray(0).anotherarray(0), config2.myarray(0).anotherarray(0)); } @Test public void require_equal_leaf_arrays_for_equal_configs() { - assertThat(config1.intarr(), is(config2.intarr())); - assertThat(config1.boolarr(), is(config2.boolarr())); - assertThat(config1.longarr(), is(config2.longarr())); - assertThat(config1.doublearr(), is(config2.doublearr())); - assertThat(config1.stringarr(), is(config2.stringarr())); - assertThat(config1.enumarr(), is(config2.enumarr())); - assertThat(config1.refarr(), is(config2.refarr())); - assertThat(config1.fileArr(), is(config2.fileArr())); + assertEquals(config1.intarr(), config2.intarr()); + assertEquals(config1.boolarr(), config2.boolarr()); + assertEquals(config1.longarr(), config2.longarr()); + assertEquals(config1.doublearr(), config2.doublearr()); + assertEquals(config1.stringarr(), config2.stringarr()); + assertEquals(config1.enumarr(), config2.enumarr()); + assertEquals(config1.refarr(), config2.refarr()); + assertEquals(config1.fileArr(), config2.fileArr()); } private static FunctionTestConfig.Builder newBuilder() { diff --git a/config-lib/src/test/java/com/yahoo/config/DoubleNodeTest.java b/config-lib/src/test/java/com/yahoo/config/DoubleNodeTest.java index 42819957a1d..1192cc6673e 100644 --- a/config-lib/src/test/java/com/yahoo/config/DoubleNodeTest.java +++ b/config-lib/src/test/java/com/yahoo/config/DoubleNodeTest.java @@ -3,8 +3,10 @@ package com.yahoo.config; import org.junit.Test; -import static org.hamcrest.CoreMatchers.is; -import static org.junit.Assert.*; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + /** * @author Ulf Lilleengen diff --git a/config-lib/src/test/java/com/yahoo/config/EnumNodeTest.java b/config-lib/src/test/java/com/yahoo/config/EnumNodeTest.java index b9ebaf1f4dd..fac765f40aa 100644 --- a/config-lib/src/test/java/com/yahoo/config/EnumNodeTest.java +++ b/config-lib/src/test/java/com/yahoo/config/EnumNodeTest.java @@ -3,8 +3,10 @@ package com.yahoo.config; import org.junit.Test; -import static org.hamcrest.CoreMatchers.is; -import static org.junit.Assert.*; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertTrue; /** * @author Ulf Lilleengen @@ -31,10 +33,10 @@ public class EnumNodeTest { public void testEnumNode() { MyNode n = new MyNode(); assertNull(n.getValue()); - assertThat(n.toString(), is("(null)")); + assertEquals("(null)", n.toString()); assertTrue(n.doSetValue("ONE")); - assertThat(n.getValue(), is("ONE")); - assertThat(n.toString(), is("ONE")); + assertEquals("ONE", n.getValue()); + assertEquals("ONE", n.toString()); assertFalse(n.doSetValue("THREE")); } } diff --git a/config-lib/src/test/java/com/yahoo/config/FileNodeTest.java b/config-lib/src/test/java/com/yahoo/config/FileNodeTest.java index 67edf5e9631..56dd7dd116d 100644 --- a/config-lib/src/test/java/com/yahoo/config/FileNodeTest.java +++ b/config-lib/src/test/java/com/yahoo/config/FileNodeTest.java @@ -3,9 +3,7 @@ package com.yahoo.config; import org.junit.Test; -import static org.hamcrest.CoreMatchers.is; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertThat; +import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; /** @@ -16,11 +14,11 @@ public class FileNodeTest { @Test public void testSetValue() { FileNode n = new FileNode(); - assertThat(n.toString(), is("(null)")); + assertEquals("(null)", n.toString()); assertTrue(n.doSetValue("foo.txt")); - assertThat(n.value().value(), is("foo.txt")); + assertEquals("foo.txt", n.value().value()); assertTrue(n.doSetValue("\"foo.txt\"")); - assertThat(n.value().value(), is("foo.txt")); - assertThat(n.toString(), is("\"foo.txt\"")); + assertEquals("foo.txt", n.value().value()); + assertEquals("\"foo.txt\"", n.toString()); } } diff --git a/config-lib/src/test/java/com/yahoo/config/IntegerNodeTest.java b/config-lib/src/test/java/com/yahoo/config/IntegerNodeTest.java index c8404b8791e..899e34edce4 100644 --- a/config-lib/src/test/java/com/yahoo/config/IntegerNodeTest.java +++ b/config-lib/src/test/java/com/yahoo/config/IntegerNodeTest.java @@ -3,9 +3,8 @@ package com.yahoo.config; import org.junit.Test; -import static org.hamcrest.CoreMatchers.is; +import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertThat; import static org.junit.Assert.assertTrue; /** @@ -18,6 +17,6 @@ public class IntegerNodeTest { IntegerNode n = new IntegerNode(); assertFalse(n.setValue("invalid")); assertTrue(n.setValue("10")); - assertThat(n.value(), is(10)); + assertEquals(10, n.value().intValue()); } } diff --git a/config-lib/src/test/java/com/yahoo/config/LongNodeTest.java b/config-lib/src/test/java/com/yahoo/config/LongNodeTest.java index c3f9ff9ae8f..fc603ba65ac 100644 --- a/config-lib/src/test/java/com/yahoo/config/LongNodeTest.java +++ b/config-lib/src/test/java/com/yahoo/config/LongNodeTest.java @@ -3,9 +3,8 @@ package com.yahoo.config; import org.junit.Test; -import static org.hamcrest.CoreMatchers.is; import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertThat; +import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; /** @@ -18,6 +17,6 @@ public class LongNodeTest { LongNode n = new LongNode(); assertFalse(n.setValue("invalid")); assertTrue(n.setValue("10")); - assertThat(n.value(), is(10l)); + assertEquals(10L, n.value().longValue()); } } diff --git a/config-lib/src/test/java/com/yahoo/config/NodeVectorTest.java b/config-lib/src/test/java/com/yahoo/config/NodeVectorTest.java index cb9bd65c406..9637ab25278 100644 --- a/config-lib/src/test/java/com/yahoo/config/NodeVectorTest.java +++ b/config-lib/src/test/java/com/yahoo/config/NodeVectorTest.java @@ -5,9 +5,8 @@ import org.junit.Test; import java.util.Arrays; -import static org.hamcrest.CoreMatchers.is; import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertThat; +import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; /** @@ -61,10 +60,10 @@ public class NodeVectorTest { StringNode val = new StringNode("foo"); TestNodeVector v = new TestNodeVector(val.getValue()); assertFalse(v.isEmpty()); - assertThat(v.indexOf(val), is(0)); - assertThat(v.indexOf(barNode()), is(-1)); - assertThat(v.lastIndexOf(val), is(0)); - assertThat(v.lastIndexOf(barNode()), is(-1)); + assertEquals(0, v.indexOf(val)); + assertEquals(-1, v.indexOf(barNode())); + assertEquals(0, v.lastIndexOf(val)); + assertEquals(-1, v.lastIndexOf(barNode())); } @Test @@ -82,11 +81,11 @@ public class NodeVectorTest { public void require_that_sublisting_works() { String val = "foo"; TestNodeVector v = new TestNodeVector(val, val, val); - assertThat(v.subList(0, 1).size(), is(1)); - assertThat(v.subList(0, 2).size(), is(2)); - assertThat(v.subList(0, 3).size(), is(3)); + assertEquals(1, v.subList(0, 1).size()); + assertEquals(2, v.subList(0, 2).size()); + assertEquals(3, v.subList(0, 3).size()); StringNode[] vals = v.toArray(new StringNode[0]); - assertThat(vals.length, is(3)); + assertEquals(3, vals.length); } private StringNode barNode() { return new StringNode("bar");} diff --git a/config-lib/src/test/java/com/yahoo/config/PathNodeTest.java b/config-lib/src/test/java/com/yahoo/config/PathNodeTest.java index 7bb465f203b..37313bbcdf3 100644 --- a/config-lib/src/test/java/com/yahoo/config/PathNodeTest.java +++ b/config-lib/src/test/java/com/yahoo/config/PathNodeTest.java @@ -5,8 +5,7 @@ import org.junit.Test; import java.io.File; -import static org.hamcrest.CoreMatchers.is; -import static org.junit.Assert.assertThat; +import static org.junit.Assert.assertEquals; /** * @author gjoranv @@ -17,10 +16,10 @@ public class PathNodeTest { @Test public void testSetValue() { PathNode n = new PathNode(); - assertThat(n.toString(), is("(null)")); + assertEquals("(null)", n.toString()); n = new PathNode(new FileReference("foo.txt")); - assertThat(n.value(), is(new File("foo.txt").toPath())); + assertEquals(new File("foo.txt").toPath(), n.value()); } } diff --git a/config-lib/src/test/java/com/yahoo/config/StringNodeTest.java b/config-lib/src/test/java/com/yahoo/config/StringNodeTest.java index 05b9361865c..5d606e12454 100644 --- a/config-lib/src/test/java/com/yahoo/config/StringNodeTest.java +++ b/config-lib/src/test/java/com/yahoo/config/StringNodeTest.java @@ -3,8 +3,7 @@ package com.yahoo.config; import org.junit.Test; -import static org.hamcrest.core.Is.is; -import static org.junit.Assert.assertThat; +import static org.junit.Assert.assertEquals; /** * @author hmusum @@ -15,11 +14,11 @@ public class StringNodeTest { @Test public void testUnescapeQuotedString() { String a = "\"Hei\""; - assertThat(StringNode.unescapeQuotedString(a), is("Hei")); - assertThat(StringNode.unescapeQuotedString("foo\"bar\""), is("foo\"bar\"")); - assertThat(StringNode.unescapeQuotedString("foo\\\"bar\\\""), is("foo\"bar\"")); - assertThat(StringNode.unescapeQuotedString("a\\rb\\tc\\fd"), is("a\rb\tc\fd")); - assertThat(StringNode.unescapeQuotedString("\\x55"), is("U")); + assertEquals("Hei", StringNode.unescapeQuotedString(a)); + assertEquals("foo\"bar\"", StringNode.unescapeQuotedString("foo\"bar\"")); + assertEquals("foo\"bar\"", StringNode.unescapeQuotedString("foo\\\"bar\\\"")); + assertEquals("a\rb\tc\fd", StringNode.unescapeQuotedString("a\\rb\\tc\\fd")); + assertEquals("U", StringNode.unescapeQuotedString("\\x55")); } @Test(expected = IllegalArgumentException.class) @@ -30,17 +29,17 @@ public class StringNodeTest { @Test public void testToString() { StringNode n = new StringNode(); - assertThat(n.toString(), is("(null)")); + assertEquals("(null)", n.toString()); n.setValue("foo"); - assertThat(n.toString(), is("\"foo\"")); + assertEquals("\"foo\"", n.toString()); } @Test public void testSetValue() { StringNode n = new StringNode(); n.setValue("\"foo\""); - assertThat(n.getValue(), is("foo")); + assertEquals("foo", n.getValue()); n.setValue("foo"); - assertThat(n.getValue(), is("foo")); + assertEquals("foo", n.getValue()); } } diff --git a/config-lib/src/test/java/com/yahoo/config/UrlNodeTest.java b/config-lib/src/test/java/com/yahoo/config/UrlNodeTest.java index 040e3304a66..abf04c34bb8 100644 --- a/config-lib/src/test/java/com/yahoo/config/UrlNodeTest.java +++ b/config-lib/src/test/java/com/yahoo/config/UrlNodeTest.java @@ -3,8 +3,7 @@ package com.yahoo.config; import org.junit.Test; -import static org.hamcrest.CoreMatchers.is; -import static org.junit.Assert.assertThat; +import static org.junit.Assert.assertEquals; /** * @author lesters @@ -14,13 +13,13 @@ public class UrlNodeTest { @Test public void testSetValue() { UrlNode url = new UrlNode(); - assertThat(url.toString(), is("(null)")); + assertEquals("(null)", url.toString()); url = new UrlNode(new UrlReference("https://docs.vespa.ai/")); - assertThat(url.getUrlReference().value(), is("https://docs.vespa.ai/")); + assertEquals("https://docs.vespa.ai/", url.getUrlReference().value()); url = new UrlNode(new UrlReference("pom.xml")); - assertThat(url.getValue(), is("pom.xml")); + assertEquals("pom.xml", url.getValue()); } } diff --git a/config-lib/src/test/java/com/yahoo/config/codegen/NamespaceAndPackageTest.java b/config-lib/src/test/java/com/yahoo/config/codegen/NamespaceAndPackageTest.java index 066b5383621..54e2271f7dc 100644 --- a/config-lib/src/test/java/com/yahoo/config/codegen/NamespaceAndPackageTest.java +++ b/config-lib/src/test/java/com/yahoo/config/codegen/NamespaceAndPackageTest.java @@ -6,8 +6,7 @@ import com.github.myproject.PackageConfig; import com.yahoo.my.namespace.NamespaceConfig; import org.junit.Test; -import static org.hamcrest.CoreMatchers.is; -import static org.junit.Assert.assertThat; +import static org.junit.Assert.assertEquals; /** * @author gjoranv @@ -18,17 +17,17 @@ public class NamespaceAndPackageTest { @Test public void namespace_is_set_from_def_file() { - assertThat(NamespaceConfig.CONFIG_DEF_NAMESPACE, is(NAMESPACE)); + assertEquals(NAMESPACE, NamespaceConfig.CONFIG_DEF_NAMESPACE); } @Test public void package_is_used_as_namespace_when_namespace_is_not_set_explicitly() { - assertThat(PackageConfig.CONFIG_DEF_NAMESPACE, is(PACKAGE)); + assertEquals(PACKAGE, PackageConfig.CONFIG_DEF_NAMESPACE); } @Test public void package_does_not_override_namespace() { - assertThat(NamespaceAndPackageConfig.CONFIG_DEF_NAMESPACE, is(NAMESPACE)); + assertEquals(NAMESPACE, NamespaceAndPackageConfig.CONFIG_DEF_NAMESPACE); } } diff --git a/config-model-api/src/main/java/com/yahoo/config/model/api/FileDistribution.java b/config-model-api/src/main/java/com/yahoo/config/model/api/FileDistribution.java index dd4706461ad..f1fc066bce3 100644 --- a/config-model-api/src/main/java/com/yahoo/config/model/api/FileDistribution.java +++ b/config-model-api/src/main/java/com/yahoo/config/model/api/FileDistribution.java @@ -3,7 +3,6 @@ package com.yahoo.config.model.api; import com.yahoo.config.FileReference; -import java.io.File; import java.util.Set; /** @@ -22,7 +21,4 @@ public interface FileDistribution { */ void startDownload(String hostName, int port, Set<FileReference> fileReferences); - // TODO: Remove when 7.508 is latest version in use - default File getFileReferencesDir() {return null; } - } diff --git a/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java b/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java index 3df93f7d08d..6c70af8cbca 100644 --- a/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java +++ b/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java @@ -110,6 +110,9 @@ public interface ModelContext { @ModelFeatureFlag(owners = {"vekterli", "geirst"}) default boolean unorderedMergeChaining() { return false; } @ModelFeatureFlag(owners = {"arnej"}) default boolean useV8GeoPositions() { return false; } @ModelFeatureFlag(owners = {"arnej", "baldersheim"}) default boolean useV8DocManagerCfg() { return false; } + @ModelFeatureFlag(owners = {"baldersheim", "geirst", "toregge"}) default int maxCompactBuffers() { return 1; } + @ModelFeatureFlag(owners = {"hmusum"}) default boolean failDeploymentWithInvalidJvmOptions() { return false; } + @ModelFeatureFlag(owners = {"baldersheim"}) default double tlsSizeFraction() { throw new UnsupportedOperationException("TODO specify default value"); } } /** Warning: As elsewhere in this package, do not make backwards incompatible changes that will break old config models! */ diff --git a/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java b/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java index ee9af73c554..e645fec5520 100644 --- a/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java +++ b/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java @@ -75,6 +75,9 @@ public class TestProperties implements ModelContext.Properties, ModelContext.Fea private boolean asyncApplyBucketDiff = false; private boolean unorderedMergeChaining = false; private List<String> zoneDnsSuffixes = List.of(); + private int maxCompactBuffers = 1; + private boolean failDeploymentWithInvalidJvmOptions = false; + private double tlsSizeFraction = 0.07; @Override public ModelContext.FeatureFlags featureFlags() { return this; } @Override public boolean multitenant() { return multitenant; } @@ -130,6 +133,9 @@ public class TestProperties implements ModelContext.Properties, ModelContext.Fea @Override public boolean asyncApplyBucketDiff() { return asyncApplyBucketDiff; } @Override public boolean unorderedMergeChaining() { return unorderedMergeChaining; } @Override public List<String> zoneDnsSuffixes() { return zoneDnsSuffixes; } + @Override public int maxCompactBuffers() { return maxCompactBuffers; } + @Override public boolean failDeploymentWithInvalidJvmOptions() { return failDeploymentWithInvalidJvmOptions; } + @Override public double tlsSizeFraction() { return tlsSizeFraction; } public TestProperties maxUnCommittedMemory(int maxUnCommittedMemory) { this.maxUnCommittedMemory = maxUnCommittedMemory; @@ -340,6 +346,21 @@ public class TestProperties implements ModelContext.Properties, ModelContext.Fea return this; } + public TestProperties maxCompactBuffers(int maxCompactBuffers) { + this.maxCompactBuffers = maxCompactBuffers; + return this; + } + + public TestProperties failDeploymentWithInvalidJvmOptions(boolean fail) { + failDeploymentWithInvalidJvmOptions = fail; + return this; + } + + public TestProperties tlsSizeFraction(double tlsSizeFraction) { + this.tlsSizeFraction = tlsSizeFraction; + return this; + } + public static class Spec implements ConfigServerSpec { private final String hostName; diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/LogForwarder.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/LogForwarder.java index a01d061c21f..cc1369c2470 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/admin/LogForwarder.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/LogForwarder.java @@ -5,6 +5,7 @@ import com.yahoo.cloud.config.LogforwarderConfig; import com.yahoo.config.model.producer.AbstractConfigProducer; import com.yahoo.vespa.model.AbstractService; import com.yahoo.vespa.model.PortAllocBridge; +import java.util.Optional; public class LogForwarder extends AbstractService implements LogforwarderConfig.Producer { @@ -79,4 +80,13 @@ public class LogForwarder extends AbstractService implements LogforwarderConfig. } } + @Override + public Optional<String> getPreShutdownCommand() { + var builder = new LogforwarderConfig.Builder(); + getConfig(builder); + var cfg = new LogforwarderConfig(builder); + var home = cfg.splunkHome(); + return Optional.of("/usr/bin/env SPLUNK_HOME="+home+" "+home+"/bin/splunk stop"); + } + } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/clustercontroller/ClusterControllerContainer.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/clustercontroller/ClusterControllerContainer.java index 8cdeda0e85f..eafc460bfb5 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/admin/clustercontroller/ClusterControllerContainer.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/clustercontroller/ClusterControllerContainer.java @@ -177,7 +177,8 @@ public class ClusterControllerContainer extends Container implements status -> clusterBuilder.documentTypes( typeName, new ReindexingConfig.Clusters.DocumentTypes.Builder() - .readyAtMillis(status.ready().toEpochMilli()))); + .readyAtMillis(status.ready().toEpochMilli()) + .speed(status.speed()))); } builder.clusters(clusterId, clusterBuilder); } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/VespaMetricSet.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/VespaMetricSet.java index 2189625ef74..be67cbb9dd6 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/VespaMetricSet.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/VespaMetricSet.java @@ -55,6 +55,7 @@ public class VespaMetricSet { metrics.add(new Metric("slobrok.heartbeats.failed.count")); metrics.add(new Metric("logd.processed.lines.count")); metrics.add(new Metric("worker.connections.max")); + metrics.add(new Metric("endpoint.certificate.expiry.seconds")); // Java (JRT) TLS metrics metrics.add(new Metric("jrt.transport.tls-certificate-verification-failures")); diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/QuotaValidator.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/QuotaValidator.java index d1dc2b84c8a..841dbb204fd 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/QuotaValidator.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/QuotaValidator.java @@ -8,7 +8,6 @@ import com.yahoo.config.provision.ClusterSpec; import com.yahoo.config.provision.NodeResources; import com.yahoo.config.provision.SystemName; import com.yahoo.vespa.model.VespaModel; -import org.jetbrains.annotations.NotNull; import java.math.BigDecimal; import java.util.Locale; @@ -57,7 +56,6 @@ public class QuotaValidator extends Validator { throwIfBudgetExceeded(maxSpend, budget, systemName); } - @NotNull private Set<ClusterSpec.Id> adminClusterIds(VespaModel model) { return model.allocatedHosts().getHosts().stream() .map(hostSpec -> hostSpec.membership().orElseThrow().cluster()) diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/ApplicationContainerCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/container/ApplicationContainerCluster.java index e8be43fdc96..c4d420f2d44 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/ApplicationContainerCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/ApplicationContainerCluster.java @@ -167,11 +167,17 @@ public final class ApplicationContainerCluster extends ContainerCluster<Applicat public void setMemoryPercentage(Integer memoryPercentage) { this.memoryPercentage = memoryPercentage; } - /** - * Returns the percentage of host physical memory this application has specified for nodes in this cluster, - * or empty if this is not specified by the application. - */ - public Optional<Integer> getMemoryPercentage() { return Optional.ofNullable(memoryPercentage); } + @Override + public Optional<Integer> getMemoryPercentage() { + if (memoryPercentage != null) { + return Optional.of(memoryPercentage); + } else if (isHostedVespa()) { + return getHostClusterId().isPresent() ? + Optional.of(heapSizePercentageOfTotalNodeMemoryWhenCombinedCluster) : + Optional.of(heapSizePercentageOfTotalNodeMemory); + } + return Optional.empty(); + } /* Create list of endpoints, these will be consumed later by the LBservicesProducer @@ -291,10 +297,6 @@ public final class ApplicationContainerCluster extends ContainerCluster<Applicat .heapsize(1536); if (getMemoryPercentage().isPresent()) { builder.jvm.heapSizeAsPercentageOfPhysicalMemory(getMemoryPercentage().get()); - } else if (isHostedVespa()) { - builder.jvm.heapSizeAsPercentageOfPhysicalMemory(getHostClusterId().isPresent() ? - heapSizePercentageOfTotalNodeMemoryWhenCombinedCluster : - heapSizePercentageOfTotalNodeMemory); } } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerCluster.java index c4f506d81ba..c73a3b2a676 100755 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerCluster.java @@ -171,6 +171,9 @@ public abstract class ContainerCluster<CONTAINER extends Container> addCommonVespaBundles(); + // TODO Vespa 8: remove LoggingRequestHandler.Context (replaced by ThreadedHttpRequestHandler.Context) + addSimpleComponent(com.yahoo.container.jdisc.LoggingRequestHandler.Context.class); + addComponent(new StatisticsComponent()); addSimpleComponent(AccessLog.class); addComponent(new DefaultThreadpoolProvider(this, deployState.featureFlags().metricsproxyNumThreads())); @@ -178,7 +181,7 @@ public abstract class ContainerCluster<CONTAINER extends Container> addSimpleComponent("com.yahoo.container.jdisc.metric.MetricConsumerProviderProvider"); addSimpleComponent("com.yahoo.container.jdisc.metric.MetricProvider"); addSimpleComponent("com.yahoo.container.jdisc.metric.MetricUpdater"); - addSimpleComponent(com.yahoo.container.jdisc.LoggingRequestHandler.Context.class); + addSimpleComponent(com.yahoo.container.jdisc.ThreadedHttpRequestHandler.Context.class); addSimpleComponent(com.yahoo.metrics.simple.MetricManager.class.getName(), null, MetricProperties.BUNDLE_SYMBOLIC_NAME); addSimpleComponent(com.yahoo.metrics.simple.jdisc.JdiscMetricsFactory.class.getName(), null, MetricProperties.BUNDLE_SYMBOLIC_NAME); addSimpleComponent("com.yahoo.container.jdisc.state.StateMonitor"); @@ -650,4 +653,9 @@ public abstract class ContainerCluster<CONTAINER extends Container> public boolean getDeferChangesUntilRestart() { return deferChangesUntilRestart; } + /** + * Returns the percentage of host physical memory this application has specified for nodes in this cluster, + * or empty if this is not specified by the application. + */ + public Optional<Integer> getMemoryPercentage() { return Optional.empty(); } } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/http/JettyHttpServer.java b/config-model/src/main/java/com/yahoo/vespa/model/container/http/JettyHttpServer.java index 8c31eabfde0..d84884665ef 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/http/JettyHttpServer.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/http/JettyHttpServer.java @@ -23,6 +23,7 @@ public class JettyHttpServer extends SimpleComponent implements ServerConfig.Pro private final ContainerCluster<?> cluster; private volatile boolean isHostedVespa; private final List<ConnectorFactory> connectorFactories = new ArrayList<>(); + private final List<String> ignoredUserAgentsList = new ArrayList<>(); public JettyHttpServer(String componentId, ContainerCluster<?> cluster, boolean isHostedVespa) { super(new ComponentModel(componentId, com.yahoo.jdisc.http.server.jetty.JettyHttpServer.class.getName(), null)); @@ -44,10 +45,15 @@ public class JettyHttpServer extends SimpleComponent implements ServerConfig.Pro return Collections.unmodifiableList(connectorFactories); } + public void addIgnoredUserAgent(String userAgent) { + ignoredUserAgentsList.add(userAgent); + } + @Override public void getConfig(ServerConfig.Builder builder) { builder.metric(new ServerConfig.Metric.Builder() .monitoringHandlerPaths(List.of("/state/v1", "/status.html", "/metrics/v2")) + .ignoredUserAgents(ignoredUserAgentsList) .searchHandlerPaths(List.of("/search")) ); if (isHostedVespa) { diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java index 6d184666bdb..c8b2197fc29 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java @@ -93,6 +93,7 @@ import org.w3c.dom.Node; import java.net.URI; import java.security.cert.X509Certificate; import java.util.ArrayList; +import java.util.Arrays; import java.util.Collection; import java.util.Collections; import java.util.List; @@ -1126,26 +1127,49 @@ public class ContainerModelBuilder extends ConfigModelBuilder<ContainerModel> { } } + /** + * Validates JVM GC options and logs a warning if anyone of them has invalid syntax or is an option + * that is unsupported for the running system (e.g. uses CMS options for hosted Vespa, which uses JDK 17) + */ private static class JvmGcOptions { + private static final Pattern validPattern = Pattern.compile("-XX:[+-]*[a-zA-z0-9=]+"); + private static final Pattern invalidCMSPattern = Pattern.compile("-XX:[+-]\\w*CMS[a-zA-z0-9=]+"); + private final DeployState deployState; private final String jvmGcOptions; private final DeployLogger logger; private final boolean isHosted; + private final boolean failDeploymentWithInvalidJvmOptions; public JvmGcOptions(DeployState deployState, String jvmGcOptions) { this.deployState = deployState; this.jvmGcOptions = jvmGcOptions; this.logger = deployState.getDeployLogger(); this.isHosted = deployState.isHosted(); + this.failDeploymentWithInvalidJvmOptions = deployState.featureFlags().failDeploymentWithInvalidJvmOptions(); } private String build() { String options = deployState.getProperties().jvmGCOptions(); if (jvmGcOptions != null) { - log(jvmGcOptions); options = jvmGcOptions; - // TODO: Verify options against lists of allowed and/or disallowed options + String[] optionList = options.split(" "); + List<String> invalidOptions = Arrays.stream(optionList) + .filter(option -> !option.isEmpty()) + .filter(option -> !Pattern.matches(validPattern.pattern(), option)) + .collect(Collectors.toList()); + + if (isHosted) { + // CMS GC options cannot be used in hosted, CMS is unsupported in JDK 17 + invalidOptions.addAll(Arrays.stream(optionList) + .filter(option -> !option.isEmpty()) + .filter(option -> Pattern.matches(invalidCMSPattern.pattern(), option) || + option.equals("-XX:+UseConcMarkSweepGC")) + .collect(Collectors.toList())); + } + + logOrFailInvalidOptions(invalidOptions); } if (options == null || options.isEmpty()) @@ -1154,9 +1178,15 @@ public class ContainerModelBuilder extends ConfigModelBuilder<ContainerModel> { return options; } - private void log(String jvmGcOptions) { - if (isHosted) - logger.logApplicationPackage(Level.INFO, "JVM GC options from services.xml: " + jvmGcOptions); + private void logOrFailInvalidOptions(List<String> options) { + if (options.isEmpty()) return; + + Collections.sort(options); + String message = "Invalid JVM GC options from services.xml: " + String.join(",", options); + if (failDeploymentWithInvalidJvmOptions) + throw new IllegalArgumentException(message); + else + logger.logApplicationPackage(WARNING, message); } } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/ContentSearchCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/content/ContentSearchCluster.java index 13c3c229acb..54d09bacfa9 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/content/ContentSearchCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/content/ContentSearchCluster.java @@ -71,23 +71,24 @@ public class ContentSearchCluster extends AbstractConfigProducer<SearchCluster> private final double defaultDiskBloatFactor; private final int defaultDocStoreCompressionLevel; private final boolean forwardIssuesToQrs; + private final int defaultMaxCompactBuffers; /** Whether the nodes of this cluster also hosts a container cluster in a hosted system */ - private final boolean combined; + private final double fractionOfMemoryReserved; public static class Builder extends VespaDomBuilder.DomConfigProducerBuilder<ContentSearchCluster> { private final Map<String, NewDocumentType> documentDefinitions; private final Set<NewDocumentType> globallyDistributedDocuments; - private final boolean combined; + private final double fractionOfMemoryReserved; private final ResourceLimits resourceLimits; public Builder(Map<String, NewDocumentType> documentDefinitions, Set<NewDocumentType> globallyDistributedDocuments, - boolean combined, ResourceLimits resourceLimits) { + double fractionOfMemoryReserved, ResourceLimits resourceLimits) { this.documentDefinitions = documentDefinitions; this.globallyDistributedDocuments = globallyDistributedDocuments; - this.combined = combined; + this.fractionOfMemoryReserved = fractionOfMemoryReserved; this.resourceLimits = resourceLimits; } @@ -105,7 +106,7 @@ public class ContentSearchCluster extends AbstractConfigProducer<SearchCluster> globallyDistributedDocuments, getFlushOnShutdown(flushOnShutdownElem), syncTransactionLog, - combined); + fractionOfMemoryReserved); ModelElement tuning = clusterElem.childByPath("engine.proton.tuning"); if (tuning != null) { @@ -207,7 +208,7 @@ public class ContentSearchCluster extends AbstractConfigProducer<SearchCluster> Set<NewDocumentType> globallyDistributedDocuments, boolean flushOnShutdown, Boolean syncTransactionLog, - boolean combined) + double fractionOfMemoryReserved) { super(parent, "search"); this.clusterName = clusterName; @@ -216,7 +217,7 @@ public class ContentSearchCluster extends AbstractConfigProducer<SearchCluster> this.flushOnShutdown = flushOnShutdown; this.syncTransactionLog = syncTransactionLog; - this.combined = combined; + this.fractionOfMemoryReserved = fractionOfMemoryReserved; this.feedSequencerType = convertFeedSequencerType(featureFlags.feedSequencerType()); this.feedTaskLimit = featureFlags.feedTaskLimit(); this.feedMasterTaskLimit = featureFlags.feedMasterTaskLimit(); @@ -225,6 +226,7 @@ public class ContentSearchCluster extends AbstractConfigProducer<SearchCluster> this.defaultDiskBloatFactor = featureFlags.diskBloatFactor(); this.defaultDocStoreCompressionLevel = featureFlags.docstoreCompressionLevel(); this.forwardIssuesToQrs = featureFlags.forwardIssuesAsErrors(); + this.defaultMaxCompactBuffers = featureFlags.maxCompactBuffers(); } public void setVisibilityDelay(double delay) { @@ -288,7 +290,8 @@ public class ContentSearchCluster extends AbstractConfigProducer<SearchCluster> Optional<Tuning> tuning = Optional.ofNullable(this.tuning); if (element == null) { searchNode = SearchNode.create(parent, "" + node.getDistributionKey(), node.getDistributionKey(), spec, - clusterName, node, flushOnShutdown, tuning, resourceLimits, parentGroup.isHosted(), combined); + clusterName, node, flushOnShutdown, tuning, resourceLimits, parentGroup.isHosted(), + fractionOfMemoryReserved, deployState.featureFlags().tlsSizeFraction()); searchNode.setHostResource(node.getHostResource()); searchNode.initService(deployState.getDeployLogger()); @@ -296,7 +299,9 @@ public class ContentSearchCluster extends AbstractConfigProducer<SearchCluster> tls.setHostResource(searchNode.getHostResource()); tls.initService(deployState.getDeployLogger()); } else { - searchNode = new SearchNode.Builder(""+node.getDistributionKey(), spec, clusterName, node, flushOnShutdown, tuning, resourceLimits, combined).build(deployState, parent, element.getXml()); + searchNode = new SearchNode.Builder(""+node.getDistributionKey(), spec, clusterName, node, flushOnShutdown, + tuning, resourceLimits, fractionOfMemoryReserved) + .build(deployState, parent, element.getXml()); tls = new TransactionLogServer.Builder(clusterName, syncTransactionLog).build(deployState, searchNode, element.getXml()); } searchNode.setTls(tls); @@ -387,6 +392,7 @@ public class ContentSearchCluster extends AbstractConfigProducer<SearchCluster> .configid(getConfigId()) .visibilitydelay(visibilityDelay) .global(globalDocType); + ddbB.allocation.max_compact_buffers(defaultMaxCompactBuffers); if (hasIndexingModeStreaming(type)) { hasAnyNonIndexedCluster = true; diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/ContentCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/ContentCluster.java index 0c0b2b80d79..628cf6bb4c7 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/ContentCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/ContentCluster.java @@ -131,7 +131,7 @@ public class ContentCluster extends AbstractConfigProducer<AbstractConfigProduce resourceLimits.getClusterControllerLimits()).build(deployState, c, contentElement.getXml()); c.search = new ContentSearchCluster.Builder(documentDefinitions, globallyDistributedDocuments, - isCombined(getClusterId(contentElement), containers), + fractionOfMemoryReserved(getClusterId(contentElement), containers), resourceLimits.getContentNodeLimits()).build(deployState, c, contentElement.getXml()); c.persistenceFactory = new EngineFactoryBuilder().build(contentElement, c); c.storageNodes = new StorageCluster.Builder().build(deployState, c, w3cContentElement); @@ -240,12 +240,15 @@ public class ContentCluster extends AbstractConfigProducer<AbstractConfigProduce } } - /** Returns whether this hosts one of the given container clusters */ - private boolean isCombined(String clusterId, Collection<ContainerModel> containers) { - return containers.stream() - .map(model -> model.getCluster().getHostClusterId()) - .filter(Optional::isPresent) - .anyMatch(id -> id.get().equals(clusterId)); + /** Returns of memory reserved on a host. Memory is reserved for the jvm if th ecluster is combined */ + private double fractionOfMemoryReserved(String clusterId, Collection<ContainerModel> containers) { + for (ContainerModel containerModel : containers) { + Optional<String> hostClusterId = containerModel.getCluster().getHostClusterId(); + if (hostClusterId.isPresent() && hostClusterId.get().equals(clusterId) && containerModel.getCluster().getMemoryPercentage().isPresent()) { + return containerModel.getCluster().getMemoryPercentage().get() * 0.01; + } + } + return 0.0; } private void setupExperimental(ContentCluster cluster, ModelElement experimental) { diff --git a/config-model/src/main/java/com/yahoo/vespa/model/search/NodeResourcesTuning.java b/config-model/src/main/java/com/yahoo/vespa/model/search/NodeResourcesTuning.java index 65efcf85b1d..3af0b49de82 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/search/NodeResourcesTuning.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/search/NodeResourcesTuning.java @@ -3,9 +3,9 @@ package com.yahoo.vespa.model.search; import com.yahoo.config.provision.NodeResources; import com.yahoo.vespa.config.search.core.ProtonConfig; -import com.yahoo.vespa.model.container.ApplicationContainerCluster; import static java.lang.Long.min; +import static java.lang.Long.max; /** * Tuning of proton config for a search node based on the resources on the node. @@ -20,17 +20,20 @@ public class NodeResourcesTuning implements ProtonConfig.Producer { private final static long MEMORY_COST_PER_DOCUMENT_STORE_ONLY = 46L; private final NodeResources resources; private final int threadsPerSearch; - private final boolean combined; + private final double fractionOfMemoryReserved; + private final double tlsSizeFraction; // "Reserve" 0.5GB of memory for other processes running on the content node (config-proxy, metrics-proxy). public static final double reservedMemoryGb = 0.5; public NodeResourcesTuning(NodeResources resources, int threadsPerSearch, - boolean combined) { + double fractionOfMemoryReserved, + double tlsSizeFraction) { this.resources = resources; this.threadsPerSearch = threadsPerSearch; - this.combined = combined; + this.fractionOfMemoryReserved = fractionOfMemoryReserved; + this.tlsSizeFraction = tlsSizeFraction; } @Override @@ -95,8 +98,8 @@ public class NodeResourcesTuning implements ProtonConfig.Producer { } private void tuneFlushStrategyTlsSize(ProtonConfig.Flush.Memory.Builder builder) { - long tlsSizeBytes = (long) ((resources.diskGb() * 0.07) * GB); - tlsSizeBytes = min(tlsSizeBytes, 100 * GB); + long tlsSizeBytes = (long) ((resources.diskGb() * tlsSizeFraction) * GB); + tlsSizeBytes = max(2*GB, min(tlsSizeBytes, 100 * GB)); builder.maxtlssize(tlsSizeBytes); } @@ -122,12 +125,7 @@ public class NodeResourcesTuning implements ProtonConfig.Producer { /** Returns the memory we can expect will be available for the content node processes */ private double usableMemoryGb() { double usableMemoryGb = resources.memoryGb() - reservedMemoryGb; - if ( ! combined) { - return usableMemoryGb; - } - - double fractionTakenByContainer = ApplicationContainerCluster.heapSizePercentageOfTotalNodeMemoryWhenCombinedCluster * 1e-2; - return usableMemoryGb * (1 - fractionTakenByContainer); + return usableMemoryGb * (1 - fractionOfMemoryReserved); } } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/search/SearchNode.java b/config-model/src/main/java/com/yahoo/vespa/model/search/SearchNode.java index 4b3b464f9e5..31513a273b2 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/search/SearchNode.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/search/SearchNode.java @@ -67,9 +67,8 @@ public class SearchNode extends AbstractService implements private AbstractService serviceLayerService; private final Optional<Tuning> tuning; private final Optional<ResourceLimits> resourceLimits; - - /** Whether this search node is co-located with a container node on a hosted system */ - private final boolean combined; + private final double fractionOfMemoryReserved; + private final double tlsSizeFraction; public static class Builder extends VespaDomBuilder.DomConfigProducerBuilder<SearchNode> { @@ -80,11 +79,11 @@ public class SearchNode extends AbstractService implements private final boolean flushOnShutdown; private final Optional<Tuning> tuning; private final Optional<ResourceLimits> resourceLimits; - private boolean combined; + private final double fractionOfMemoryReserved; public Builder(String name, NodeSpec nodeSpec, String clusterName, ContentNode node, boolean flushOnShutdown, Optional<Tuning> tuning, Optional<ResourceLimits> resourceLimits, - boolean combined) { + double fractionOfMemoryReserved) { this.name = name; this.nodeSpec = nodeSpec; this.clusterName = clusterName; @@ -92,13 +91,14 @@ public class SearchNode extends AbstractService implements this.flushOnShutdown = flushOnShutdown; this.tuning = tuning; this.resourceLimits = resourceLimits; - this.combined = combined; + this.fractionOfMemoryReserved = fractionOfMemoryReserved; } @Override protected SearchNode doBuild(DeployState deployState, AbstractConfigProducer ancestor, Element producerSpec) { return new SearchNode(ancestor, name, contentNode.getDistributionKey(), nodeSpec, clusterName, contentNode, - flushOnShutdown, tuning, resourceLimits, deployState.isHosted(), combined); + flushOnShutdown, tuning, resourceLimits, deployState.isHosted(), fractionOfMemoryReserved, + deployState.featureFlags().tlsSizeFraction()); } } @@ -106,16 +106,16 @@ public class SearchNode extends AbstractService implements public static SearchNode create(AbstractConfigProducer parent, String name, int distributionKey, NodeSpec nodeSpec, String clusterName, AbstractService serviceLayerService, boolean flushOnShutdown, Optional<Tuning> tuning, Optional<ResourceLimits> resourceLimits, boolean isHostedVespa, - boolean combined) { - return new SearchNode(parent, name, distributionKey, nodeSpec, clusterName, serviceLayerService, - flushOnShutdown, tuning, resourceLimits, isHostedVespa, combined); + double fractionOfMemoryReserved, double tlsSizeFraction) { + return new SearchNode(parent, name, distributionKey, nodeSpec, clusterName, serviceLayerService, flushOnShutdown, + tuning, resourceLimits, isHostedVespa, fractionOfMemoryReserved, tlsSizeFraction); } private SearchNode(AbstractConfigProducer parent, String name, int distributionKey, NodeSpec nodeSpec, String clusterName, AbstractService serviceLayerService, boolean flushOnShutdown, Optional<Tuning> tuning, Optional<ResourceLimits> resourceLimits, boolean isHostedVespa, - boolean combined) { - this(parent, name, nodeSpec, clusterName, flushOnShutdown, tuning, resourceLimits, isHostedVespa, combined); + double fractionOfMemoryReserved, double tlsSizeFraction) { + this(parent, name, nodeSpec, clusterName, flushOnShutdown, tuning, resourceLimits, isHostedVespa, fractionOfMemoryReserved, tlsSizeFraction); this.distributionKey = distributionKey; this.serviceLayerService = serviceLayerService; setPropertiesElastic(clusterName, distributionKey); @@ -123,11 +123,12 @@ public class SearchNode extends AbstractService implements private SearchNode(AbstractConfigProducer parent, String name, NodeSpec nodeSpec, String clusterName, boolean flushOnShutdown, Optional<Tuning> tuning, Optional<ResourceLimits> resourceLimits, boolean isHostedVespa, - boolean combined) { + double fractionOfMemoryReserved, double tlsSizeFraction) { super(parent, name); setOmpNumThreads(1); this.isHostedVespa = isHostedVespa; - this.combined = combined; + this.fractionOfMemoryReserved = fractionOfMemoryReserved; + this.tlsSizeFraction = tlsSizeFraction; this.nodeSpec = nodeSpec; this.clusterName = clusterName; this.flushOnShutdown = flushOnShutdown; @@ -281,7 +282,7 @@ public class SearchNode extends AbstractService implements if (nodeResources.isPresent()) { var nodeResourcesTuning = new NodeResourcesTuning(nodeResources.get(), tuning.map(Tuning::threadsPerSearch).orElse(1), - combined); + fractionOfMemoryReserved, tlsSizeFraction); nodeResourcesTuning.getConfig(builder); tuning.ifPresent(t -> t.getConfig(builder)); diff --git a/config-model/src/test/java/com/yahoo/config/model/provision/ModelProvisioningTest.java b/config-model/src/test/java/com/yahoo/config/model/provision/ModelProvisioningTest.java index 0866e1174ee..912161019fe 100644 --- a/config-model/src/test/java/com/yahoo/config/model/provision/ModelProvisioningTest.java +++ b/config-model/src/test/java/com/yahoo/config/model/provision/ModelProvisioningTest.java @@ -278,6 +278,43 @@ public class ModelProvisioningTest { } } + @Test + public void testCombinedClusterWithJvmHeapSizeOverride() { + var containerElements = Set.of("jdisc", "container"); + for (var containerElement : containerElements) { + String xmlWithNodes = + "<?xml version='1.0' encoding='utf-8' ?>" + + "<services>" + + " <" + containerElement + " version='1.0' id='container1'>" + + " <search/>" + + " <nodes of='content1'>" + + " <jvm allocated-memory=\"30%\"/>" + + " </nodes>" + + " </" + containerElement + ">" + + " <content version='1.0' id='content1'>" + + " <redundancy>2</redundancy>" + + " <documents>" + + " <document type='type1' mode='index'/>" + + " </documents>" + + " <nodes count='2'>" + + " <resources vcpu='1' memory='3Gb' disk='9Gb'/>" + + " </nodes>" + + " </content>" + + "</services>"; + VespaModelTester tester = new VespaModelTester(); + tester.addHosts(5); + VespaModel model = tester.createModel(xmlWithNodes, true); + assertEquals("Nodes in content1", 2, model.getContentClusters().get("content1").getRootGroup().getNodes().size()); + assertEquals("Nodes in container1", 2, model.getContainerClusters().get("container1").getContainers().size()); + assertEquals("Heap size is lowered with combined clusters", + 30, physicalMemoryPercentage(model.getContainerClusters().get("container1"))); + assertEquals("Memory for proton is lowered to account for the jvm heap", + (long)((3 - reservedMemoryGb) * (Math.pow(1024, 3)) * (1 - 0.30)), protonMemorySize(model.getContentClusters().get("content1"))); + assertProvisioned(0, ClusterSpec.Id.from("container1"), ClusterSpec.Type.container, model); + assertProvisioned(2, ClusterSpec.Id.from("content1"), ClusterSpec.Id.from("container1"), ClusterSpec.Type.combined, model); + } + } + /** For comparison with the above */ @Test public void testNonCombinedCluster() { @@ -929,6 +966,10 @@ public class ModelProvisioningTest { assertNull(hostResource.getService("logserver")); assertNotNull(hostResource.getService("container")); assertNotNull(hostResource.getService("logforwarder")); + + var lfs = hostResource.getService("logforwarder"); + String shutdown = lfs.getPreShutdownCommand().orElse("<none>"); + assertThat(shutdown, is("/usr/bin/env SPLUNK_HOME=/opt/splunkforwarder /opt/splunkforwarder/bin/splunk stop")); } diff --git a/config-model/src/test/java/com/yahoo/vespa/model/HostPortsTest.java b/config-model/src/test/java/com/yahoo/vespa/model/HostPortsTest.java index f842ca72d92..31d089d64db 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/HostPortsTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/HostPortsTest.java @@ -12,7 +12,7 @@ import java.util.List; import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.is; -import static org.junit.Assert.assertThat; +import static org.hamcrest.MatcherAssert.assertThat; /** * @author arnej diff --git a/config-model/src/test/java/com/yahoo/vespa/model/HostResourceTest.java b/config-model/src/test/java/com/yahoo/vespa/model/HostResourceTest.java index a0f988f7f3b..80a388a822e 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/HostResourceTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/HostResourceTest.java @@ -9,13 +9,11 @@ import com.yahoo.config.provision.HostSpec; import com.yahoo.config.provision.NodeResources; import org.junit.Test; -import java.util.List; import java.util.Optional; import static com.yahoo.config.provision.ClusterSpec.Type.container; -import static org.hamcrest.Matchers.endsWith; import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertThat; +import static org.junit.Assert.assertTrue; /** * @author gjoranv @@ -30,7 +28,7 @@ public class HostResourceTest { try { service.initService(root.deployLogger()); } catch (RuntimeException e) { - assertThat(e.getMessage(), endsWith("No host found for service 'hostresourcetest$testservice0'. " + + assertTrue(e.getMessage().endsWith("No host found for service 'hostresourcetest$testservice0'. " + "The hostalias is probably missing from hosts.xml.")); } } diff --git a/config-model/src/test/java/com/yahoo/vespa/model/VespaModelFactoryTest.java b/config-model/src/test/java/com/yahoo/vespa/model/VespaModelFactoryTest.java index bf9ffb13a7a..bff90080115 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/VespaModelFactoryTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/VespaModelFactoryTest.java @@ -26,9 +26,8 @@ import java.util.ArrayList; import java.util.List; import java.util.Optional; -import static org.hamcrest.CoreMatchers.is; +import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNotNull; -import static org.junit.Assert.assertThat; import static org.junit.Assert.assertTrue; /** @@ -122,10 +121,10 @@ public class VespaModelFactoryTest { Model model = new VespaModelFactory(new NullConfigModelRegistry()).createModel(modelContext); List<HostInfo> allocatedHosts = new ArrayList<>(model.getHosts()); - assertThat(allocatedHosts.size(), is(1)); + assertEquals(1, allocatedHosts.size()); HostInfo hostInfo = allocatedHosts.get(0); - assertThat(hostInfo.getHostname(), is(hostName)); + assertEquals(hostName, hostInfo.getHostname()); assertTrue("Routing service should run on host " + hostName, hostInfo.getServices().stream() .map(ServiceInfo::getConfigId) diff --git a/config-model/src/test/java/com/yahoo/vespa/model/admin/AdminTestCase.java b/config-model/src/test/java/com/yahoo/vespa/model/admin/AdminTestCase.java index f296b0e83e2..7d12b9d6386 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/admin/AdminTestCase.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/admin/AdminTestCase.java @@ -30,12 +30,10 @@ import org.junit.Test; import java.util.Set; import static com.yahoo.config.model.api.container.ContainerServiceType.METRICS_PROXY_CONTAINER; -import static org.hamcrest.CoreMatchers.is; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertNotSame; -import static org.junit.Assert.assertThat; import static org.junit.Assert.assertTrue; @@ -55,7 +53,7 @@ public class AdminTestCase { VespaModel vespaModel = getVespaModel(TESTDIR + "adminconfig20"); // Verify that the admin plugin has been loaded (always loads routing). - assertThat(vespaModel.configModelRepo().asMap().size(), is(2)); + assertEquals(2, vespaModel.configModelRepo().asMap().size()); ApplicationConfigProducerRoot root = vespaModel.getVespa(); assertNotNull(root); @@ -102,12 +100,12 @@ public class AdminTestCase { SentinelConfig.Builder b = new SentinelConfig.Builder(); vespaModel.getConfig(b, localhostConfigId); SentinelConfig sentinelConfig = new SentinelConfig(b); - assertThat(sentinelConfig.service().size(), is(5)); - assertThat(sentinelConfig.service(0).name(), is("logserver")); - assertThat(sentinelConfig.service(1).name(), is("slobrok")); - assertThat(sentinelConfig.service(2).name(), is("slobrok2")); - assertThat(sentinelConfig.service(3).name(), is(METRICS_PROXY_CONTAINER.serviceName)); - assertThat(sentinelConfig.service(4).name(), is("logd")); + assertEquals(5, sentinelConfig.service().size()); + assertEquals("logserver", sentinelConfig.service(0).name()); + assertEquals("slobrok", sentinelConfig.service(1).name()); + assertEquals("slobrok2", sentinelConfig.service(2).name()); + assertEquals(METRICS_PROXY_CONTAINER.serviceName, sentinelConfig.service(3).name()); + assertEquals("logd", sentinelConfig.service(4).name()); } /** @@ -119,7 +117,7 @@ public class AdminTestCase { VespaModel vespaModel = getVespaModel(TESTDIR + "simpleadminconfig20"); // Verify that the admin plugin has been loaded (always loads routing). - assertThat(vespaModel.configModelRepo().asMap().size(), is(2)); + assertEquals(2, vespaModel.configModelRepo().asMap().size()); ApplicationConfigProducerRoot root = vespaModel.getVespa(); assertNotNull(root); @@ -138,12 +136,12 @@ public class AdminTestCase { SentinelConfig.Builder b = new SentinelConfig.Builder(); vespaModel.getConfig(b, localhostConfigId); SentinelConfig sentinelConfig = new SentinelConfig(b); - assertThat(sentinelConfig.service().size(), is(4)); - assertThat(sentinelConfig.service(0).name(), is("logserver")); - assertThat(sentinelConfig.service(1).name(), is("slobrok")); - assertThat(sentinelConfig.service(2).name(), is(METRICS_PROXY_CONTAINER.serviceName)); - assertThat(sentinelConfig.service(3).name(), is("logd")); - assertThat(sentinelConfig.service(0).affinity().cpuSocket(), is(-1)); + assertEquals(4, sentinelConfig.service().size()); + assertEquals("logserver", sentinelConfig.service(0).name()); + assertEquals("slobrok", sentinelConfig.service(1).name()); + assertEquals(METRICS_PROXY_CONTAINER.serviceName, sentinelConfig.service(2).name()); + assertEquals("logd", sentinelConfig.service(3).name()); + assertEquals(-1, sentinelConfig.service(0).affinity().cpuSocket()); assertTrue(sentinelConfig.service(0).preShutdownCommand().isEmpty()); // Confirm slobrok config @@ -168,11 +166,11 @@ public class AdminTestCase { TestRoot root = new TestDriver().buildModel(state); String localhost = HostName.getLocalhost(); SentinelConfig config = root.getConfig(SentinelConfig.class, "hosts/" + localhost); - assertThat(config.application().tenant(), is("quux")); - assertThat(config.application().name(), is("foo")); - assertThat(config.application().environment(), is("dev")); - assertThat(config.application().region(), is("baz")); - assertThat(config.application().instance(), is("bim")); + assertEquals("quux", config.application().tenant()); + assertEquals("foo", config.application().name()); + assertEquals("dev", config.application().environment()); + assertEquals("baz", config.application().region()); + assertEquals("bim", config.application().instance()); } @Test @@ -180,7 +178,7 @@ public class AdminTestCase { VespaModel vespaModel = getVespaModel(TESTDIR + "multipleconfigservers"); // Verify that the admin plugin has been loaded (always loads routing). - assertThat(vespaModel.configModelRepo().asMap().size(), is(2)); + assertEquals(2, vespaModel.configModelRepo().asMap().size()); ApplicationConfigProducerRoot root = vespaModel.getVespa(); assertNotNull(root); @@ -196,14 +194,14 @@ public class AdminTestCase { assertTrue(configIds.contains("admin/configservers/configserver.0")); assertTrue(configIds.contains("admin/configservers/configserver.1")); - assertThat(admin.getConfigservers().size(), is(2)); + assertEquals(2, admin.getConfigservers().size()); // Default configserver is the first one in the list and should have the default ports too Configserver server1 = admin.getConfigservers().get(0); assertEquals(admin.getConfigserver(), server1); - assertThat(server1.getPortCount(), is(2)); - assertThat(server1.getRelativePort(0), is(19070)); - assertThat(server1.getRelativePort(1), is(19071)); + assertEquals(2, server1.getPortCount()); + assertEquals(19070, server1.getRelativePort(0)); + assertEquals(19071, server1.getRelativePort(1)); // Second configserver should be on second host but have the same port number @@ -212,9 +210,9 @@ public class AdminTestCase { assertNotSame(server1, server2); assertNotSame(server1.getHostName(), server2.getHostName()); - assertThat(server2.getPortCount(), is(2)); - assertThat(server2.getRelativePort(0), is(19070)); - assertThat(server2.getRelativePort(1), is(19071)); + assertEquals(2, server2.getPortCount()); + assertEquals(19070, server2.getRelativePort(0)); + assertEquals(19071, server2.getRelativePort(1)); } @Test @@ -293,11 +291,11 @@ public class AdminTestCase { TestRoot root = new TestDriver().buildModel(state); String localhost = HostName.getLocalhost(); SentinelConfig sentinelConfig = root.getConfig(SentinelConfig.class, "hosts/" + localhost); - assertThat(sentinelConfig.service().size(), is(4)); - assertThat(sentinelConfig.service(0).name(), is("logserver")); - assertThat(sentinelConfig.service(1).name(), is("slobrok")); - assertThat(sentinelConfig.service(2).name(), is(METRICS_PROXY_CONTAINER.serviceName)); - assertThat(sentinelConfig.service(3).name(), is("logd")); + assertEquals(4, sentinelConfig.service().size()); + assertEquals("logserver", sentinelConfig.service(0).name()); + assertEquals("slobrok", sentinelConfig.service(1).name()); + assertEquals(METRICS_PROXY_CONTAINER.serviceName, sentinelConfig.service(2).name()); + assertEquals("logd", sentinelConfig.service(3).name()); } } diff --git a/config-model/src/test/java/com/yahoo/vespa/model/admin/ClusterControllerTestCase.java b/config-model/src/test/java/com/yahoo/vespa/model/admin/ClusterControllerTestCase.java index fcec74e0ec9..474013c17fc 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/admin/ClusterControllerTestCase.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/admin/ClusterControllerTestCase.java @@ -43,11 +43,9 @@ import java.util.Collection; import java.util.List; import java.util.Optional; -import static org.hamcrest.CoreMatchers.is; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertNull; -import static org.junit.Assert.assertThat; import static org.junit.Assert.assertTrue; /** @@ -106,11 +104,11 @@ public class ClusterControllerTestCase extends DomBuilderTest { model.getConfig(builder, "admin/cluster-controllers/" + i + "/components/clustercontroller-bar-configurer"); FleetcontrollerConfig cfg = new FleetcontrollerConfig(builder); - assertThat(cfg.index(), is(i)); - assertThat(cfg.fleet_controller_count(), is(3)); - assertThat(cfg.init_progress_time(), is(34567000)); - assertThat(cfg.storage_transition_time(), is(4000)); - assertThat(cfg.stable_state_time_period(), is(3600000)); + assertEquals(i, cfg.index()); + assertEquals(3, cfg.fleet_controller_count()); + assertEquals(34567000, cfg.init_progress_time()); + assertEquals(4000, cfg.storage_transition_time()); + assertEquals(3600000, cfg.stable_state_time_period()); } } @@ -222,15 +220,15 @@ public class ClusterControllerTestCase extends DomBuilderTest { ZookeepersConfig.Builder builder = new ZookeepersConfig.Builder(); root.getConfig(builder, "admin/standalone"); ZookeepersConfig config = new ZookeepersConfig(builder); - assertThat(config.zookeeperserverlist().split(",").length, is(3)); + assertEquals(3, config.zookeeperserverlist().split(",").length); } private void assertZookeeperServerConfig(TestRoot root, int id) { ZookeeperServerConfig.Builder builder = new ZookeeperServerConfig.Builder(); root.getConfig(builder, "admin/standalone/cluster-controllers/" + id); ZookeeperServerConfig config = new ZookeeperServerConfig(builder); - assertThat(config.server().size(), is(3)); - assertThat(config.myid(), is(id)); + assertEquals(3, config.server().size()); + assertEquals(id, config.myid()); Collection<Integer> serverIds = Collections2.transform(config.server(), ZookeeperServerConfig.Server::id); assertTrue(serverIds.contains(id)); } @@ -281,15 +279,15 @@ public class ClusterControllerTestCase extends DomBuilderTest { assertTrue(existsHostsWithClusterControllerConfigId(model)); assertGroupSize(model, "admin/cluster-controllers/0/components/clustercontroller-bar-configurer", 1); - assertThat(model.getAdmin().getClusterControllers().getContainers().size(), is(1)); + assertEquals(1, model.getAdmin().getClusterControllers().getContainers().size()); FleetcontrollerConfig.Builder builder = new FleetcontrollerConfig.Builder(); model.getConfig(builder, "admin/cluster-controllers/0/components/clustercontroller-bar-configurer"); FleetcontrollerConfig cfg = new FleetcontrollerConfig(builder); - assertThat(cfg.index(), is(0)); - assertThat(cfg.fleet_controller_count(), is(1)); - assertThat(cfg.init_progress_time(), is(34567000)); + assertEquals(0, cfg.index()); + assertEquals(1, cfg.fleet_controller_count()); + assertEquals(34567000, cfg.init_progress_time()); Service cc = model.getService("admin/cluster-controllers/0").get(); assertTrue(cc instanceof ClusterControllerContainer); @@ -344,7 +342,7 @@ public class ClusterControllerTestCase extends DomBuilderTest { VespaModel model = createVespaModel(xml); - assertThat(model.getAdmin().getClusterControllers().getContainers().size(), is(3)); + assertEquals(3, model.getAdmin().getClusterControllers().getContainers().size()); assertGroupSize(model, "admin/cluster-controllers/0/components/clustercontroller-bar-configurer", 1); assertGroupSize(model, "admin/cluster-controllers/1/components/clustercontroller-bar-configurer", 1); assertGroupSize(model, "admin/cluster-controllers/2/components/clustercontroller-bar-configurer", 1); @@ -475,7 +473,7 @@ public class ClusterControllerTestCase extends DomBuilderTest { StorDistributionConfig.Builder builder = new StorDistributionConfig.Builder(); model.getConfig(builder, configId); StorDistributionConfig cfg = new StorDistributionConfig(builder); - assertThat(cfg.group().size(), is(size)); + assertEquals(size, cfg.group().size()); } private VespaModel createVespaModel(String servicesXml) throws IOException, SAXException { @@ -505,6 +503,7 @@ public class ClusterControllerTestCase extends DomBuilderTest { assertEquals(1, reindexingConfig.clusters().size()); String contentClusterId = "bar"; assertEquals(Instant.EPOCH.toEpochMilli(), reindexingConfig.clusters(contentClusterId).documentTypes("type1").readyAtMillis()); + assertEquals(1.0, reindexingConfig.clusters(contentClusterId).documentTypes("type1").speed(), 0); } private static void assertReindexingConfiguredOnAdminCluster(VespaModel model) { diff --git a/config-model/src/test/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsConsumersTest.java b/config-model/src/test/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsConsumersTest.java index 38e4afaf74b..b5dd4552023 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsConsumersTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsConsumersTest.java @@ -6,9 +6,7 @@ import com.yahoo.vespa.model.VespaModel; import com.yahoo.vespa.model.admin.monitoring.Metric; import com.yahoo.vespa.model.admin.monitoring.MetricSet; import com.yahoo.vespa.model.admin.monitoring.MetricsConsumer; -import org.junit.Rule; import org.junit.Test; -import org.junit.rules.ExpectedException; import static com.yahoo.vespa.model.admin.metricsproxy.MetricsProxyModelTester.TestMode.hosted; import static com.yahoo.vespa.model.admin.metricsproxy.MetricsProxyModelTester.TestMode.self_hosted; @@ -26,6 +24,7 @@ import static com.yahoo.vespa.model.admin.monitoring.VespaMetricSet.vespaMetricS import static java.util.Collections.singleton; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; /** * Tests for {@link MetricsProxyContainerCluster} related to metrics consumers. @@ -34,15 +33,12 @@ import static org.junit.Assert.assertTrue; */ public class MetricsConsumersTest { - private static int numPublicDefaultMetrics = defaultMetricSet.getMetrics().size(); - private static int numDefaultVespaMetrics = defaultVespaMetricSet.getMetrics().size(); - private static int numVespaMetrics = vespaMetricSet.getMetrics().size(); - private static int numSystemMetrics = systemMetricSet.getMetrics().size(); - private static int numNetworkMetrics = networkMetricSet.getMetrics().size(); - private static int numMetricsForVespaConsumer = numVespaMetrics + numSystemMetrics + numNetworkMetrics; - - @Rule - public ExpectedException thrown = ExpectedException.none(); + private static final int numPublicDefaultMetrics = defaultMetricSet.getMetrics().size(); + private static final int numDefaultVespaMetrics = defaultVespaMetricSet.getMetrics().size(); + private static final int numVespaMetrics = vespaMetricSet.getMetrics().size(); + private static final int numSystemMetrics = systemMetricSet.getMetrics().size(); + private static final int numNetworkMetrics = networkMetricSet.getMetrics().size(); + private static final int numMetricsForVespaConsumer = numVespaMetrics + numSystemMetrics + numNetworkMetrics; @Test public void default_public_consumer_is_set_up_for_self_hosted() { @@ -103,9 +99,12 @@ public class MetricsConsumersTest { " </admin>", "</services>" ); - thrown.expect(IllegalArgumentException.class); - thrown.expectMessage("'" + consumerId + "' is not allowed as metrics consumer id"); - consumersConfigFromXml(services, self_hosted); + try { + consumersConfigFromXml(services, self_hosted); + fail(); + } catch (IllegalArgumentException e) { + assertEquals("'" + consumerId + "' is not allowed as metrics consumer id (case is ignored.)", e.getMessage()); + } } @Test @@ -147,9 +146,13 @@ public class MetricsConsumersTest { " </admin>", "</services>" ); - thrown.expect(IllegalArgumentException.class); - thrown.expectMessage("'a' is used as id for two metrics consumers"); - consumersConfigFromXml(services, self_hosted); + + try { + consumersConfigFromXml(services, self_hosted); + fail(); + } catch (IllegalArgumentException e) { + assertEquals("'a' is used as id for two metrics consumers (case is ignored.)", e.getMessage()); + } } @Test @@ -166,9 +169,12 @@ public class MetricsConsumersTest { " </admin>", "</services>" ); - thrown.expect(IllegalArgumentException.class); - thrown.expectMessage("No such metric-set: non-existent"); - consumersConfigFromXml(services, self_hosted); + try { + consumersConfigFromXml(services, self_hosted); + fail(); + } catch (IllegalArgumentException e) { + assertEquals("No such metric-set: non-existent", e.getMessage()); + } } @Test diff --git a/config-model/src/test/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainerClusterTest.java b/config-model/src/test/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainerClusterTest.java index 6a7008f4ba2..c00a5f65d08 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainerClusterTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainerClusterTest.java @@ -20,6 +20,7 @@ import com.yahoo.vespa.model.container.component.Handler; import org.junit.Test; import java.util.Collection; +import java.util.stream.Collectors; import static com.yahoo.vespa.model.admin.metricsproxy.MetricsProxyContainerCluster.METRICS_PROXY_BUNDLE_FILE; import static com.yahoo.vespa.model.admin.metricsproxy.MetricsProxyContainerCluster.zoneString; @@ -34,10 +35,8 @@ import static com.yahoo.vespa.model.admin.metricsproxy.MetricsProxyModelTester.g import static com.yahoo.vespa.model.admin.metricsproxy.MetricsProxyModelTester.getModel; import static com.yahoo.vespa.model.admin.metricsproxy.MetricsProxyModelTester.servicesWithAdminOnly; import static java.util.stream.Collectors.toList; -import static org.hamcrest.CoreMatchers.endsWith; -import static org.hamcrest.CoreMatchers.hasItem; import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertThat; +import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; /** @@ -51,7 +50,10 @@ public class MetricsProxyContainerClusterTest { var builder = new PlatformBundlesConfig.Builder(); model.getConfig(builder, CLUSTER_CONFIG_ID); PlatformBundlesConfig config = builder.build(); - assertThat(config.bundlePaths(), hasItem(endsWith(METRICS_PROXY_BUNDLE_FILE.toString()))); + assertFalse(config.bundlePaths().stream() + .filter(p -> p.endsWith(METRICS_PROXY_BUNDLE_FILE.toString())) + .collect(Collectors.toList()) + .isEmpty()); } @Test @@ -71,10 +73,10 @@ public class MetricsProxyContainerClusterTest { Collection<Handler<?>> handlers = model.getAdmin().getMetricsProxyCluster().getHandlers(); Collection<ComponentSpecification> handlerClasses = handlers.stream().map(Component::getClassId).collect(toList()); - assertThat(handlerClasses, hasItem(ComponentSpecification.fromString(MetricsV1Handler.class.getName()))); - assertThat(handlerClasses, hasItem(ComponentSpecification.fromString(PrometheusHandler.class.getName()))); - assertThat(handlerClasses, hasItem(ComponentSpecification.fromString(YamasHandler.class.getName()))); - assertThat(handlerClasses, hasItem(ComponentSpecification.fromString(ApplicationMetricsHandler.class.getName()))); + assertTrue(handlerClasses.contains(ComponentSpecification.fromString(MetricsV1Handler.class.getName()))); + assertTrue(handlerClasses.contains(ComponentSpecification.fromString(PrometheusHandler.class.getName()))); + assertTrue(handlerClasses.contains(ComponentSpecification.fromString(YamasHandler.class.getName()))); + assertTrue(handlerClasses.contains(ComponentSpecification.fromString(ApplicationMetricsHandler.class.getName()))); } @Test diff --git a/config-model/src/test/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainerTest.java b/config-model/src/test/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainerTest.java index 6a5a1aad7a1..ad3a163f052 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainerTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainerTest.java @@ -20,10 +20,8 @@ import static com.yahoo.vespa.model.admin.metricsproxy.MetricsProxyModelTester.g import static com.yahoo.vespa.model.admin.metricsproxy.MetricsProxyModelTester.getNodeDimensionsConfig; import static com.yahoo.vespa.model.admin.metricsproxy.MetricsProxyModelTester.getRpcConnectorConfig; import static com.yahoo.vespa.model.admin.metricsproxy.MetricsProxyModelTester.getVespaServicesConfig; -import static org.hamcrest.CoreMatchers.is; -import static org.hamcrest.CoreMatchers.notNullValue; import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertThat; +import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertTrue; /** @@ -38,15 +36,15 @@ public class MetricsProxyContainerTest { tester.addHosts(numberOfHosts); VespaModel model = tester.createModel(hostedServicesWithManyNodes(), true); - assertThat(model.getRoot().hostSystem().getHosts().size(), is(numberOfHosts)); + assertEquals(numberOfHosts, model.getRoot().hostSystem().getHosts().size()); for (var host : model.hostSystem().getHosts()) { - assertThat(host.getService(METRICS_PROXY_CONTAINER.serviceName), notNullValue()); + assertNotNull(host.getService(METRICS_PROXY_CONTAINER.serviceName)); long metricsProxies = host.getServices().stream() .filter(s -> s.getClass().equals(MetricsProxyContainer.class)) .count(); - assertThat(metricsProxies, is(1L)); + assertEquals(1, metricsProxies); } } @@ -57,15 +55,15 @@ public class MetricsProxyContainerTest { tester.addHosts(numberOfHosts); VespaModel model = tester.createModel(hostedServicesWithManyNodes(), true); - assertThat(model.getRoot().hostSystem().getHosts().size(), is(numberOfHosts)); + assertEquals(numberOfHosts, model.getRoot().hostSystem().getHosts().size()); for (var host : model.hostSystem().getHosts()) { - assertThat(host.getService(METRICS_PROXY_CONTAINER.serviceName), notNullValue()); + assertNotNull(host.getService(METRICS_PROXY_CONTAINER.serviceName)); long metricsProxies = host.getServices().stream() .filter(s -> s.getClass().equals(MetricsProxyContainer.class)) .count(); - assertThat(metricsProxies, is(1L)); + assertEquals(1, metricsProxies); } } diff --git a/config-model/src/test/java/com/yahoo/vespa/model/admin/metricsproxy/MonitoringElementTest.java b/config-model/src/test/java/com/yahoo/vespa/model/admin/metricsproxy/MonitoringElementTest.java index ee3496aa7bd..734844c9c7c 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/admin/metricsproxy/MonitoringElementTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/admin/metricsproxy/MonitoringElementTest.java @@ -3,30 +3,29 @@ package com.yahoo.vespa.model.admin.metricsproxy; import ai.vespa.metricsproxy.core.MonitoringConfig; import com.yahoo.vespa.model.VespaModel; -import org.junit.Rule; import org.junit.Test; -import org.junit.rules.ExpectedException; import static com.yahoo.vespa.model.admin.metricsproxy.MetricsProxyModelTester.CLUSTER_CONFIG_ID; import static com.yahoo.vespa.model.admin.metricsproxy.MetricsProxyModelTester.TestMode.hosted; import static com.yahoo.vespa.model.admin.metricsproxy.MetricsProxyModelTester.TestMode.self_hosted; import static com.yahoo.vespa.model.admin.metricsproxy.MetricsProxyModelTester.getModel; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.fail; /** * @author gjoranv */ public class MonitoringElementTest { - @Rule - public ExpectedException thrown = ExpectedException.none(); - @Test public void monitoring_element_is_disallowed_for_hosted_vespa() { String services = servicesWithMonitoringElement(); - thrown.expect(IllegalArgumentException.class); - thrown.expectMessage("The 'monitoring' element cannot be used"); - getModel(services, hosted); + try { + getModel(services, hosted); + fail(); + } catch (IllegalArgumentException e) { + assertEquals("The 'monitoring' element cannot be used on hosted Vespa.", e.getMessage()); + } } @Test diff --git a/config-model/src/test/java/com/yahoo/vespa/model/admin/metricsproxy/TelegrafTest.java b/config-model/src/test/java/com/yahoo/vespa/model/admin/metricsproxy/TelegrafTest.java index 4db7bf302b5..5bd364f4e21 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/admin/metricsproxy/TelegrafTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/admin/metricsproxy/TelegrafTest.java @@ -12,10 +12,8 @@ import static com.yahoo.vespa.model.admin.metricsproxy.MetricsProxyModelTester.C import static com.yahoo.vespa.model.admin.metricsproxy.MetricsProxyModelTester.TestMode.hosted; import static com.yahoo.vespa.model.admin.metricsproxy.MetricsProxyModelTester.TestMode.self_hosted; import static com.yahoo.vespa.model.admin.metricsproxy.MetricsProxyModelTester.getModel; -import static org.hamcrest.CoreMatchers.hasItem; -import static org.hamcrest.CoreMatchers.not; import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertThat; +import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; /** @@ -29,8 +27,8 @@ public class TelegrafTest { VespaModel hostedModel = getModel(services, hosted); var clusterComponents = hostedModel.getAdmin().getMetricsProxyCluster().getComponentsMap(); - assertThat(clusterComponents.keySet(), hasItem(ComponentId.fromString(Telegraf.class.getName()))); - assertThat(clusterComponents.keySet(), hasItem(ComponentId.fromString(TelegrafRegistry.class.getName()))); + assertTrue(clusterComponents.containsKey(ComponentId.fromString(Telegraf.class.getName()))); + assertTrue(clusterComponents.containsKey(ComponentId.fromString(TelegrafRegistry.class.getName()))); } @Test @@ -47,8 +45,8 @@ public class TelegrafTest { VespaModel hostedModel = getModel(services, hosted); var clusterComponents = hostedModel.getAdmin().getMetricsProxyCluster().getComponentsMap(); - assertThat(clusterComponents.keySet(), not(hasItem(ComponentId.fromString(Telegraf.class.getName())))); - assertThat(clusterComponents.keySet(), not(hasItem(ComponentId.fromString(TelegrafRegistry.class.getName())))); + assertFalse(clusterComponents.containsKey(ComponentId.fromString(Telegraf.class.getName()))); + assertFalse(clusterComponents.containsKey(ComponentId.fromString(TelegrafRegistry.class.getName()))); } @Test diff --git a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/ComponentValidatorTest.java b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/ComponentValidatorTest.java index dec223562a4..a375621d391 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/ComponentValidatorTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/ComponentValidatorTest.java @@ -11,9 +11,8 @@ import java.io.IOException; import java.util.jar.JarFile; import java.util.logging.Level; -import static org.hamcrest.CoreMatchers.containsString; -import static org.hamcrest.CoreMatchers.is; -import static org.junit.Assert.assertThat; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; public class ComponentValidatorTest { private static final String JARS_DIR = "src/test/cfg/application/validation/testjars/"; @@ -36,7 +35,7 @@ public class ComponentValidatorTest { componentValidator.validateAll(new BaseDeployLogger()); assert (false); } catch (IllegalArgumentException e) { - assertThat(e.getMessage(), is(exceptionMessage)); + assertEquals(e.getMessage(), exceptionMessage); } } @@ -52,6 +51,6 @@ public class ComponentValidatorTest { }; new ComponentValidator(new JarFile(JARS_DIR + "snapshot_bundle.jar")).validateAll(logger); - assertThat(buffer.toString(), containsString("Deploying snapshot bundle")); + assertTrue(buffer.toString().contains("Deploying snapshot bundle")); } } diff --git a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/ConfigChangeTestUtils.java b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/ConfigChangeTestUtils.java index 7d84a4a844e..00a1ec7e6d8 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/ConfigChangeTestUtils.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/ConfigChangeTestUtils.java @@ -10,8 +10,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; -import static org.hamcrest.CoreMatchers.equalTo; -import static org.junit.Assert.assertThat; +import static org.junit.Assert.assertEquals; public class ConfigChangeTestUtils { @@ -63,7 +62,7 @@ public class ConfigChangeTestUtils { var mutableAct = new ArrayList<>(act); mutableExp.sort((lhs, rhs) -> lhs.getMessage().compareTo(rhs.getMessage())); mutableAct.sort((lhs, rhs) -> lhs.getMessage().compareTo(rhs.getMessage())); - assertThat(mutableAct, equalTo(mutableExp)); + assertEquals(mutableExp, mutableAct); } } diff --git a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/StreamingSchemaClusterChangeValidatorTest.java b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/StreamingSchemaClusterChangeValidatorTest.java index dddb6a38db1..2e5b27bee6f 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/StreamingSchemaClusterChangeValidatorTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/StreamingSchemaClusterChangeValidatorTest.java @@ -19,8 +19,7 @@ import java.util.List; import static com.yahoo.vespa.model.application.validation.change.ConfigChangeTestUtils.assertEqualActions; import static com.yahoo.vespa.model.application.validation.change.ConfigChangeTestUtils.normalizeServicesInActions; -import static org.hamcrest.CoreMatchers.is; -import static org.junit.Assert.assertThat; +import static org.junit.Assert.assertTrue; public class StreamingSchemaClusterChangeValidatorTest { @@ -77,7 +76,7 @@ public class StreamingSchemaClusterChangeValidatorTest { } public void assertValidation() { - assertThat(validate().size(), is(0)); + assertTrue(validate().isEmpty()); } public void assertValidation(ConfigChangeAction exp) { diff --git a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/search/ContentClusterFixture.java b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/search/ContentClusterFixture.java index 6a9c3025e79..030f1d7d85d 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/search/ContentClusterFixture.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/search/ContentClusterFixture.java @@ -12,9 +12,8 @@ import com.yahoo.vespa.model.search.DocumentDatabase; import java.util.Arrays; import java.util.List; -import static org.hamcrest.CoreMatchers.equalTo; -import static org.hamcrest.CoreMatchers.is; -import static org.junit.Assert.assertThat; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; /** * Test fixture to setup current and next content clusters used for change validation. @@ -62,7 +61,7 @@ public abstract class ContentClusterFixture { public void assertValidation() { List<VespaConfigChangeAction> act = validate(); - assertThat(act.size(), is(0)); + assertTrue(act.isEmpty()); } public void assertValidation(VespaConfigChangeAction exp) { @@ -71,7 +70,7 @@ public abstract class ContentClusterFixture { public void assertValidation(List<VespaConfigChangeAction> exp) { List<VespaConfigChangeAction> act = validate(); - assertThat(act, equalTo(exp)); + assertEquals(exp, act); } public abstract List<VespaConfigChangeAction> validate(); diff --git a/config-model/src/test/java/com/yahoo/vespa/model/builder/xml/dom/DomAdminV2BuilderTest.java b/config-model/src/test/java/com/yahoo/vespa/model/builder/xml/dom/DomAdminV2BuilderTest.java index cfcf06702c7..1a41d2689a2 100755 --- a/config-model/src/test/java/com/yahoo/vespa/model/builder/xml/dom/DomAdminV2BuilderTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/builder/xml/dom/DomAdminV2BuilderTest.java @@ -20,10 +20,8 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.List; -import static org.hamcrest.CoreMatchers.is; -import static org.hamcrest.core.IsNot.not; +import static org.junit.Assert.assertNotEquals; import static org.junit.Assert.assertNotNull; -import static org.junit.Assert.assertThat; import static org.junit.Assert.assertEquals; /** @@ -122,12 +120,12 @@ public class DomAdminV2BuilderTest extends DomBuilderTest { new TestProperties.Spec("test2", 19070, 2181), new TestProperties.Spec("test3", 19070, 2181)); Admin admin = buildAdmin(servicesMultitenantAdminOnly(), true, configServerSpecs); - assertThat(admin.getConfigservers().size(), is(3)); - assertThat(admin.getSlobroks().size(), is(1)); + assertEquals(3, admin.getConfigservers().size()); + assertEquals(1, admin.getSlobroks().size()); assertNotNull(admin.hostSystem().getHostByHostname("test1")); for (Configserver configserver : admin.getConfigservers()) { for (Slobrok slobrok : admin.getSlobroks()) { - assertThat(slobrok.getHostName(), is(not(configserver.getHostName()))); + assertNotEquals(configserver.getHostName(), slobrok.getHostName()); } } } @@ -138,7 +136,7 @@ public class DomAdminV2BuilderTest extends DomBuilderTest { @Test public void adminWithConfigserverElement() { Admin admin = buildAdmin(servicesConfigserver()); - assertThat(admin.getConfigservers().size(), is(1)); + assertEquals(1, admin.getConfigservers().size()); } /** @@ -147,15 +145,15 @@ public class DomAdminV2BuilderTest extends DomBuilderTest { @Test public void adminWithConfigserversElement() { Admin admin = buildAdmin(servicesConfigservers()); - assertThat(admin.getConfigservers().size(), is(1)); + assertEquals(1, admin.getConfigservers().size()); } @Test public void basicYamasNoXml() { Admin admin = buildAdmin(servicesNoYamas()); Monitoring y = admin.getMonitoring(); - assertThat(y.getClustername(), is("vespa")); - assertThat(y.getInterval(), is(1)); + assertEquals("vespa", y.getClustername()); + assertEquals(1, y.getInterval().intValue()); } @Test @@ -168,16 +166,16 @@ public class DomAdminV2BuilderTest extends DomBuilderTest { public void basicYamasXml() { Admin admin = buildAdmin(servicesYamas()); Monitoring y = admin.getMonitoring(); - assertThat(y.getClustername(), is("foo")); - assertThat(y.getInterval(), is(1)); + assertEquals("foo", y.getClustername()); + assertEquals(1, y.getInterval().intValue()); } @Test public void yamasWithIntervalOverride() { Admin admin = buildAdmin(servicesYamasIntervalOverride()); Monitoring y = admin.getMonitoring(); - assertThat(y.getClustername(), is("foo")); - assertThat(y.getInterval(), is(5)); + assertEquals("foo", y.getClustername()); + assertEquals(5, y.getInterval().intValue()); } /** @@ -196,11 +194,11 @@ public class DomAdminV2BuilderTest extends DomBuilderTest { @Test public void configOverridesCanBeUsedInAdmin() { Admin admin = buildAdmin(servicesOverride()); - assertThat(admin.getUserConfigs().size(), is(1)); + assertEquals(1, admin.getUserConfigs().size()); LogdConfig.Builder logdBuilder = new LogdConfig.Builder(); admin.addUserConfig(logdBuilder); LogdConfig config = new LogdConfig(logdBuilder); - assertThat(config.logserver().host(), is("foobar")); + assertEquals("foobar", config.logserver().host()); } private Admin buildAdmin(Element xml) { diff --git a/config-model/src/test/java/com/yahoo/vespa/model/builder/xml/dom/DomComponentBuilderTest.java b/config-model/src/test/java/com/yahoo/vespa/model/builder/xml/dom/DomComponentBuilderTest.java index ed342d1ca87..307cbf292db 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/builder/xml/dom/DomComponentBuilderTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/builder/xml/dom/DomComponentBuilderTest.java @@ -8,9 +8,8 @@ import com.yahoo.vespa.model.container.component.Component; import org.junit.Test; import static com.yahoo.collections.CollectionUtil.first; -import static org.hamcrest.CoreMatchers.is; +import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNotNull; -import static org.junit.Assert.assertThat; /** * @author gjoranv @@ -23,9 +22,9 @@ public class DomComponentBuilderTest extends DomBuilderTest { "<handler id='theId' class='theClass' bundle='theBundle' />")); BundleInstantiationSpecification instantiationSpecification = handler.model.bundleInstantiationSpec; - assertThat(instantiationSpecification.id.stringValue(), is("theId")); - assertThat(instantiationSpecification.classId.stringValue(), is("theClass")); - assertThat(instantiationSpecification.bundle.stringValue(), is("theBundle")); + assertEquals("theId", instantiationSpecification.id.stringValue()); + assertEquals("theClass", instantiationSpecification.classId.stringValue()); + assertEquals("theBundle", instantiationSpecification.bundle.stringValue()); } @Test @@ -36,10 +35,10 @@ public class DomComponentBuilderTest extends DomBuilderTest { " <component id='child' />", "</component>")); - assertThat(parent.getGlobalComponentId(), is(ComponentId.fromString("parent"))); + assertEquals(ComponentId.fromString("parent"), parent.getGlobalComponentId()); Component<?, ?> child = first(parent.getChildren().values()); assertNotNull(child); - assertThat(child.getGlobalComponentId(), is(ComponentId.fromString("child@parent"))); + assertEquals(ComponentId.fromString("child@parent"), child.getGlobalComponentId()); } } diff --git a/config-model/src/test/java/com/yahoo/vespa/model/builder/xml/dom/DomConfigPayloadBuilderTest.java b/config-model/src/test/java/com/yahoo/vespa/model/builder/xml/dom/DomConfigPayloadBuilderTest.java index 40513b019b8..91482919e11 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/builder/xml/dom/DomConfigPayloadBuilderTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/builder/xml/dom/DomConfigPayloadBuilderTest.java @@ -23,8 +23,7 @@ import java.io.FileReader; import java.io.Reader; import java.io.StringReader; -import static org.hamcrest.CoreMatchers.is; -import static org.junit.Assert.assertThat; +import static org.junit.Assert.assertEquals; import static org.junit.Assert.fail; /** @@ -37,7 +36,7 @@ public class DomConfigPayloadBuilderTest { @Test public void testFunctionTest_DefaultValues() throws FileNotFoundException { - Element configRoot = getDocument(new FileReader(new File("src/test/cfg/admin/userconfigs/functiontest-defaultvalues.xml"))); + Element configRoot = getDocument(new FileReader("src/test/cfg/admin/userconfigs/functiontest-defaultvalues.xml")); ConfigPayload config = ConfigPayload.fromBuilder(new DomConfigPayloadBuilder(null).build(configRoot)); String expected = "" + "{" @@ -65,7 +64,7 @@ public class DomConfigPayloadBuilderTest { try { ByteArrayOutputStream a = new ByteArrayOutputStream(); new JsonFormat(true).encode(a, payload.getSlime()); - assertThat(a.toString(), is(expected)); + assertEquals(expected, a.toString()); } catch (Exception e) { fail("Exception thrown when encoding slime: " + e.getMessage()); } @@ -74,7 +73,7 @@ public class DomConfigPayloadBuilderTest { // Multi line strings are not tested in 'DefaultValues', so here it is. @Test public void verifyThatWhitespaceIsPreservedForStrings() throws Exception { - Element configRoot = getDocument(new FileReader(new File("src/test/cfg/admin/userconfigs/whitespace-test.xml"))); + Element configRoot = getDocument(new FileReader("src/test/cfg/admin/userconfigs/whitespace-test.xml")); ConfigPayload config = ConfigPayload.fromBuilder(new DomConfigPayloadBuilder(null).build(configRoot)); assertPayload("{\"stringVal\":\" This is a string\\n that contains different kinds of whitespace \"}", config); } @@ -164,8 +163,7 @@ public class DomConfigPayloadBuilderTest { new DomConfigPayloadBuilder(null).build(configRoot); fail("Expected exception for wrong tag name."); } catch (ConfigurationRuntimeException e) { - assertThat(e.getMessage(), - is("The root element must be 'config', but was 'configs'.")); + assertEquals("The root element must be 'config', but was 'configs'.", e.getMessage()); } } @@ -177,8 +175,7 @@ public class DomConfigPayloadBuilderTest { new DomConfigPayloadBuilder(null).build(configRoot); fail("Expected exception for mismatch between def-name and xml name attribute."); } catch (ConfigurationRuntimeException e) { - assertThat(e.getMessage(), - is("The 'config' element must have a 'name' attribute that matches the name of the config definition.")); + assertEquals("The 'config' element must have a 'name' attribute that matches the name of the config definition.", e.getMessage()); } } @@ -188,8 +185,8 @@ public class DomConfigPayloadBuilderTest { "<int_val>1</int_val> +" + "</config>")); ConfigDefinitionKey key = DomConfigPayloadBuilder.parseConfigName(configRoot); - assertThat(key.getName(), is("function-test")); - assertThat(key.getNamespace(), is("test")); + assertEquals("function-test", key.getName()); + assertEquals("test", key.getNamespace()); } @Test(expected = ConfigurationRuntimeException.class) @@ -273,7 +270,7 @@ public class DomConfigPayloadBuilderTest { new FileReader(new File("src/test/resources/configdefinitions/test.simpletypes.def"))); ConfigDefinition def = ConfigDefinitionBuilder.createConfigDefinition(defParser.getTree()); ConfigPayloadBuilder builder = new DomConfigPayloadBuilder(def).build(configRoot); - //assertThat(builder.warnings().size(), is(1)); + //assertEquals(1, builder.warnings().size()); } private Element getDocument(Reader xmlReader) { diff --git a/config-model/src/test/java/com/yahoo/vespa/model/builder/xml/dom/LegacyConfigModelBuilderTest.java b/config-model/src/test/java/com/yahoo/vespa/model/builder/xml/dom/LegacyConfigModelBuilderTest.java index 8236528f3d0..2bcb1a7459b 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/builder/xml/dom/LegacyConfigModelBuilderTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/builder/xml/dom/LegacyConfigModelBuilderTest.java @@ -14,6 +14,7 @@ import org.w3c.dom.Element; import java.util.List; import static org.hamcrest.CoreMatchers.is; +import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertThat; /** @@ -27,7 +28,7 @@ public class LegacyConfigModelBuilderTest { ModelBuilder builder = new ModelBuilder(); Model model = builder.build(DeployState.createTestState(new MockApplicationPackage.Builder().withServices(services).build()), null, null, new MockRoot(), XML.getDocument(services).getDocumentElement()); - assertThat(model.getContext().getParentProducer().getUserConfigs().size(), is(1)); + assertEquals(1, model.getContext().getParentProducer().getUserConfigs().size()); } public static class Model extends ConfigModel { diff --git a/config-model/src/test/java/com/yahoo/vespa/model/builder/xml/dom/chains/search/DomFederationSearcherBuilderTest.java b/config-model/src/test/java/com/yahoo/vespa/model/builder/xml/dom/chains/search/DomFederationSearcherBuilderTest.java index ae087f71855..d68f306c043 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/builder/xml/dom/chains/search/DomFederationSearcherBuilderTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/builder/xml/dom/chains/search/DomFederationSearcherBuilderTest.java @@ -11,12 +11,9 @@ import org.junit.Test; import java.util.ArrayList; import java.util.List; -import static org.hamcrest.CoreMatchers.hasItems; -import static org.hamcrest.CoreMatchers.is; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertTrue; -import static org.junit.Assert.assertThat; /** * Test of DomFederationSearcherBuilder. @@ -46,7 +43,7 @@ public class DomFederationSearcherBuilderTest extends DomBuilderTest { assertEquals(2, model.targets.size()); assertTrue("source-set option was ignored", model.inheritDefaultSources); - assertThat(targetNames(model.targets), hasItems("source1", "source2")); + assertTrue(targetNames(model.targets).containsAll(List.of("source1", "source2"))); } private List<String> targetNames(List<FederationSearcherModel.TargetSpec> targets) { @@ -71,7 +68,7 @@ public class DomFederationSearcherBuilderTest extends DomBuilderTest { FederationConfig.Builder builder = new FederationConfig.Builder(); searcher.getConfig(builder); - assertThat(new FederationConfig(builder).targetSelector(), is(targetSelectorId)); + assertEquals(targetSelectorId, new FederationConfig(builder).targetSelector()); } } diff --git a/config-model/src/test/java/com/yahoo/vespa/model/builder/xml/dom/chains/search/DomSchemaChainsBuilderTest.java b/config-model/src/test/java/com/yahoo/vespa/model/builder/xml/dom/chains/search/DomSchemaChainsBuilderTest.java index 86c448c403c..d052d4dd1f2 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/builder/xml/dom/chains/search/DomSchemaChainsBuilderTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/builder/xml/dom/chains/search/DomSchemaChainsBuilderTest.java @@ -8,7 +8,11 @@ import com.yahoo.config.model.test.MockRoot; import com.yahoo.container.core.ChainsConfig; import com.yahoo.search.searchchain.model.federation.FederationOptions; import com.yahoo.vespa.model.container.component.chain.ChainedComponent; -import com.yahoo.vespa.model.container.search.searchchain.*; +import com.yahoo.vespa.model.container.search.searchchain.GenericTarget; +import com.yahoo.vespa.model.container.search.searchchain.Provider; +import com.yahoo.vespa.model.container.search.searchchain.SearchChain; +import com.yahoo.vespa.model.container.search.searchchain.SearchChains; +import com.yahoo.vespa.model.container.search.searchchain.Source; import org.hamcrest.BaseMatcher; import org.hamcrest.Description; import org.hamcrest.Matcher; @@ -23,11 +27,10 @@ import java.util.List; import static com.yahoo.container.core.ChainsConfig.Chains; import static com.yahoo.container.core.ChainsConfig.Components; import static org.hamcrest.CoreMatchers.hasItem; -import static org.hamcrest.core.Is.is; -import static org.hamcrest.Matchers.containsString; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertThat; +import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; /** @@ -92,7 +95,7 @@ public class DomSchemaChainsBuilderTest extends DomBuilderTest { new DomSearchChainsBuilder().build(root.getDeployState(), root, element); fail("Expected exception when referring to an outer 'federation' as a 'searcher'."); } catch (RuntimeException e) { - assertThat(e.getMessage(), containsString("Two different types declared for the component with name 'federationSearcher'")); + assertTrue(e.getMessage().contains("Two different types declared for the component with name 'federationSearcher'")); } } @@ -163,10 +166,10 @@ public class DomSchemaChainsBuilderTest extends DomBuilderTest { Components searcher = getSearcherConfig(config.components(), partOfSearcherName); ComponentId searcherId = ComponentId.fromString(searcher.id()); - assertThat(searcherId.getNamespace(), is(getSearchChain(searchChainName).getComponentId())); + assertEquals(getSearchChain(searchChainName).getComponentId(), searcherId.getNamespace()); Chains searchChain = getSearchChainConfig(config.chains(), searchChainName); - assertThat(ComponentId.fromString(searchChain.components(0)), is(searcherId)); + assertEquals(searcherId, ComponentId.fromString(searchChain.components(0))); } private Chains getSearchChainConfig(List<Chains> searchChains, diff --git a/config-model/src/test/java/com/yahoo/vespa/model/container/ContainerClusterTest.java b/config-model/src/test/java/com/yahoo/vespa/model/container/ContainerClusterTest.java index 560ac28b6f7..eb1cf668cc9 100755 --- a/config-model/src/test/java/com/yahoo/vespa/model/container/ContainerClusterTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/container/ContainerClusterTest.java @@ -31,7 +31,6 @@ import com.yahoo.vespa.model.container.component.Component; import com.yahoo.vespa.model.container.docproc.ContainerDocproc; import com.yahoo.vespa.model.container.search.ContainerSearch; import com.yahoo.vespa.model.container.search.searchchain.SearchChains; -import org.hamcrest.CoreMatchers; import org.junit.Test; import java.util.ArrayList; @@ -40,7 +39,6 @@ import java.util.Iterator; import java.util.List; import java.util.Objects; import java.util.OptionalInt; -import java.util.OptionalLong; import java.util.Set; import java.util.stream.Collectors; @@ -51,12 +49,8 @@ import static com.yahoo.config.model.api.ApplicationClusterEndpoint.Scope.applic import static com.yahoo.config.model.api.ApplicationClusterEndpoint.Scope.global; import static com.yahoo.config.provision.SystemName.cd; import static com.yahoo.config.provision.SystemName.main; -import static org.hamcrest.Matchers.containsInAnyOrder; -import static org.hamcrest.Matchers.hasItem; -import static org.hamcrest.Matchers.hasKey; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertThat; import static org.junit.Assert.assertTrue; /** @@ -100,8 +94,8 @@ public class ContainerClusterTest { assertEquals("cd", config.system()); } - private ApplicationContainerCluster createContainerCluster(MockRoot root, boolean isCombinedCluster) { - return createContainerCluster(root, isCombinedCluster, null); + private ApplicationContainerCluster createContainerCluster(MockRoot root) { + return createContainerCluster(root, false, null); } private ApplicationContainerCluster createContainerCluster(MockRoot root, boolean isCombinedCluster, Integer memoryPercentage) { ApplicationContainerCluster cluster = new ApplicationContainerCluster(root, "container0", "container1", root.getDeployState()); @@ -162,7 +156,7 @@ public class ContainerClusterTest { private void verifyJvmArgs(boolean isHosted, boolean hasDocProc) { MockRoot root = createRoot(isHosted); - ApplicationContainerCluster cluster = createContainerCluster(root, false); + ApplicationContainerCluster cluster = createContainerCluster(root); if (hasDocProc) { cluster.setDocproc(new ContainerDocproc(cluster, null)); } @@ -227,7 +221,7 @@ public class ContainerClusterTest { public void requireThatJvmOmitStackTraceInFastThrowOptionWorks() { // Empty option if option not set in property MockRoot root = createRoot(new DeployState.Builder().build()); - ApplicationContainerCluster cluster = createContainerCluster(root, false); + ApplicationContainerCluster cluster = createContainerCluster(root); addContainer(root, cluster, "c1", "host-c1"); ApplicationContainer container = cluster.getContainers().get(0); assertEquals("", container.getJvmOptions()); @@ -235,7 +229,7 @@ public class ContainerClusterTest { String jvmOption = "-XX:-foo"; DeployState deployState = new DeployState.Builder().properties(new TestProperties().setJvmOmitStackTraceInFastThrowOption(jvmOption)).build(); root = createRoot(deployState); - cluster = createContainerCluster(root, false); + cluster = createContainerCluster(root); addContainer(root, cluster, "c1", "host-c1"); container = cluster.getContainers().get(0); assertEquals(jvmOption, container.getJvmOptions()); @@ -244,7 +238,7 @@ public class ContainerClusterTest { @Test public void requireThatWeCanHandleNull() { MockRoot root = createRoot(false); - ApplicationContainerCluster cluster = createContainerCluster(root, false); + ApplicationContainerCluster cluster = createContainerCluster(root); addContainer(root, cluster, "c1", "host-c1"); Container container = cluster.getContainers().get(0); container.setJvmOptions(""); @@ -256,7 +250,7 @@ public class ContainerClusterTest { @Test public void requireThatNonHostedUsesExpectedDefaultThreadpoolConfiguration() { MockRoot root = new MockRoot("foo"); - ApplicationContainerCluster cluster = createContainerCluster(root, false); + ApplicationContainerCluster cluster = createContainerCluster(root); addContainer(root, cluster, "c1", "host-c1"); root.freezeModelTopology(); @@ -268,13 +262,13 @@ public class ContainerClusterTest { @Test public void container_cluster_has_default_threadpool_provider() { MockRoot root = new MockRoot("foo"); - ApplicationContainerCluster cluster = createContainerCluster(root, false); + ApplicationContainerCluster cluster = createContainerCluster(root); addContainer(root, cluster, "c1", "host-c1"); root.freezeModelTopology(); ComponentId expectedComponentId = new ComponentId("default-threadpool"); var components = cluster.getComponentsMap(); - assertThat(components, hasKey(expectedComponentId)); + assertTrue(components.containsKey(expectedComponentId)); Component<?, ?> component = components.get(expectedComponentId); assertEquals(ThreadPoolProvider.class.getName(), component.getClassId().getName()); } @@ -287,7 +281,7 @@ public class ContainerClusterTest { .properties(new TestProperties().setHostedVespa(true)) .applicationPackage(new MockApplicationPackage.Builder().build()) .build()); - ApplicationContainerCluster cluster = createContainerCluster(root, false); + ApplicationContainerCluster cluster = createContainerCluster(root); addContainer(root, cluster, "c1", "host-c1"); root.freezeModelTopology(); @@ -304,7 +298,7 @@ public class ContainerClusterTest { .properties(new TestProperties().setHostedVespa(true)) .applicationPackage(new MockApplicationPackage.Builder().build()) .build()); - ApplicationContainerCluster cluster = createContainerCluster(root, false); + ApplicationContainerCluster cluster = createContainerCluster(root); addContainer(root, cluster, "c1", "host-c1"); root.freezeModelTopology(); @@ -485,7 +479,7 @@ public class ContainerClusterTest { cluster.getConfig(bundleBuilder); List<String> installedBundles = bundleBuilder.build().bundlePaths(); - expectedBundleNames.forEach(b -> assertThat(installedBundles, hasItem(CoreMatchers.endsWith(b)))); + expectedBundleNames.forEach(b -> assertTrue(installedBundles.stream().filter(p -> p.endsWith(b)).count() > 0)); } diff --git a/config-model/src/test/java/com/yahoo/vespa/model/container/ContainerIncludeTest.java b/config-model/src/test/java/com/yahoo/vespa/model/container/ContainerIncludeTest.java index 0e35b72a75b..cb1b84d272a 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/container/ContainerIncludeTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/container/ContainerIncludeTest.java @@ -11,10 +11,9 @@ import org.junit.Test; import java.util.HashMap; import java.util.Map; -import static org.hamcrest.CoreMatchers.containsString; -import static org.hamcrest.CoreMatchers.is; -import static org.hamcrest.CoreMatchers.notNullValue; -import static org.junit.Assert.assertThat; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; /** @@ -28,62 +27,62 @@ public class ContainerIncludeTest { VespaModelCreatorWithFilePkg creator = new VespaModelCreatorWithFilePkg("src/test/cfg/container/data/containerinclude/"); VespaModel model = creator.create(); - assertThat(model.getContainerClusters().size(), is(1)); + assertEquals(1, model.getContainerClusters().size()); ContainerCluster cluster = model.getContainerClusters().values().iterator().next(); - assertThat(cluster.getSearchChains(), notNullValue()); + assertNotNull(cluster.getSearchChains()); Map<String, SearchChain> searchChainMap = new HashMap<>(); for (SearchChain searchChain : cluster.getSearchChains().allChains().allComponents()) { searchChainMap.put(searchChain.getId().stringValue(), searchChain); } - assertThat(searchChainMap.get("searchchain1"), notNullValue()); - assertThat(searchChainMap.get("searchchain1").getInnerComponents().size(), is(1)); - assertThat(searchChainMap.get("searchchain1").getInnerComponents().iterator().next().getComponentId().stringValue(), is("com.yahoo.Searcher1")); + assertNotNull(searchChainMap.get("searchchain1")); + assertEquals(1, searchChainMap.get("searchchain1").getInnerComponents().size()); + assertEquals("com.yahoo.Searcher1", searchChainMap.get("searchchain1").getInnerComponents().iterator().next().getComponentId().stringValue()); - assertThat(searchChainMap.get("searchchain2"), notNullValue()); - assertThat(searchChainMap.get("searchchain2").getInnerComponents().size(), is(1)); - assertThat(searchChainMap.get("searchchain2").getInnerComponents().iterator().next().getComponentId().stringValue(), is("com.yahoo.Searcher2")); + assertNotNull(searchChainMap.get("searchchain2")); + assertEquals(1, searchChainMap.get("searchchain2").getInnerComponents().size()); + assertEquals("com.yahoo.Searcher2", searchChainMap.get("searchchain2").getInnerComponents().iterator().next().getComponentId().stringValue()); - assertThat(searchChainMap.get("searchchain3"), notNullValue()); - assertThat(searchChainMap.get("searchchain3").getInnerComponents().size(), is(1)); - assertThat(searchChainMap.get("searchchain3").getInnerComponents().iterator().next().getComponentId().stringValue(), is("com.yahoo.Searcher3")); + assertNotNull(searchChainMap.get("searchchain3")); + assertEquals(1, searchChainMap.get("searchchain3").getInnerComponents().size()); + assertEquals("com.yahoo.Searcher3", searchChainMap.get("searchchain3").getInnerComponents().iterator().next().getComponentId().stringValue()); - assertThat(searchChainMap.get("searchchain4"), notNullValue()); - assertThat(searchChainMap.get("searchchain4").getInnerComponents().size(), is(1)); - assertThat(searchChainMap.get("searchchain4").getInnerComponents().iterator().next().getComponentId().stringValue(), is("com.yahoo.Searcher4")); + assertNotNull(searchChainMap.get("searchchain4")); + assertEquals(1, searchChainMap.get("searchchain4").getInnerComponents().size()); + assertEquals("com.yahoo.Searcher4", searchChainMap.get("searchchain4").getInnerComponents().iterator().next().getComponentId().stringValue()); - assertThat(cluster.getDocprocChains(), notNullValue()); + assertNotNull(cluster.getDocprocChains()); Map<String, DocprocChain> docprocChainMap = new HashMap<>(); for (DocprocChain docprocChain : cluster.getDocprocChains().allChains().allComponents()) { docprocChainMap.put(docprocChain.getId().stringValue(), docprocChain); } - assertThat(docprocChainMap.get("docprocchain1"), notNullValue()); - assertThat(docprocChainMap.get("docprocchain1").getInnerComponents().size(), is(1)); - assertThat(docprocChainMap.get("docprocchain1").getInnerComponents().iterator().next().getComponentId().stringValue(), is("com.yahoo.DocumentProcessor1")); + assertNotNull(docprocChainMap.get("docprocchain1")); + assertEquals(1, docprocChainMap.get("docprocchain1").getInnerComponents().size()); + assertEquals("com.yahoo.DocumentProcessor1", docprocChainMap.get("docprocchain1").getInnerComponents().iterator().next().getComponentId().stringValue()); - assertThat(docprocChainMap.get("docprocchain2"), notNullValue()); - assertThat(docprocChainMap.get("docprocchain2").getInnerComponents().size(), is(1)); - assertThat(docprocChainMap.get("docprocchain2").getInnerComponents().iterator().next().getComponentId().stringValue(), is("com.yahoo.DocumentProcessor2")); + assertNotNull(docprocChainMap.get("docprocchain2")); + assertEquals(1, docprocChainMap.get("docprocchain2").getInnerComponents().size()); + assertEquals("com.yahoo.DocumentProcessor2", docprocChainMap.get("docprocchain2").getInnerComponents().iterator().next().getComponentId().stringValue()); - assertThat(cluster.getProcessingChains(), notNullValue()); + assertNotNull(cluster.getProcessingChains()); Map<String, ProcessingChain> processingChainMap = new HashMap<>(); for (ProcessingChain processingChain : cluster.getProcessingChains().allChains().allComponents()) { processingChainMap.put(processingChain.getId().stringValue(), processingChain); } - assertThat(processingChainMap.get("processingchain1"), notNullValue()); - assertThat(processingChainMap.get("processingchain1").getInnerComponents().size(), is(1)); - assertThat(processingChainMap.get("processingchain1").getInnerComponents().iterator().next().getComponentId().stringValue(), is("com.yahoo.Processor1")); + assertNotNull(processingChainMap.get("processingchain1")); + assertEquals(1, processingChainMap.get("processingchain1").getInnerComponents().size()); + assertEquals("com.yahoo.Processor1", processingChainMap.get("processingchain1").getInnerComponents().iterator().next().getComponentId().stringValue()); - assertThat(processingChainMap.get("processingchain2"), notNullValue()); - assertThat(processingChainMap.get("processingchain2").getInnerComponents().size(), is(1)); - assertThat(processingChainMap.get("processingchain2").getInnerComponents().iterator().next().getComponentId().stringValue(), is("com.yahoo.Processor2")); + assertNotNull(processingChainMap.get("processingchain2")); + assertEquals(1, processingChainMap.get("processingchain2").getInnerComponents().size()); + assertEquals("com.yahoo.Processor2", processingChainMap.get("processingchain2").getInnerComponents().iterator().next().getComponentId().stringValue()); } @Test(expected = IllegalArgumentException.class) @@ -123,8 +122,8 @@ public class ContainerIncludeTest { creator.create(true); fail("Expected exception due to xml schema violation ('zearcer')"); } catch (IllegalArgumentException e) { - assertThat(e.getMessage(), containsString("Invalid XML according to XML schema")); - assertThat(e.getMessage(), containsString("zearcer")); + assertTrue(e.getMessage().contains("Invalid XML according to XML schema")); + assertTrue(e.getMessage().contains("zearcer")); } } diff --git a/config-model/src/test/java/com/yahoo/vespa/model/container/configserver/ConfigserverClusterTest.java b/config-model/src/test/java/com/yahoo/vespa/model/container/configserver/ConfigserverClusterTest.java index 42d9ccf7089..7f49efa8770 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/container/configserver/ConfigserverClusterTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/container/configserver/ConfigserverClusterTest.java @@ -29,12 +29,8 @@ import java.util.Optional; import java.util.function.Function; import java.util.stream.Collectors; -import static org.hamcrest.CoreMatchers.not; -import static org.hamcrest.Matchers.hasItem; -import static org.hamcrest.Matchers.is; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertThat; import static org.junit.Assert.assertTrue; /** @@ -93,30 +89,30 @@ public class ConfigserverClusterTest { @Test public void testStatisticsConfig() { StatisticsConfig config = getConfig(StatisticsConfig.class); - assertThat((int) config.collectionintervalsec(), is(60)); - assertThat((int) config.loggingintervalsec(), is(60)); + assertEquals(60, (int) config.collectionintervalsec()); + assertEquals(60, (int) config.loggingintervalsec()); } @Test public void testHealthMonitorConfig() { HealthMonitorConfig config = getConfig(HealthMonitorConfig.class); - assertThat(((int) config.snapshot_interval()), is(60)); + assertEquals(60, (int) config.snapshot_interval()); } @Test public void testConfigserverConfig() { ConfigserverConfig config = getConfig(ConfigserverConfig.class); - assertThat(config.configModelPluginDir().size(), is(1)); - assertThat(config.configModelPluginDir().get(0), is(Defaults.getDefaults().underVespaHome("lib/jars/config-models"))); - assertThat(config.rpcport(), is(12345)); - assertThat(config.httpport(), is(1337)); - assertThat(config.serverId(), is(HostName.getLocalhost())); + assertEquals(1, config.configModelPluginDir().size()); + assertEquals(Defaults.getDefaults().underVespaHome("lib/jars/config-models"), config.configModelPluginDir().get(0)); + assertEquals(12345, config.rpcport()); + assertEquals(1337, config.httpport()); + assertEquals(HostName.getLocalhost(), config.serverId()); assertTrue(config.useVespaVersionInRequest()); - assertThat(config.numParallelTenantLoaders(), is(4)); + assertEquals(4, config.numParallelTenantLoaders()); assertFalse(config.multitenant()); assertTrue(config.hostedVespa()); - assertThat(config.environment(), is("test")); - assertThat(config.region(), is("bar")); + assertEquals("test", config.environment()); + assertEquals("bar", config.region()); } @Test @@ -132,8 +128,8 @@ public class ConfigserverClusterTest { public void model_evaluation_bundles_are_not_installed_via_config() { // These bundles must be pre-installed because they are used by config-model. PlatformBundlesConfig config = getConfig(PlatformBundlesConfig.class); - assertThat(config.bundlePaths(), not(hasItem(ContainerModelEvaluation.MODEL_INTEGRATION_BUNDLE_FILE.toString()))); - assertThat(config.bundlePaths(), not(hasItem(ContainerModelEvaluation.MODEL_EVALUATION_BUNDLE_FILE.toString()))); + assertFalse(config.bundlePaths().contains(ContainerModelEvaluation.MODEL_INTEGRATION_BUNDLE_FILE.toString())); + assertFalse(config.bundlePaths().contains(ContainerModelEvaluation.MODEL_EVALUATION_BUNDLE_FILE.toString())); } @SuppressWarnings("varargs") diff --git a/config-model/src/test/java/com/yahoo/vespa/model/container/docproc/StandaloneDocprocContainerTest.java b/config-model/src/test/java/com/yahoo/vespa/model/container/docproc/StandaloneDocprocContainerTest.java index de37240c269..5bb93255a8f 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/container/docproc/StandaloneDocprocContainerTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/container/docproc/StandaloneDocprocContainerTest.java @@ -14,8 +14,8 @@ import org.w3c.dom.Element; import java.util.Map; -import static org.hamcrest.CoreMatchers.is; -import static org.junit.Assert.assertThat; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; /** * @author Einar M R Rosenvinge @@ -59,8 +59,8 @@ public class StandaloneDocprocContainerTest extends DomBuilderTest { if (componentId.stringValue().contains("MbusClient")) foundAtLeastOneClient = true; if (componentId.stringValue().contains("MbusServer")) foundAtLeastOneServer = true; } - assertThat(foundAtLeastOneClient, is(true)); - assertThat(foundAtLeastOneServer, is(true)); + assertTrue(foundAtLeastOneClient); + assertTrue(foundAtLeastOneServer); } @@ -76,7 +76,7 @@ public class StandaloneDocprocContainerTest extends DomBuilderTest { if (componentId.stringValue().contains("MbusClient")) foundAtLeastOneClient = true; if (componentId.stringValue().contains("MbusServer")) foundAtLeastOneServer = true; } - assertThat(foundAtLeastOneClient, is(false)); - assertThat(foundAtLeastOneServer, is(false)); + assertFalse(foundAtLeastOneClient); + assertFalse(foundAtLeastOneServer); } } diff --git a/config-model/src/test/java/com/yahoo/vespa/model/container/http/BlockFeedGlobalEndpointsFilterTest.java b/config-model/src/test/java/com/yahoo/vespa/model/container/http/BlockFeedGlobalEndpointsFilterTest.java index 1d774526a9b..1691868ee65 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/container/http/BlockFeedGlobalEndpointsFilterTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/container/http/BlockFeedGlobalEndpointsFilterTest.java @@ -5,8 +5,6 @@ package com.yahoo.vespa.model.container.http; import com.yahoo.config.model.api.ApplicationClusterEndpoint; import com.yahoo.config.model.api.ContainerEndpoint; import com.yahoo.jdisc.http.filter.security.rule.RuleBasedFilterConfig; -import org.hamcrest.Matchers; -import org.junit.Assert; import org.junit.Test; import java.util.Collections; @@ -15,6 +13,7 @@ import java.util.Set; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertThat; +import static org.junit.Assert.assertTrue; public class BlockFeedGlobalEndpointsFilterTest { @@ -25,7 +24,7 @@ public class BlockFeedGlobalEndpointsFilterTest { var config = getConfig(filter); assertEquals(1, config.rule().size()); var rule = config.rule().get(0); - assertThat(rule.hostNames(), Matchers.containsInAnyOrder("foo", "bar")); + assertTrue(rule.hostNames().containsAll(List.of("foo", "bar"))); assertEquals(rule.action(), RuleBasedFilterConfig.Rule.Action.Enum.BLOCK); } diff --git a/config-model/src/test/java/com/yahoo/vespa/model/container/http/FilterChainsTest.java b/config-model/src/test/java/com/yahoo/vespa/model/container/http/FilterChainsTest.java index 97ce8030fbd..6f223afd536 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/container/http/FilterChainsTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/container/http/FilterChainsTest.java @@ -10,9 +10,8 @@ import org.junit.Test; import org.w3c.dom.Element; import static com.yahoo.collections.CollectionUtil.first; -import static org.hamcrest.CoreMatchers.is; +import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNotNull; -import static org.junit.Assert.assertThat; /** * @author gjoranv @@ -55,7 +54,7 @@ public class FilterChainsTest extends DomBuilderTest { public void filters_in_chains_are_built() { Filter filter = first(getChain("myChain").getInnerComponents()); assertNotNull(filter); - assertThat(filter.getComponentId().getName(), is("inner")); + assertEquals("inner", filter.getComponentId().getName()); } private Chain<Filter> getChain(String chainName) { diff --git a/config-model/src/test/java/com/yahoo/vespa/model/container/http/FilterConfigTest.java b/config-model/src/test/java/com/yahoo/vespa/model/container/http/FilterConfigTest.java index e30628eb94f..e17c5eed242 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/container/http/FilterConfigTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/container/http/FilterConfigTest.java @@ -10,11 +10,9 @@ import org.w3c.dom.Element; import static com.yahoo.collections.CollectionUtil.first; import static com.yahoo.vespa.model.container.http.FilterConfigProvider.configProviderId; -import static org.hamcrest.Matchers.empty; -import static org.hamcrest.Matchers.hasSize; -import static org.hamcrest.Matchers.is; -import static org.hamcrest.Matchers.nullValue; -import static org.junit.Assert.assertThat; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertTrue; /** * @author gjoranv @@ -59,7 +57,7 @@ public class FilterConfigTest extends DomBuilderTest { public void filter_without_config_does_not_have_FilterConfigProvider() { Filter noConfigFilter = getOuterFilter("no-config"); - assertThat(getProvider(noConfigFilter), nullValue()); + assertNull(getProvider(noConfigFilter)); } @Test @@ -67,7 +65,7 @@ public class FilterConfigTest extends DomBuilderTest { Filter emptyConfigFilter = getOuterFilter("empty-config"); HttpFilterConfig config = getHttpFilterConfig(emptyConfigFilter); - assertThat(config.filterName(), is("empty-config")); + assertEquals("empty-config", config.filterName()); } @Test @@ -75,7 +73,7 @@ public class FilterConfigTest extends DomBuilderTest { Filter emptyConfigFilter = getOuterFilter("empty-config"); HttpFilterConfig config = getHttpFilterConfig(emptyConfigFilter); - assertThat(config.filterClass(), is("EmptyConfigFilter")); + assertEquals("EmptyConfigFilter", config.filterClass()); } @Test @@ -83,7 +81,7 @@ public class FilterConfigTest extends DomBuilderTest { Filter emptyConfigFilter = getOuterFilter("empty-config"); HttpFilterConfig config = getHttpFilterConfig(emptyConfigFilter); - assertThat(config.param(), is(empty())); + assertTrue(config.param().isEmpty()); } @Test @@ -91,14 +89,14 @@ public class FilterConfigTest extends DomBuilderTest { Filter configWithParamsFilter = getOuterFilter("config-with-params"); HttpFilterConfig config = getHttpFilterConfig(configWithParamsFilter); - assertThat(config.param(), hasSize(1)); - assertThat(config.param(0).name(), is("key1")); - assertThat(config.param(0).value(), is("value1")); + assertEquals(1, config.param().size()); + assertEquals("key1", config.param(0).name()); + assertEquals("value1", config.param(0).value()); } @Test public void inner_filter_can_have_filter_config() { - Filter innerFilter = (Filter) + Filter innerFilter = first(http.getFilterChains().allChains().getComponent("myChain").getInnerComponents()); getHttpFilterConfig(innerFilter); diff --git a/config-model/src/test/java/com/yahoo/vespa/model/container/search/searchchain/FederationSearcherTest.java b/config-model/src/test/java/com/yahoo/vespa/model/container/search/searchchain/FederationSearcherTest.java index 2482fc131f0..3f95fb99087 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/container/search/searchchain/FederationSearcherTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/container/search/searchchain/FederationSearcherTest.java @@ -25,10 +25,8 @@ import static java.util.Collections.emptyList; import static java.util.Collections.emptySet; import static java.util.Collections.singletonList; import static java.util.stream.Collectors.toList; -import static org.hamcrest.Matchers.contains; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertSame; -import static org.junit.Assert.assertThat; import static org.junit.Assert.assertTrue; /** @@ -99,8 +97,9 @@ public class FederationSearcherTest { assertEquals(target.id(), "source"); assertTrue("Not used by default", target.useByDefault()); assertEquals(2, target.searchChain().size()); - assertThat(target.searchChain().stream().map(FederationConfig.Target.SearchChain::providerId).collect(toList()), - contains("provider1", "provider2")); + assertTrue(target.searchChain().stream() + .map(FederationConfig.Target.SearchChain::providerId) + .collect(toList()).containsAll(List.of("provider1", "provider2"))); } @Test diff --git a/config-model/src/test/java/com/yahoo/vespa/model/container/search/searchchain/FederationTest.java b/config-model/src/test/java/com/yahoo/vespa/model/container/search/searchchain/FederationTest.java index 39fcb9d5af2..3945a74cf15 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/container/search/searchchain/FederationTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/container/search/searchchain/FederationTest.java @@ -7,8 +7,10 @@ import org.w3c.dom.Element; import java.util.List; -import static org.hamcrest.core.Is.is; -import static org.junit.Assert.*; +import static org.assertj.core.api.Fail.fail; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; /** * Test generated config for federation. @@ -65,7 +67,7 @@ public class FederationTest extends SchemaChainsTestBase { } } - assertThat(config.target().size(), is(5)); + assertEquals(5, config.target().size()); assertUseByDefault(config, "source1", false); assertUseByDefault(config, "source2", false); @@ -82,7 +84,7 @@ public class FederationTest extends SchemaChainsTestBase { FederationConfig.Target target = getTarget(config.target(), sourceName); FederationConfig.Target.SearchChain searchChain = getProvider(target, providerName); - assertThat(searchChain.useByDefault(), is(expectedValue)); + assertEquals(expectedValue, searchChain.useByDefault()); } private FederationConfig.Target.SearchChain getProvider(FederationConfig.Target target, String providerName) { @@ -96,8 +98,8 @@ public class FederationTest extends SchemaChainsTestBase { private void assertUseByDefault(FederationConfig config, String chainName, boolean expectedValue) { FederationConfig.Target target = getTarget(config.target(), chainName); - assertThat(target.searchChain().size(), is(1)); - assertThat(target.searchChain().get(0).useByDefault(), is(expectedValue)); + assertEquals(1, target.searchChain().size()); + assertEquals(expectedValue, target.searchChain().get(0).useByDefault()); } private FederationConfig.Target getTarget(List<FederationConfig.Target> targets, String chainId) { diff --git a/config-model/src/test/java/com/yahoo/vespa/model/container/search/searchchain/SchemaChainsTest.java b/config-model/src/test/java/com/yahoo/vespa/model/container/search/searchchain/SchemaChainsTest.java index 9ccfa768dc1..d74f271f6ef 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/container/search/searchchain/SchemaChainsTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/container/search/searchchain/SchemaChainsTest.java @@ -5,14 +5,12 @@ import com.yahoo.component.ComponentId; import com.yahoo.container.core.ChainsConfig; import com.yahoo.prelude.cluster.ClusterSearcher; import com.yahoo.search.config.ClusterConfig; -import com.yahoo.search.federation.ProviderConfig; import com.yahoo.vespa.defaults.Defaults; import org.junit.Before; import org.junit.Test; import org.w3c.dom.Element; -import static org.hamcrest.CoreMatchers.is; -import static org.junit.Assert.*; +import static org.junit.Assert.assertEquals; /** @@ -24,7 +22,6 @@ import static org.junit.Assert.*; public class SchemaChainsTest extends SchemaChainsTestBase { private ChainsConfig chainsConfig; - private ProviderConfig providerConfig; private ClusterConfig clusterConfig; @Before @@ -93,7 +90,7 @@ public class SchemaChainsTest extends SchemaChainsTestBase { @Test public void require_that_source_chain_spec_id_is_namespaced_in_provider_id() { Source source = (Source) getSearchChains().allChains().getComponent("source:1@provider:1"); - assertThat(source.getChainSpecification().componentId.getNamespace(), is(ComponentId.fromString("provider:1"))); + assertEquals(ComponentId.fromString("provider:1"), source.getChainSpecification().componentId.getNamespace()); } @Test diff --git a/config-model/src/test/java/com/yahoo/vespa/model/container/search/searchchain/SourceGroupTest.java b/config-model/src/test/java/com/yahoo/vespa/model/container/search/searchchain/SourceGroupTest.java index 3db112355a9..a9bab88a065 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/container/search/searchchain/SourceGroupTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/container/search/searchchain/SourceGroupTest.java @@ -2,19 +2,17 @@ package com.yahoo.vespa.model.container.search.searchchain; import com.yahoo.component.ComponentId; -import com.yahoo.component.ComponentSpecification; -import com.yahoo.component.chain.Phase; import com.yahoo.component.chain.model.ChainSpecification; import com.yahoo.config.model.test.MockRoot; import com.yahoo.search.searchchain.model.federation.FederationOptions; import org.junit.Before; import org.junit.Test; -import java.util.Collections; +import java.util.List; +import java.util.Set; -import static org.junit.Assert.assertThat; +import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; -import static org.hamcrest.Matchers.containsString; /** * @author Tony Vaagenes @@ -41,7 +39,7 @@ public class SourceGroupTest { searchChains.validate(); } catch (Exception e) { - assertThat(e.getMessage(), containsString("Missing leader for the source s1.")); + assertTrue(e.getMessage().contains("Missing leader for the source s1.")); return; } fail("Expected exception"); @@ -54,8 +52,8 @@ public class SourceGroupTest { private ChainSpecification createSearchChainSpecification(String id) { return new ChainSpecification(ComponentId.fromString(id), new ChainSpecification.Inheritance(null, null), - Collections.<Phase>emptyList(), - Collections.<ComponentSpecification>emptySet()); + List.of(), + Set.of()); } private Source createSource(String sourceId, Source.GroupOption groupOption) { @@ -82,8 +80,8 @@ public class SourceGroupTest { searchChains.validate(); fail("Expected exception"); } catch (Exception e) { - assertThat(e.getMessage(), containsString("Same id used for a source")); - assertThat(e.getMessage(), containsString("'sameId'")); + assertTrue(e.getMessage().contains("Same id used for a source")); + assertTrue(e.getMessage().contains("'sameId'")); } } } diff --git a/config-model/src/test/java/com/yahoo/vespa/model/container/xml/BundleInstantiationSpecificationBuilderTest.java b/config-model/src/test/java/com/yahoo/vespa/model/container/xml/BundleInstantiationSpecificationBuilderTest.java index 18520dd2df5..909ac9edef2 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/container/xml/BundleInstantiationSpecificationBuilderTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/container/xml/BundleInstantiationSpecificationBuilderTest.java @@ -15,8 +15,7 @@ import java.io.IOException; import java.io.InputStream; import java.nio.charset.StandardCharsets; -import static org.hamcrest.CoreMatchers.is; -import static org.junit.Assert.assertThat; +import static org.junit.Assert.assertEquals; /** * @author gjoranv @@ -56,6 +55,6 @@ public class BundleInstantiationSpecificationBuilderTest { Element component = XmlHelper.getDocumentBuilder().parse(xmlStream).getDocumentElement(); BundleInstantiationSpecification spec = BundleInstantiationSpecificationBuilder.build(component); - assertThat(spec.bundle, is(ComponentSpecification.fromString(expectedBundle))); + assertEquals(ComponentSpecification.fromString(expectedBundle), spec.bundle); } } diff --git a/config-model/src/test/java/com/yahoo/vespa/model/container/xml/ContainerDocumentApiBuilderTest.java b/config-model/src/test/java/com/yahoo/vespa/model/container/xml/ContainerDocumentApiBuilderTest.java index ed26dafe60c..b4242336c5c 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/container/xml/ContainerDocumentApiBuilderTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/container/xml/ContainerDocumentApiBuilderTest.java @@ -17,13 +17,10 @@ import java.util.HashMap; import java.util.Map; import java.util.Set; -import static org.hamcrest.CoreMatchers.equalTo; -import static org.hamcrest.CoreMatchers.is; -import static org.hamcrest.CoreMatchers.not; -import static org.hamcrest.CoreMatchers.nullValue; -import static org.hamcrest.Matchers.hasItem; import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertThat; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertTrue; /** * @author Einar M R Rosenvinge @@ -36,7 +33,7 @@ public class ContainerDocumentApiBuilderTest extends ContainerModelBuilderTestBa Map<String, Handler<?>> handlerMap = new HashMap<>(); Collection<Handler<?>> handlers = cluster.getHandlers(); for (Handler<?> handler : handlers) { - assertThat(handlerMap.containsKey(handler.getComponentId().toString()), is(false)); //die on overwrites + assertFalse(handlerMap.containsKey(handler.getComponentId().toString())); //die on overwrites handlerMap.put(handler.getComponentId().toString(), handler); } return handlerMap; @@ -59,10 +56,10 @@ public class ContainerDocumentApiBuilderTest extends ContainerModelBuilderTestBa private void verifyCustomBindings(String id) { Handler<?> handler = getHandlers("cluster1").get(id); - assertThat(handler.getServerBindings(), hasItem(UserBindingPattern.fromHttpPath("/document-api/reserved-for-internal-use/feedapi"))); - assertThat(handler.getServerBindings(), hasItem(UserBindingPattern.fromHttpPath("/document-api/reserved-for-internal-use/feedapi/"))); + assertTrue(handler.getServerBindings().contains(UserBindingPattern.fromHttpPath("/document-api/reserved-for-internal-use/feedapi"))); + assertTrue(handler.getServerBindings().contains(UserBindingPattern.fromHttpPath("/document-api/reserved-for-internal-use/feedapi/"))); - assertThat(handler.getServerBindings().size(), is(2)); + assertEquals(2, handler.getServerBindings().size()); } @Test @@ -76,18 +73,16 @@ public class ContainerDocumentApiBuilderTest extends ContainerModelBuilderTestBa Map<String, Handler<?>> handlerMap = getHandlers("cluster1"); - assertThat(handlerMap.get("com.yahoo.container.handler.VipStatusHandler"), not(nullValue())); - assertThat(handlerMap.get("com.yahoo.container.handler.observability.ApplicationStatusHandler"), not(nullValue())); - assertThat(handlerMap.get("com.yahoo.container.jdisc.state.StateHandler"), not(nullValue())); - - assertThat(handlerMap.get("com.yahoo.vespa.http.server.FeedHandler"), not(nullValue())); - assertThat(handlerMap.get("com.yahoo.vespa.http.server.FeedHandler").getServerBindings() - .contains(SystemBindingPattern.fromHttpPath("/reserved-for-internal-use/feedapi")), - is(true)); - assertThat(handlerMap.get("com.yahoo.vespa.http.server.FeedHandler").getServerBindings() - .contains(SystemBindingPattern.fromHttpPath("/reserved-for-internal-use/feedapi")), - is(true)); - assertThat(handlerMap.get("com.yahoo.vespa.http.server.FeedHandler").getServerBindings().size(), equalTo(2)); + assertNotNull(handlerMap.get("com.yahoo.container.handler.VipStatusHandler")); + assertNotNull(handlerMap.get("com.yahoo.container.handler.observability.ApplicationStatusHandler")); + assertNotNull(handlerMap.get("com.yahoo.container.jdisc.state.StateHandler")); + + assertNotNull(handlerMap.get("com.yahoo.vespa.http.server.FeedHandler")); + assertTrue(handlerMap.get("com.yahoo.vespa.http.server.FeedHandler").getServerBindings() + .contains(SystemBindingPattern.fromHttpPath("/reserved-for-internal-use/feedapi"))); + assertTrue(handlerMap.get("com.yahoo.vespa.http.server.FeedHandler").getServerBindings() + .contains(SystemBindingPattern.fromHttpPath("/reserved-for-internal-use/feedapi"))); + assertEquals(2, handlerMap.get("com.yahoo.vespa.http.server.FeedHandler").getServerBindings().size()); } @Test @@ -102,7 +97,7 @@ public class ContainerDocumentApiBuilderTest extends ContainerModelBuilderTestBa Map<String, Handler<?>> handlers = getHandlers("cluster1"); Handler<?> feedApiHandler = handlers.get("com.yahoo.vespa.http.server.FeedHandler"); Set<String> injectedComponentIds = feedApiHandler.getInjectedComponentIds(); - assertThat(injectedComponentIds, hasItem("threadpool@feedapi-handler")); + assertTrue(injectedComponentIds.contains("threadpool@feedapi-handler")); ContainerThreadpoolConfig config = root.getConfig( ContainerThreadpoolConfig.class, "cluster1/component/com.yahoo.vespa.http.server.FeedHandler/threadpool@feedapi-handler"); diff --git a/config-model/src/test/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilderTest.java b/config-model/src/test/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilderTest.java index 8ceb74c3d7e..f7dbd65ab7d 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilderTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilderTest.java @@ -80,19 +80,16 @@ import static com.yahoo.vespa.model.container.ContainerCluster.ROOT_HANDLER_BIND import static com.yahoo.vespa.model.container.ContainerCluster.STATE_HANDLER_BINDING_1; import static org.hamcrest.CoreMatchers.is; import static org.hamcrest.CoreMatchers.not; -import static org.hamcrest.CoreMatchers.notNullValue; -import static org.hamcrest.CoreMatchers.nullValue; import static org.hamcrest.Matchers.contains; import static org.hamcrest.Matchers.containsInAnyOrder; import static org.hamcrest.Matchers.containsString; -import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.hasItem; -import static org.hamcrest.Matchers.isEmptyString; +import static org.hamcrest.MatcherAssert.assertThat; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotEquals; import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertNull; -import static org.junit.Assert.assertThat; import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; @@ -112,8 +109,8 @@ public class ContainerModelBuilderTest extends ContainerModelBuilderTestBase { public void model_evaluation_bundles_are_deployed() { createBasicContainerModel(); PlatformBundlesConfig config = root.getConfig(PlatformBundlesConfig.class, "default"); - assertThat(config.bundlePaths(), hasItem(ContainerModelEvaluation.MODEL_EVALUATION_BUNDLE_FILE.toString())); - assertThat(config.bundlePaths(), hasItem(ContainerModelEvaluation.MODEL_INTEGRATION_BUNDLE_FILE.toString())); + assertTrue(config.bundlePaths().contains(ContainerModelEvaluation.MODEL_EVALUATION_BUNDLE_FILE.toString())); + assertTrue(config.bundlePaths().contains(ContainerModelEvaluation.MODEL_INTEGRATION_BUNDLE_FILE.toString())); } @Test @@ -140,7 +137,7 @@ public class ContainerModelBuilderTest extends ContainerModelBuilderTestBase { "</container>" ); createModel(root, clusterElem); AbstractService container = (AbstractService)root.getProducer("container/container.0"); - assertThat(container.getRelativePort(0), is(getDefaults().vespaWebServicePort())); + assertEquals(getDefaults().vespaWebServicePort(), container.getRelativePort(0)); } @Test @@ -154,8 +151,8 @@ public class ContainerModelBuilderTest extends ContainerModelBuilderTestBase { "</container>" ); createModel(root, clusterElem); AbstractService container = (AbstractService)root.getProducer("container/container.0"); - assertThat(container.getRelativePort(0), is(9000)); - assertThat(container.getRelativePort(1), is(not(9001))); + assertEquals(9000, container.getRelativePort(0)); + assertNotEquals(9001, container.getRelativePort(1)); } @Test @@ -265,11 +262,11 @@ public class ContainerModelBuilderTest extends ContainerModelBuilderTestBase { // The handler is still set up. ComponentsConfig.Components userRootHandler = getComponent(componentsConfig(), BindingsOverviewHandler.class.getName()); - assertThat(userRootHandler, notNullValue()); + assertNotNull(userRootHandler); // .. but it has no bindings var discBindingsConfig = root.getConfig(JdiscBindingsConfig.class, "default"); - assertThat(discBindingsConfig.handlers(BindingsOverviewHandler.class.getName()), is(nullValue())); + assertNull(discBindingsConfig.handlers(BindingsOverviewHandler.class.getName())); } @Test @@ -439,7 +436,7 @@ public class ContainerModelBuilderTest extends ContainerModelBuilderTestBase { root = ContentClusterUtils.createMockRoot(new String[]{"host1", "host2"}); createModel(root, containerElem); ContainerCluster cluster = (ContainerCluster)root.getChildren().get("default"); - assertThat(cluster.getContainers().size(), is(2)); + assertEquals(2, cluster.getContainers().size()); assertEquals(root.getConfig(QrMonitorConfig.class, "default/container.0").requesttimeout(), 111); assertEquals(root.getConfig(QrMonitorConfig.class, "default/container.1").requesttimeout(), 222); } @@ -472,7 +469,7 @@ public class ContainerModelBuilderTest extends ContainerModelBuilderTestBase { Map<ComponentId, Component<?, ?>> componentsMap = cluster.getComponentsMap(); Component<?,?> example = componentsMap.get( ComponentId.fromString("test.Exampledocproc")); - assertThat(example.getComponentId().getName(), is("test.Exampledocproc")); + assertEquals("test.Exampledocproc", example.getComponentId().getName()); } @Test @@ -488,7 +485,7 @@ public class ContainerModelBuilderTest extends ContainerModelBuilderTestBase { "</container>"); createModel(root, clusterElem); assertTrue(getContainerCluster("default").getContainers().get(0).getAffinity().isPresent()); - assertThat(getContainerCluster("default").getContainers().get(0).getAffinity().get().cpuSocket(), is(0)); + assertEquals(0, getContainerCluster("default").getContainers().get(0).getAffinity().get().cpuSocket()); } @Test @@ -504,7 +501,7 @@ public class ContainerModelBuilderTest extends ContainerModelBuilderTestBase { .withServices(servicesXml) .build(); VespaModel model = new VespaModel(applicationPackage); - assertThat(model.hostSystem().getHosts().size(), is(1)); + assertEquals(1, model.hostSystem().getHosts().size()); } @Test @@ -950,9 +947,10 @@ public class ContainerModelBuilderTest extends ContainerModelBuilderTestBase { assertEquals("KEY", connectorConfig.ssl().privateKey()); assertEquals(4443, connectorConfig.listenPort()); - assertThat("Connector must use Athenz truststore in a non-public system.", - connectorConfig.ssl().caCertificateFile(), equalTo("/opt/yahoo/share/ssl/certs/athenz_certificate_bundle.pem")); - assertThat(connectorConfig.ssl().caCertificate(), isEmptyString()); + assertEquals("Connector must use Athenz truststore in a non-public system.", + "/opt/yahoo/share/ssl/certs/athenz_certificate_bundle.pem", + connectorConfig.ssl().caCertificateFile()); + assertTrue(connectorConfig.ssl().caCertificate().isEmpty()); } @Test @@ -986,9 +984,10 @@ public class ContainerModelBuilderTest extends ContainerModelBuilderTestBase { assertEquals("KEY", connectorConfig.ssl().privateKey()); assertEquals(4443, connectorConfig.listenPort()); - assertThat("Connector must use Athenz truststore in a non-public system.", - connectorConfig.ssl().caCertificateFile(), equalTo("/opt/yahoo/share/ssl/certs/athenz_certificate_bundle.pem")); - assertThat(connectorConfig.ssl().caCertificate(), isEmptyString()); + assertEquals("Connector must use Athenz truststore in a non-public system.", + "/opt/yahoo/share/ssl/certs/athenz_certificate_bundle.pem", + connectorConfig.ssl().caCertificateFile()); + assertTrue(connectorConfig.ssl().caCertificate().isEmpty()); } @Test diff --git a/config-model/src/test/java/com/yahoo/vespa/model/container/xml/DocprocBuilderTest.java b/config-model/src/test/java/com/yahoo/vespa/model/container/xml/DocprocBuilderTest.java index 5bf64338aa9..514310aa4fb 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/container/xml/DocprocBuilderTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/container/xml/DocprocBuilderTest.java @@ -25,11 +25,8 @@ import java.util.HashMap; import java.util.List; import java.util.Map; -import static org.hamcrest.CoreMatchers.is; -import static org.hamcrest.core.IsNull.notNullValue; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNotNull; -import static org.junit.Assert.assertThat; import static org.junit.Assert.assertTrue; @@ -85,45 +82,45 @@ public class DocprocBuilderTest extends DomBuilderTest { // TODO: re-enable assertions when the appropriate attributes are handled by the builder @Test public void testDocprocCluster() { - assertThat(cluster.getName(), is("banan")); - assertThat(cluster.getDocproc().isCompressDocuments(), is(true)); - //assertThat(cluster.getContainerDocproc().isPreferLocalNode(), is(true)); - //assertThat(cluster.getContainerDocproc().getNumNodesPerClient(), is(2)); + assertEquals("banan", cluster.getName()); + assertTrue(cluster.getDocproc().isCompressDocuments()); + //assertTrue(cluster.getContainerDocproc().isPreferLocalNode()); + //assertEquals(2, cluster.getContainerDocproc().getNumNodesPerClient()); List<ApplicationContainer> services = cluster.getContainers(); - assertThat(services.size(), is(1)); + assertEquals(1, services.size()); ApplicationContainer service = services.get(0); - assertThat(service, notNullValue()); + assertNotNull(service); Map<String, DocprocChain> chains = new HashMap<>(); for (DocprocChain chain : cluster.getDocprocChains().allChains().allComponents()) { chains.put(chain.getId().stringValue(), chain); } - assertThat(chains.size(), is(1)); + assertEquals(1, chains.size()); DocprocChain chain = chains.get("chein"); - assertThat(chain.getId().stringValue(), is("chein")); - assertThat(chain.getInnerComponents().size(), is(1)); + assertEquals("chein", chain.getId().stringValue()); + assertEquals(1, chain.getInnerComponents().size()); DocumentProcessor processor = chain.getInnerComponents().iterator().next(); - assertThat(processor.getComponentId().stringValue(), is("docproc2")); + assertEquals("docproc2", processor.getComponentId().stringValue()); } @Test public void testDocumentManagerConfig() { - assertThat(documentmanagerConfig.enablecompression(), is(true)); + assertTrue(documentmanagerConfig.enablecompression()); } @Test public void testDocprocConfig() { - assertThat(docprocConfig.maxqueuetimems(), is(200000)); + assertEquals(200000, docprocConfig.maxqueuetimems()); } @Test public void testContainerMbusConfig() { - assertThat(containerMbusConfig.enabled(), is(true)); + assertTrue(containerMbusConfig.enabled()); assertTrue(containerMbusConfig.port() >= HostPorts.BASE_PORT); - assertThat(containerMbusConfig.maxpendingcount(), is(300)); - assertThat(containerMbusConfig.maxpendingsize(), is(100)); + assertEquals(300, containerMbusConfig.maxpendingcount()); + assertEquals(100, containerMbusConfig.maxpendingsize()); } @Test @@ -135,37 +132,37 @@ public class DocprocBuilderTest extends DomBuilderTest { } ComponentsConfig.Components docprocHandler = components.get("com.yahoo.docproc.jdisc.DocumentProcessingHandler"); - assertThat(docprocHandler.id(), is("com.yahoo.docproc.jdisc.DocumentProcessingHandler")); - assertThat(docprocHandler.configId(), is("banan/component/com.yahoo.docproc.jdisc.DocumentProcessingHandler")); - assertThat(docprocHandler.classId(), is("com.yahoo.docproc.jdisc.DocumentProcessingHandler")); - assertThat(docprocHandler.bundle(), is("container-search-and-docproc")); + assertEquals("com.yahoo.docproc.jdisc.DocumentProcessingHandler", docprocHandler.id()); + assertEquals("banan/component/com.yahoo.docproc.jdisc.DocumentProcessingHandler", docprocHandler.configId()); + assertEquals("com.yahoo.docproc.jdisc.DocumentProcessingHandler", docprocHandler.classId()); + assertEquals("container-search-and-docproc", docprocHandler.bundle()); ComponentsConfig.Components docproc1 = components.get("docproc1"); - assertThat(docproc1.id(), is("docproc1")); - assertThat(docproc1.configId(), is("banan/docprocchains/component/docproc1")); - assertThat(docproc1.classId(), is("com.yahoo.Docproc1")); - assertThat(docproc1.bundle(), is("docproc1bundle")); + assertEquals("docproc1", docproc1.id()); + assertEquals("banan/docprocchains/component/docproc1", docproc1.configId()); + assertEquals("com.yahoo.Docproc1", docproc1.classId()); + assertEquals("docproc1bundle", docproc1.bundle()); ComponentsConfig.Components docproc2 = components.get("docproc2@chein"); - assertThat(docproc2.id(), is("docproc2@chein")); - assertThat(docproc2.configId(), is("banan/docprocchains/chain/chein/component/docproc2")); - assertThat(docproc2.classId(), is("docproc2")); - assertThat(docproc2.bundle(), is("docproc2")); + assertEquals("docproc2@chein", docproc2.id()); + assertEquals("banan/docprocchains/chain/chein/component/docproc2", docproc2.configId()); + assertEquals("docproc2", docproc2.classId()); + assertEquals("docproc2", docproc2.bundle()); /* ComponentsConfig.Components health = components.get("com.yahoo.container.jdisc.state.StateHandler"); - assertThat(health.id(), is("com.yahoo.container.jdisc.state.StateHandler")); - assertThat(health.classId(), is("com.yahoo.container.jdisc.state.StateHandler")); - assertThat(health.bundle(), is("com.yahoo.container.jdisc.state.StateHandler")); + assertEquals("com.yahoo.container.jdisc.state.StateHandler", health.id())); + assertEquals("com.yahoo.container.jdisc.state.StateHandler", health.classId()); + assertEquals("com.yahoo.container.jdisc.state.StateHandler", health.bundle())); */ ComponentsConfig.Components sourceClient = components.get("source@MbusClient"); assertNotNull(sourceClient); - assertThat(sourceClient.classId(), is("com.yahoo.container.jdisc.messagebus.MbusClientProvider")); - assertThat(sourceClient.bundle(), is("com.yahoo.container.jdisc.messagebus.MbusClientProvider")); + assertEquals("com.yahoo.container.jdisc.messagebus.MbusClientProvider", sourceClient.classId()); + assertEquals("com.yahoo.container.jdisc.messagebus.MbusClientProvider", sourceClient.bundle()); ComponentsConfig.Components intermediateClient = components.get("chain.chein@MbusClient"); assertNotNull(intermediateClient); - assertThat(intermediateClient.classId(), is("com.yahoo.container.jdisc.messagebus.MbusClientProvider")); - assertThat(intermediateClient.bundle(), is("com.yahoo.container.jdisc.messagebus.MbusClientProvider")); + assertEquals("com.yahoo.container.jdisc.messagebus.MbusClientProvider", intermediateClient.classId()); + assertEquals("com.yahoo.container.jdisc.messagebus.MbusClientProvider", intermediateClient.bundle()); } @Test @@ -175,32 +172,32 @@ public class DocprocBuilderTest extends DomBuilderTest { components.put(component.id(), component); } - assertThat(components.size(), is(2)); + assertEquals(2, components.size()); ChainsConfig.Components docproc1 = components.get("docproc1"); - assertThat(docproc1.id(), is("docproc1")); - assertThat(docproc1.dependencies().provides().size(), is(0)); - assertThat(docproc1.dependencies().before().size(), is(0)); - assertThat(docproc1.dependencies().after().size(), is(0)); + assertEquals("docproc1", docproc1.id()); + assertTrue(docproc1.dependencies().provides().isEmpty()); + assertTrue(docproc1.dependencies().before().isEmpty()); + assertTrue(docproc1.dependencies().after().isEmpty()); ChainsConfig.Components docproc2 = components.get("docproc2@chein"); - assertThat(docproc2.id(), is("docproc2@chein")); - assertThat(docproc2.dependencies().provides().size(), is(0)); - assertThat(docproc2.dependencies().before().size(), is(0)); - assertThat(docproc2.dependencies().after().size(), is(0)); + assertEquals("docproc2@chein", docproc2.id()); + assertTrue(docproc2.dependencies().provides().isEmpty()); + assertTrue(docproc2.dependencies().before().isEmpty()); + assertTrue(docproc2.dependencies().after().isEmpty()); Map<String, ChainsConfig.Chains> chainsMap = new HashMap<>(); for (ChainsConfig.Chains chain : chainsConfig.chains()) { chainsMap.put(chain.id(), chain); } - assertThat(chainsMap.size(), is(1)); - assertThat(chainsMap.get("chein").id(), is("chein")); - assertThat(chainsMap.get("chein").components().size(), is(1)); - assertThat(chainsMap.get("chein").components(0), is("docproc2@chein")); - assertThat(chainsMap.get("chein").inherits().size(), is(0)); - assertThat(chainsMap.get("chein").excludes().size(), is(0)); - assertThat(chainsMap.get("chein").phases().size(), is(0)); + assertEquals(1, chainsMap.size()); + assertEquals("chein", chainsMap.get("chein").id()); + assertEquals(1, chainsMap.get("chein").components().size()); + assertEquals("docproc2@chein", chainsMap.get("chein").components(0)); + assertTrue(chainsMap.get("chein").inherits().isEmpty()); + assertTrue(chainsMap.get("chein").excludes().isEmpty()); + assertTrue(chainsMap.get("chein").phases().isEmpty()); } @Test diff --git a/config-model/src/test/java/com/yahoo/vespa/model/container/xml/JvmOptionsTest.java b/config-model/src/test/java/com/yahoo/vespa/model/container/xml/JvmOptionsTest.java index a674a06d45e..ec3b9386dad 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/container/xml/JvmOptionsTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/container/xml/JvmOptionsTest.java @@ -9,7 +9,6 @@ import com.yahoo.config.model.builder.xml.test.DomBuilderTest; import com.yahoo.config.model.deploy.DeployState; import com.yahoo.config.model.deploy.TestProperties; import com.yahoo.config.model.test.MockApplicationPackage; - import com.yahoo.search.config.QrStartConfig; import com.yahoo.vespa.model.VespaModel; import com.yahoo.vespa.model.container.ContainerCluster; @@ -18,11 +17,15 @@ import org.w3c.dom.Element; import org.xml.sax.SAXException; import java.io.IOException; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; import java.util.logging.Level; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; /** * @author baldersheim @@ -145,35 +148,79 @@ public class JvmOptionsTest extends ContainerModelBuilderTestBase { } @Test - public void requireThatJvmGcOptionsAreLogged() throws IOException, SAXException { - verifyLoggingOfJvmOptions(true, "gc-options", "-XX:+UseCMSInitiatingOccupancyOnly foo bar"); - verifyLoggingOfJvmOptions(true, "gc-options", "-XX:+UseConcMarkSweepGC"); - verifyLoggingOfJvmOptions(false, "gc-options", "-XX:+UseConcMarkSweepGC"); + public void requireThatInvalidJvmGcOptionsAreLogged() throws IOException, SAXException { + verifyLoggingOfJvmGcOptions(true, + "-XX:+ParallelGCThreads=8 foo bar", + "foo", "bar"); + verifyLoggingOfJvmGcOptions(true, + "-XX:+UseCMSInitiatingOccupancyOnly foo bar", + "-XX:+UseCMSInitiatingOccupancyOnly", "foo", "bar"); + verifyLoggingOfJvmGcOptions(true, + "-XX:+UseConcMarkSweepGC", + "-XX:+UseConcMarkSweepGC"); + verifyLoggingOfJvmGcOptions(true, + "$(touch /tmp/hello-from-gc-options)", + "$(touch", "/tmp/hello-from-gc-options)"); + + verifyLoggingOfJvmGcOptions(false, + "$(touch /tmp/hello-from-gc-options)", + "$(touch", "/tmp/hello-from-gc-options)"); + + // Valid options, should not log anything + verifyLoggingOfJvmGcOptions(true, "-XX:+ParallelGCThreads=8"); + verifyLoggingOfJvmGcOptions(true, "-XX:MaxTenuringThreshold"); // No + or - after colon + verifyLoggingOfJvmGcOptions(false, "-XX:+UseConcMarkSweepGC"); + } + + @Test + public void requireThatInvalidJvmGcOptionsFailDeployment() throws IOException, SAXException { + try { + buildModelWithJvmOptions(new TestProperties().setHostedVespa(true).failDeploymentWithInvalidJvmOptions(true), + new TestLogger(), + "gc-options", + "-XX:+ParallelGCThreads=8 foo bar"); + fail(); + } catch (IllegalArgumentException e) { + assertTrue(e.getMessage().contains("Invalid JVM GC options from services.xml: bar,foo")); + } } - private void verifyLoggingOfJvmOptions(boolean isHosted, String optionName, String override) throws IOException, SAXException { + private void verifyLoggingOfJvmGcOptions(boolean isHosted, String override, String... invalidOptions) throws IOException, SAXException { + verifyLoggingOfJvmOptions(isHosted, "gc-options", override, invalidOptions); + } + + private void verifyLoggingOfJvmOptions(boolean isHosted, String optionName, String override, String... invalidOptions) throws IOException, SAXException { + TestLogger logger = new TestLogger(); + buildModelWithJvmOptions(isHosted, logger, optionName, override); + + List<String> strings = Arrays.asList(invalidOptions.clone()); + if (strings.isEmpty()) return; + + Collections.sort(strings); + Pair<Level, String> firstOption = logger.msgs.get(0); + assertEquals(Level.WARNING, firstOption.getFirst()); + assertEquals("Invalid JVM " + (optionName.equals("gc-options") ? "GC " : "") + + "options from services.xml: " + String.join(",", strings), firstOption.getSecond()); + } + + private void buildModelWithJvmOptions(boolean isHosted, TestLogger logger, String optionName, String override) throws IOException, SAXException { + buildModelWithJvmOptions(new TestProperties().setHostedVespa(isHosted), logger, optionName, override); + } + + private void buildModelWithJvmOptions(TestProperties properties, TestLogger logger, String optionName, String override) throws IOException, SAXException { String servicesXml = "<container version='1.0'>" + - " <nodes>" + - " <jvm " + optionName + "='" + override + "'/>" + - " <node hostalias='mockhost'/>" + - " </nodes>" + - "</container>"; + " <nodes>" + + " <jvm " + optionName + "='" + override + "'/>" + + " <node hostalias='mockhost'/>" + + " </nodes>" + + "</container>"; ApplicationPackage app = new MockApplicationPackage.Builder().withServices(servicesXml).build(); - TestLogger logger = new TestLogger(); new VespaModel(new NullConfigModelRegistry(), new DeployState.Builder() .applicationPackage(app) .deployLogger(logger) - .properties(new TestProperties().setHostedVespa(isHosted)) + .properties(properties) .build()); - if (isHosted) { - Pair<Level, String> firstOption = logger.msgs.get(0); - assertEquals(Level.INFO, firstOption.getFirst()); - assertEquals("JVM " + (optionName.equals("gc-options") ? "GC " : "") + - "options from services.xml: " + override, firstOption.getSecond()); - } else { - assertEquals(0, logger.msgs.size()); - } } @Test diff --git a/config-model/src/test/java/com/yahoo/vespa/model/container/xml/SchemaBuilderTest.java b/config-model/src/test/java/com/yahoo/vespa/model/container/xml/SchemaBuilderTest.java index e7901ebd5f0..050a2254c9b 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/container/xml/SchemaBuilderTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/container/xml/SchemaBuilderTest.java @@ -19,9 +19,7 @@ import org.w3c.dom.Element; import static com.yahoo.config.model.api.container.ContainerServiceType.QRSERVER; import static com.yahoo.test.Matchers.hasItemWithMethod; import static com.yahoo.vespa.model.container.search.ContainerSearch.QUERY_PROFILE_REGISTRY_CLASS; -import static org.hamcrest.Matchers.containsString; -import static org.hamcrest.Matchers.hasItem; -import static org.hamcrest.Matchers.not; + import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertNotNull; @@ -50,7 +48,7 @@ public class SchemaBuilderTest extends ContainerModelBuilderTestBase { createModel(root, clusterElem); String discBindingsConfig = root.getConfig(JdiscBindingsConfig.class, "default").toString(); - assertThat(discBindingsConfig, containsString(GUIHandler.BINDING_PATH)); + assertTrue(discBindingsConfig.contains(GUIHandler.BINDING_PATH)); ApplicationContainerCluster cluster = (ApplicationContainerCluster)root.getChildren().get("default"); @@ -77,9 +75,9 @@ public class SchemaBuilderTest extends ContainerModelBuilderTestBase { createModel(root, clusterElem); String discBindingsConfig = root.getConfig(JdiscBindingsConfig.class, "default").toString(); - assertThat(discBindingsConfig, containsString(".serverBindings[0] \"http://*/binding0\"")); - assertThat(discBindingsConfig, containsString(".serverBindings[1] \"http://*/binding1\"")); - assertThat(discBindingsConfig, not(containsString("/search/*"))); + assertTrue(discBindingsConfig.contains(".serverBindings[0] \"http://*/binding0\"")); + assertTrue(discBindingsConfig.contains(".serverBindings[1] \"http://*/binding1\"")); + assertFalse(discBindingsConfig.contains("/search/*")); } @Test @@ -238,7 +236,7 @@ public class SchemaBuilderTest extends ContainerModelBuilderTestBase { .findAny() .get(); - assertThat(searchHandler.getInjectedComponentIds(), hasItem("threadpool@search-handler")); + assertTrue(searchHandler.getInjectedComponentIds().contains("threadpool@search-handler")); ContainerThreadpoolConfig config = root.getConfig( ContainerThreadpoolConfig.class, "default/component/" + SearchHandler.HANDLER_CLASS + "/threadpool@search-handler"); diff --git a/config-model/src/test/java/com/yahoo/vespa/model/content/ClusterResourceLimitsTest.java b/config-model/src/test/java/com/yahoo/vespa/model/content/ClusterResourceLimitsTest.java index 94e9b3477a7..8cc7805fe3e 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/content/ClusterResourceLimitsTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/content/ClusterResourceLimitsTest.java @@ -5,16 +5,15 @@ import com.yahoo.config.model.api.ModelContext; import com.yahoo.config.model.deploy.TestProperties; import com.yahoo.text.XML; import com.yahoo.vespa.model.builder.xml.dom.ModelElement; -import org.junit.Rule; import org.junit.Test; -import org.junit.rules.ExpectedException; import org.w3c.dom.Document; import java.util.Optional; -import static org.hamcrest.CoreMatchers.containsString; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; /** * @author geirst @@ -68,9 +67,6 @@ public class ClusterResourceLimitsTest { } } - @Rule - public ExpectedException expectedException = ExpectedException.none(); - @Test public void content_node_limits_are_derived_from_cluster_controller_limits_if_not_set() { assertLimits(0.4, 0.7, 0.7, 0.85, @@ -137,9 +133,12 @@ public class ClusterResourceLimitsTest { @Test public void hosted_exception_is_thrown_when_resource_limits_are_specified() { - expectedException.expect(IllegalArgumentException.class); - expectedException.expectMessage(containsString("Element 'resource-limits' is not allowed to be set")); - hostedBuild(); + try { + hostedBuild(); + fail(); + } catch (IllegalArgumentException e) { + assertTrue(e.getMessage().contains("Element 'resource-limits' is not allowed to be set")); + } } @Test @@ -158,14 +157,22 @@ public class ClusterResourceLimitsTest { public void exception_is_thrown_when_resource_limits_are_out_of_range() { TestProperties featureFlags = new TestProperties(); featureFlags.setResourceLimitDisk(1.1); - expectedException.expect(IllegalArgumentException.class); - expectedException.expectMessage(containsString("Resource limit for disk is set to illegal value 1.1, but must be in the range [0.0, 1.0]")); - hostedBuild(featureFlags, false); + + try { + hostedBuild(featureFlags, false); + fail(); + } catch (IllegalArgumentException e) { + assertTrue(e.getMessage().contains("Resource limit for disk is set to illegal value 1.1, but must be in the range [0.0, 1.0]")); + } featureFlags = new TestProperties(); featureFlags.setResourceLimitDisk(-0.1); - expectedException.expectMessage(containsString("Resource limit for disk is set to illegal value -0.1, but must be in the range [0.0, 1.0]")); - hostedBuild(featureFlags, false); + try { + hostedBuild(featureFlags, false); + fail(); + } catch (IllegalArgumentException e) { + assertTrue(e.getMessage().contains("Resource limit for disk is set to illegal value -0.1, but must be in the range [0.0, 1.0]")); + } } private ClusterResourceLimits hostedBuild() { diff --git a/config-model/src/test/java/com/yahoo/vespa/model/content/ContentClusterTest.java b/config-model/src/test/java/com/yahoo/vespa/model/content/ContentClusterTest.java index 729348a0e3a..191f7f5652a 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/content/ContentClusterTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/content/ContentClusterTest.java @@ -38,15 +38,15 @@ import com.yahoo.vespa.model.routing.DocumentProtocol; import com.yahoo.vespa.model.routing.Routing; import com.yahoo.vespa.model.test.utils.ApplicationPackageUtils; import com.yahoo.vespa.model.test.utils.VespaModelCreatorWithMockPkg; -import org.junit.Rule; import org.junit.Test; -import org.junit.rules.ExpectedException; import java.util.Arrays; import java.util.Collections; import java.util.List; import java.util.Map; import java.util.Optional; +import java.util.OptionalDouble; +import java.util.OptionalInt; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; @@ -60,9 +60,6 @@ public class ContentClusterTest extends ContentBaseTest { private final static String HOSTS = "<admin version='2.0'><adminserver hostalias='mockhost' /></admin>"; - @Rule - public ExpectedException expectedException = ExpectedException.none(); - ContentCluster parse(String xml) { xml = HOSTS + xml; TestRoot root = new TestDriver().buildModel(xml); @@ -866,9 +863,6 @@ public class ContentClusterTest extends ContentBaseTest { @Test public void reserved_document_name_throws_exception() { - expectedException.expect(IllegalArgumentException.class); - expectedException.expectMessage("The following document types conflict with reserved keyword names: 'true'."); - String xml = "<content version=\"1.0\" id=\"storage\">" + " <redundancy>1</redundancy>" + " <documents>" + @@ -880,7 +874,12 @@ public class ContentClusterTest extends ContentBaseTest { "</content>"; List<String> sds = ApplicationPackageUtils.generateSchemas("true"); - new VespaModelCreatorWithMockPkg(null, xml, sds).create(); + try { + new VespaModelCreatorWithMockPkg(null, xml, sds).create(); + fail(); + } catch (IllegalArgumentException e) { + assertTrue(e.getMessage().startsWith("The following document types conflict with reserved keyword names: 'true'.")); + } } private void assertClusterHasBucketSpaceMappings(AllClustersBucketSpacesConfig config, String clusterId, @@ -1026,6 +1025,48 @@ public class ContentClusterTest extends ContentBaseTest { assertTrue(resolveThreePhaseUpdateConfigWithFeatureFlag(true)); } + private int resolveMaxCompactBuffers(OptionalInt maxCompactBuffers) { + TestProperties testProperties = new TestProperties(); + if (maxCompactBuffers.isPresent()) { + testProperties.maxCompactBuffers(maxCompactBuffers.getAsInt()); + } + VespaModel model = createEnd2EndOneNode(testProperties); + ContentCluster cc = model.getContentClusters().get("storage"); + ProtonConfig.Builder protonBuilder = new ProtonConfig.Builder(); + cc.getSearch().getConfig(protonBuilder); + ProtonConfig protonConfig = new ProtonConfig(protonBuilder); + assertEquals(1, protonConfig.documentdb().size()); + return protonConfig.documentdb(0).allocation().max_compact_buffers(); + } + + @Test + public void default_max_compact_buffers_config_controlled_by_properties() { + assertEquals(1, resolveMaxCompactBuffers(OptionalInt.empty())); + assertEquals(2, resolveMaxCompactBuffers(OptionalInt.of(2))); + assertEquals(7, resolveMaxCompactBuffers(OptionalInt.of(7))); + } + + private long resolveMaxTLSSize(OptionalDouble tlsSizeFraction, Optional<Flavor> flavor) throws Exception { + TestProperties testProperties = new TestProperties(); + if (tlsSizeFraction.isPresent()) { + testProperties.tlsSizeFraction(tlsSizeFraction.getAsDouble()); + } + ContentCluster cc = createOneNodeCluster(testProperties, flavor); + ProtonConfig.Builder protonBuilder = new ProtonConfig.Builder(); + cc.getSearch().getSearchNodes().get(0).getConfig(protonBuilder); + ProtonConfig protonConfig = new ProtonConfig(protonBuilder); + return protonConfig.flush().memory().maxtlssize(); + } + @Test + public void default_max_tls_size_controlled_by_properties() throws Exception { + var flavor = new Flavor(new FlavorsConfig.Flavor(new FlavorsConfig.Flavor.Builder().name("test").minDiskAvailableGb(100))); + assertEquals(21474836480L, resolveMaxTLSSize(OptionalDouble.empty(), Optional.empty())); + assertEquals(21474836480L, resolveMaxTLSSize(OptionalDouble.of(0.02), Optional.empty())); + assertEquals(7516192768L, resolveMaxTLSSize(OptionalDouble.empty(), Optional.of(flavor))); + assertEquals(2147483648L, resolveMaxTLSSize(OptionalDouble.of(0.02), Optional.of(flavor))); + assertEquals(3221225472L, resolveMaxTLSSize(OptionalDouble.of(0.03), Optional.of(flavor))); + } + void assertZookeeperServerImplementation(String expectedClassName, ClusterControllerContainerCluster clusterControllerCluster) { for (ClusterControllerContainer c : clusterControllerCluster.getContainers()) { diff --git a/config-model/src/test/java/com/yahoo/vespa/model/content/GenericConfigTest.java b/config-model/src/test/java/com/yahoo/vespa/model/content/GenericConfigTest.java index 2b484529cd3..e6e8a02e951 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/content/GenericConfigTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/content/GenericConfigTest.java @@ -10,8 +10,7 @@ import com.yahoo.vespa.model.test.utils.VespaModelCreatorWithMockPkg; import org.junit.Before; import org.junit.Test; -import static org.hamcrest.CoreMatchers.is; -import static org.junit.Assert.assertThat; +import static org.junit.Assert.assertEquals; /** * @author gjoranv @@ -58,7 +57,7 @@ public class GenericConfigTest { StorageCluster cluster = model.getContentClusters().get("storage").getStorageCluster(); StorFilestorConfig config = model.getConfig(StorFilestorConfig.class, cluster.getConfigId()); - assertThat(config.num_threads(), is(7)); + assertEquals(7, config.num_threads()); } @Test @@ -66,7 +65,7 @@ public class GenericConfigTest { ContentCluster cluster = model.getContentClusters().get("storage"); StorFilestorConfig config = model.getConfig(StorFilestorConfig.class, cluster.getConfigId()); - assertThat(config.num_threads(), is(7)); + assertEquals(7, config.num_threads()); } } diff --git a/config-model/src/test/java/com/yahoo/vespa/model/content/GlobalDistributionValidatorTest.java b/config-model/src/test/java/com/yahoo/vespa/model/content/GlobalDistributionValidatorTest.java index 1143543a213..a47d25ab391 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/content/GlobalDistributionValidatorTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/content/GlobalDistributionValidatorTest.java @@ -3,9 +3,7 @@ package com.yahoo.vespa.model.content; import com.yahoo.documentmodel.NewDocumentType; import com.yahoo.vespa.model.test.utils.VespaModelCreatorWithFilePkg; -import org.junit.Rule; import org.junit.Test; -import org.junit.rules.ExpectedException; import java.util.HashMap; import java.util.HashSet; @@ -16,15 +14,14 @@ import java.util.stream.Stream; import static java.util.Collections.emptyMap; import static java.util.Collections.emptySet; import static java.util.stream.Collectors.toSet; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.fail; /** * @author bjorncs */ public class GlobalDistributionValidatorTest { - @Rule - public final ExpectedException exceptionRule = ExpectedException.none(); - @Test public void validation_succeeds_on_no_documents() { new GlobalDistributionValidator() @@ -44,10 +41,13 @@ public class GlobalDistributionValidatorTest { Fixture fixture = new Fixture() .addNonGlobalDocument(parent) .addNonGlobalDocument(createDocumentType("child", parent)); - exceptionRule.expect(IllegalArgumentException.class); - exceptionRule.expectMessage( - "The following document types are referenced from other documents, but are not globally distributed: 'parent'"); - validate(fixture); + try { + validate(fixture); + fail(); + } catch (IllegalArgumentException e) { + assertEquals("The following document types are referenced from other documents, but are not globally distributed: 'parent'", + e.getMessage()); + } } @Test @@ -65,18 +65,25 @@ public class GlobalDistributionValidatorTest { NewDocumentType child = createDocumentType("child", unknown); Fixture fixture = new Fixture() .addNonGlobalDocument(child); - exceptionRule.expect(IllegalArgumentException.class); - exceptionRule.expectMessage( - "The following document types are referenced from other documents, but are not listed in services.xml: 'unknown'"); - validate(fixture); + + try { + validate(fixture); + fail(); + } catch (IllegalArgumentException e) { + assertEquals("The following document types are referenced from other documents, but are not listed in services.xml: 'unknown'", + e.getMessage()); + } } @Test public void throws_exception_if_referenced_document_not_global_end_to_end() { - exceptionRule.expect(IllegalArgumentException.class); - exceptionRule.expectMessage( - "The following document types are referenced from other documents, but are not globally distributed: 'parent'"); - new VespaModelCreatorWithFilePkg("src/test/cfg/application/validation/global_distribution_validation/").create(); + try { + new VespaModelCreatorWithFilePkg("src/test/cfg/application/validation/global_distribution_validation/").create(); + fail(); + } catch (IllegalArgumentException e) { + assertEquals("The following document types are referenced from other documents, but are not globally distributed: 'parent'", + e.getMessage()); + } } private static NewDocumentType createDocumentType(String name, NewDocumentType... references) { diff --git a/config-model/src/test/java/com/yahoo/vespa/model/content/IndexedHierarchicDistributionTest.java b/config-model/src/test/java/com/yahoo/vespa/model/content/IndexedHierarchicDistributionTest.java index 125d8c7fc40..7d7ba7994aa 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/content/IndexedHierarchicDistributionTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/content/IndexedHierarchicDistributionTest.java @@ -15,11 +15,10 @@ import java.util.Optional; import static com.yahoo.config.model.test.TestUtil.joinLines; import static com.yahoo.vespa.model.content.utils.ContentClusterUtils.createCluster; import static com.yahoo.vespa.model.content.utils.ContentClusterUtils.createClusterXml; -import static org.hamcrest.Matchers.containsString; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertThat; import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; /** * Unit tests for hierarchic distribution in an indexed content cluster. @@ -178,9 +177,9 @@ public class IndexedHierarchicDistributionTest { public void requireThatWeMustHaveOnlyOneGroupLevel() { try { getIllegalMultipleGroupsLevelCluster(); - assertFalse("Did not get expected Exception", true); + fail("Did not get expected Exception"); } catch (Exception e) { - assertThat(e.getMessage(), containsString("sub group 'group0' contains 2 sub groups.")); + assertTrue(e.getMessage().contains("sub group 'group0' contains 2 sub groups.")); } } @@ -188,9 +187,9 @@ public class IndexedHierarchicDistributionTest { public void requireThatLeafGroupsMustHaveEqualNumberOfNodes() { try { getIllegalGroupsCluster(); - assertFalse("Did not get expected Exception", true); + fail("Did not get expected Exception"); } catch (Exception e) { - assertThat(e.getMessage(), containsString("leaf group 'group0' contains 1 node(s) while leaf group 'group1' contains 2 node(s)")); + assertTrue(e.getMessage().contains("leaf group 'group0' contains 1 node(s) while leaf group 'group1' contains 2 node(s)")); } } @@ -200,7 +199,7 @@ public class IndexedHierarchicDistributionTest { DispatchGroup dg = c.getSearch().getIndexed().getRootDispatch(); assertEquals(8, dg.getRowBits()); assertEquals(3, dg.getNumPartitions()); - assertEquals(true, dg.useFixedRowInDispatch()); + assertTrue(dg.useFixedRowInDispatch()); ArrayList<SearchInterface> list = new ArrayList<>(); for(SearchInterface si : dg.getSearchersIterable()) { list.add(si); @@ -222,9 +221,9 @@ public class IndexedHierarchicDistributionTest { public void requireThatLeafGroupsCountMustBeAFactorOfRedundancy() { try { getTwoGroupsCluster(3, 3, "2|*"); - assertFalse("Did not get expected Exception", true); + fail("Did not get expected Exception"); } catch (Exception e) { - assertThat(e.getMessage(), containsString("Expected number of leaf groups (2) to be a factor of redundancy (3)")); + assertTrue(e.getMessage().contains("Expected number of leaf groups (2) to be a factor of redundancy (3)")); } } @@ -232,9 +231,9 @@ public class IndexedHierarchicDistributionTest { public void requireThatRedundancyPerGroupMustBeIsEqual() { try { getTwoGroupsCluster(4, 4, "1|*"); - assertFalse("Did not get expected Exception", true); + fail("Did not get expected Exception"); } catch (Exception e) { - assertThat(e.getMessage(), containsString("Expected distribution partitions should be '2|*'")); + assertTrue(e.getMessage().contains("Expected distribution partitions should be '2|*'")); } } @@ -242,9 +241,9 @@ public class IndexedHierarchicDistributionTest { public void requireThatReadyCopiesMustBeEqualToRedundancy() { try { getTwoGroupsCluster(4, 3, "2|*"); - assertFalse("Did not get expected Exception", true); + fail("Did not get expected Exception"); } catch (Exception e) { - assertThat(e.getMessage(), containsString("Expected equal amount of ready copies per group")); + assertTrue(e.getMessage().contains("Expected equal amount of ready copies per group")); } } diff --git a/config-model/src/test/java/com/yahoo/vespa/model/content/IndexedTest.java b/config-model/src/test/java/com/yahoo/vespa/model/content/IndexedTest.java index 4522e5cfd03..d9dbd6eeca7 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/content/IndexedTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/content/IndexedTest.java @@ -17,14 +17,11 @@ import com.yahoo.vespa.model.test.utils.ApplicationPackageUtils; import com.yahoo.vespa.model.test.utils.VespaModelCreatorWithMockPkg; import org.junit.Test; -import java.util.Arrays; import java.util.List; -import static org.hamcrest.core.Is.is; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertNotNull; -import static org.junit.Assert.assertThat; /** * Test for using the content model to create indexed search clusters. @@ -113,12 +110,12 @@ public class IndexedTest extends ContentBaseTest { private VespaModelCreatorWithMockPkg getIndexedVespaModelCreator() { List<String> sds = ApplicationPackageUtils.generateSchemas("type1", "type2", "type3"); - return new VespaModelCreatorWithMockPkg(getHosts(), createProtonIndexedVespaServices(Arrays.asList("type1", "type2", "type3")), sds); + return new VespaModelCreatorWithMockPkg(getHosts(), createProtonIndexedVespaServices(List.of("type1", "type2", "type3")), sds); } private VespaModel getStreamingVespaModel() { List<String> sds = ApplicationPackageUtils.generateSchemas("type1"); - return new VespaModelCreatorWithMockPkg(getHosts(), createProtonStreamingVespaServices(Arrays.asList("type1")), sds).create(); + return new VespaModelCreatorWithMockPkg(getHosts(), createProtonStreamingVespaServices(List.of("type1")), sds).create(); } @Test @@ -190,9 +187,9 @@ public class IndexedTest extends ContentBaseTest { assertNotNull(s); assertFalse(s.getSearch().hasIndexedCluster()); ClusterListConfig config = model.getConfig(ClusterListConfig.class, VespaModel.ROOT_CONFIGID); - assertThat(config.storage().size(), is(1)); - assertThat(config.storage(0).name(), is("test")); - assertThat(config.storage(0).configid(), is("test")); + assertEquals(1, config.storage().size()); + assertEquals("test", config.storage(0).name()); + assertEquals("test", config.storage(0).configid()); } @Test diff --git a/config-model/src/test/java/com/yahoo/vespa/model/content/IndexingAndDocprocRoutingTest.java b/config-model/src/test/java/com/yahoo/vespa/model/content/IndexingAndDocprocRoutingTest.java index d2dcc307871..cea8d724c30 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/content/IndexingAndDocprocRoutingTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/content/IndexingAndDocprocRoutingTest.java @@ -23,10 +23,8 @@ import java.util.HashMap; import java.util.List; import java.util.Map; -import static org.hamcrest.Matchers.*; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNotNull; -import static org.junit.Assert.assertThat; import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; @@ -247,7 +245,7 @@ public class IndexingAndDocprocRoutingTest extends ContentBaseTest { expectedDocprocChainStrings.add(spec.name); } - assertThat(actualDocprocChains, hasItems(expectedDocprocChainStrings.toArray(new String[0]))); + assertTrue(actualDocprocChains.containsAll(expectedDocprocChainStrings)); } } @@ -279,32 +277,32 @@ public class IndexingAndDocprocRoutingTest extends ContentBaseTest { HopBlueprint indexingHop = table.getHop("indexing"); - assertThat(indexingHop, not(nullValue())); + assertNotNull(indexingHop); - assertThat(indexingHop.getNumDirectives(), is(1)); - assertThat(indexingHop.getDirective(0), instanceOf(PolicyDirective.class)); - assertThat(indexingHop.getDirective(0).toString(), is("[DocumentRouteSelector]")); + assertEquals(1, indexingHop.getNumDirectives()); + assertTrue(indexingHop.getDirective(0) instanceof PolicyDirective); + assertEquals("[DocumentRouteSelector]", indexingHop.getDirective(0).toString()); //assertThat(indexingHop.getNumRecipients(), is(1)); //assertThat(indexingHop.getRecipient(0).getServiceName(), is(searchClusterName)); Route route = table.getRoute(searchClusterName); assertNotNull(route); - assertThat(route.getNumHops(), is(1)); + assertEquals(1, route.getNumHops()); Hop messageTypeHop = route.getHop(0); - assertThat(messageTypeHop.getNumDirectives(), is(1)); - assertThat(messageTypeHop.getDirective(0), instanceOf(PolicyDirective.class)); - assertThat(messageTypeHop.getDirective(0).toString(), is("[MessageType:" + searchClusterName + "]")); + assertEquals(1, messageTypeHop.getNumDirectives()); + assertTrue(messageTypeHop.getDirective(0) instanceof PolicyDirective); + assertEquals("[MessageType:" + searchClusterName + "]", messageTypeHop.getDirective(0).toString()); PolicyDirective messageTypeDirective = (PolicyDirective) messageTypeHop.getDirective(0); - assertThat(messageTypeDirective.getName(), is("MessageType")); - assertThat(messageTypeDirective.getParam(), is(searchClusterName)); + assertEquals("MessageType", messageTypeDirective.getName()); + assertEquals(searchClusterName, messageTypeDirective.getParam()); String indexingRouteName = DocumentProtocol.getIndexedRouteName(model.getContentClusters().get(searchClusterName).getConfigId()); Route indexingRoute = table.getRoute(indexingRouteName); - assertThat(indexingRoute.getNumHops(), is(2)); - assertThat(indexingRoute.getHop(0).getServiceName(), is(indexingHopName)); - assertThat(indexingRoute.getHop(1), not(nullValue())); + assertEquals(2, indexingRoute.getNumHops()); + assertEquals(indexingHopName, indexingRoute.getHop(0).getServiceName()); + assertNotNull(indexingRoute.getHop(1)); } private void assertFeedingRouteIndexed(VespaModel model, String searchClusterName, String indexingHopName) { @@ -323,9 +321,9 @@ public class IndexingAndDocprocRoutingTest extends ContentBaseTest { RoutingTable table = new RoutingTable(documentProtocol.getRoutingTableSpec()); Route indexingRoute = table.getRoute("searchcluster-index"); - assertThat(indexingRoute.getNumHops(), is(2)); - assertThat(indexingRoute.getHop(0).toString(), is(indexingHopName)); - assertThat(indexingRoute.getHop(1).toString(), is("[Content:cluster=" + searchClusterName + "]")); + assertEquals(2, indexingRoute.getNumHops()); + assertEquals(indexingHopName, indexingRoute.getHop(0).toString()); + assertEquals("[Content:cluster=" + searchClusterName + "]", indexingRoute.getHop(1).toString()); } diff --git a/config-model/src/test/java/com/yahoo/vespa/model/content/MonitoringConfigSnoopTest.java b/config-model/src/test/java/com/yahoo/vespa/model/content/MonitoringConfigSnoopTest.java index ff34b6ac0d8..d1ce16bfe73 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/content/MonitoringConfigSnoopTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/content/MonitoringConfigSnoopTest.java @@ -6,8 +6,7 @@ import com.yahoo.config.model.test.TestRoot; import com.yahoo.metrics.MetricsmanagerConfig; import org.junit.Test; -import static org.hamcrest.Matchers.is; -import static org.junit.Assert.assertThat; +import static org.junit.Assert.assertEquals; /** @@ -45,11 +44,11 @@ public class MonitoringConfigSnoopTest { } @Test - public void correct_config_is_snooped() throws Exception { + public void correct_config_is_snooped() { initRoot(60); - assertThat(getConfig().snapshot().periods().size(), is(2)); - assertThat(getConfig().snapshot().periods(0), is(60)); - assertThat(getConfig().snapshot().periods(1), is(300)); + assertEquals(2, getConfig().snapshot().periods().size()); + assertEquals(60, getConfig().snapshot().periods(0)); + assertEquals(300, getConfig().snapshot().periods(1)); } @Test @@ -60,13 +59,13 @@ public class MonitoringConfigSnoopTest { TestDriver tester = new TestDriver(); root = tester.buildModel(getAdminXmlIntervalNotSpecified + getContent()); - assertThat(getConfig().snapshot().periods().size(), is(2)); - assertThat(getConfig().snapshot().periods(0), is(60)); - assertThat(getConfig().snapshot().periods(1), is(300)); + assertEquals(2, getConfig().snapshot().periods().size()); + assertEquals(60, getConfig().snapshot().periods(0)); + assertEquals(300, getConfig().snapshot().periods(1)); } @Test(expected = Exception.class) - public void invalid_model_1() throws Exception { + public void invalid_model_1() { initRoot(120); } } diff --git a/config-model/src/test/java/com/yahoo/vespa/model/content/ReservedDocumentTypeNameValidatorTest.java b/config-model/src/test/java/com/yahoo/vespa/model/content/ReservedDocumentTypeNameValidatorTest.java index e73cd98f32e..684f07d99f0 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/content/ReservedDocumentTypeNameValidatorTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/content/ReservedDocumentTypeNameValidatorTest.java @@ -2,9 +2,7 @@ package com.yahoo.vespa.model.content; import com.yahoo.documentmodel.NewDocumentType; -import org.junit.Rule; import org.junit.Test; -import org.junit.rules.ExpectedException; import java.util.Arrays; import java.util.Collections; @@ -14,10 +12,11 @@ import java.util.TreeMap; import java.util.function.Function; import java.util.stream.Collectors; -public class ReservedDocumentTypeNameValidatorTest { +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; - @Rule - public ExpectedException expectedException = ExpectedException.none(); +public class ReservedDocumentTypeNameValidatorTest { private static Map<String, NewDocumentType> asDocTypeMapping(List<String> typeNames) { return typeNames.stream().collect(Collectors.toMap(Function.identity(), n -> new NewDocumentType(new NewDocumentType.Name(n)))); @@ -25,16 +24,19 @@ public class ReservedDocumentTypeNameValidatorTest { @Test public void exception_thrown_on_reserved_names() { - expectedException.expect(IllegalArgumentException.class); - expectedException.expectMessage("The following document types conflict with reserved keyword names: " + - "'and', 'false', 'id', 'not', 'null', 'or', 'true'. " + - "Reserved keywords are 'and', 'false', 'id', 'not', 'null', 'or', 'true'"); - // Ensure ordering is consistent for testing Map<String, NewDocumentType> orderedDocTypes = new TreeMap<>(asDocTypeMapping(ReservedDocumentTypeNameValidator.ORDERED_RESERVED_NAMES)); ReservedDocumentTypeNameValidator validator = new ReservedDocumentTypeNameValidator(); - validator.validate(orderedDocTypes); + try { + validator.validate(orderedDocTypes); + fail(); + } catch (IllegalArgumentException e) { + assertEquals("The following document types conflict with reserved keyword names: " + + "'and', 'false', 'id', 'not', 'null', 'or', 'true'. " + + "Reserved keywords are 'and', 'false', 'id', 'not', 'null', 'or', 'true'", + e.getMessage()); + } } @Test @@ -45,13 +47,15 @@ public class ReservedDocumentTypeNameValidatorTest { @Test public void validation_is_case_insensitive() { - expectedException.expect(IllegalArgumentException.class); - expectedException.expectMessage("The following document types conflict with reserved keyword names: " + - "'NULL', 'True', 'anD'."); - ReservedDocumentTypeNameValidator validator = new ReservedDocumentTypeNameValidator(); Map<String, NewDocumentType> orderedDocTypes = new TreeMap<>(asDocTypeMapping(Arrays.asList("NULL", "True", "anD"))); - validator.validate(orderedDocTypes); + try { + validator.validate(orderedDocTypes); + fail(); + } catch (IllegalArgumentException e) { + assertTrue(e.getMessage().startsWith("The following document types conflict with reserved keyword names: " + + "'NULL', 'True', 'anD'.")); + } } } diff --git a/config-model/src/test/java/com/yahoo/vespa/model/generic/GenericServicesModelTest.java b/config-model/src/test/java/com/yahoo/vespa/model/generic/GenericServicesModelTest.java index 234a313651c..7b7fc83131e 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/generic/GenericServicesModelTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/generic/GenericServicesModelTest.java @@ -13,7 +13,7 @@ import org.xml.sax.SAXException; import java.io.IOException; import static org.hamcrest.CoreMatchers.is; -import static org.junit.Assert.assertThat; +import static org.hamcrest.MatcherAssert.assertThat; /** * @author Ulf Lilleengen diff --git a/config-model/src/test/java/com/yahoo/vespa/model/search/NodeResourcesTuningTest.java b/config-model/src/test/java/com/yahoo/vespa/model/search/NodeResourcesTuningTest.java index 649a445e7a5..e2645e0b39e 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/search/NodeResourcesTuningTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/search/NodeResourcesTuningTest.java @@ -5,6 +5,7 @@ import com.yahoo.collections.Pair; import com.yahoo.config.provision.Flavor; import com.yahoo.config.provisioning.FlavorsConfig; import com.yahoo.vespa.config.search.core.ProtonConfig; +import com.yahoo.vespa.model.container.ApplicationContainerCluster; import org.junit.Test; import java.util.Arrays; @@ -31,8 +32,8 @@ public class NodeResourcesTuningTest { @Test public void require_that_hwinfo_memory_size_is_set() { - assertEquals(24 * GB, configFromMemorySetting(24 + reservedMemoryGb, false).hwinfo().memory().size()); - assertEquals(combinedFactor * 24 * GB, configFromMemorySetting(24 + reservedMemoryGb, true).hwinfo().memory().size(), 1000); + assertEquals(24 * GB, configFromMemorySetting(24 + reservedMemoryGb, 0).hwinfo().memory().size()); + assertEquals(combinedFactor * 24 * GB, configFromMemorySetting(24 + reservedMemoryGb, ApplicationContainerCluster.heapSizePercentageOfTotalNodeMemoryWhenCombinedCluster*0.01).hwinfo().memory().size(), 1000); } @Test @@ -132,11 +133,13 @@ public class NodeResourcesTuningTest { @Test public void require_that_flush_strategy_tls_size_is_set_based_on_available_disk() { - assertFlushStrategyTlsSize(7 * GB, 100); - assertFlushStrategyTlsSize(35 * GB, 500); - assertFlushStrategyTlsSize(84 * GB, 1200); - assertFlushStrategyTlsSize(100 * GB, 1720); - assertFlushStrategyTlsSize(100 * GB, 24000); + assertFlushStrategyTlsSize(2 * GB, 10, 0.05); + assertFlushStrategyTlsSize(7 * GB, 100, 0.07); + assertFlushStrategyTlsSize(5 * GB, 100, 0.05); + assertFlushStrategyTlsSize(35 * GB, 500, 0.07); + assertFlushStrategyTlsSize(84 * GB, 1200, 0.07); + assertFlushStrategyTlsSize(100 * GB, 1720, 0.07); + assertFlushStrategyTlsSize(100 * GB, 24000, 0.07); } @Test @@ -153,14 +156,14 @@ public class NodeResourcesTuningTest { @Test public void require_that_summary_cache_max_bytes_is_set_based_on_memory() { - assertEquals(1*GB / 20, configFromMemorySetting(1 + reservedMemoryGb, false).summary().cache().maxbytes()); - assertEquals(256*GB / 20, configFromMemorySetting(256 + reservedMemoryGb, false).summary().cache().maxbytes()); + assertEquals(1*GB / 20, configFromMemorySetting(1 + reservedMemoryGb, 0).summary().cache().maxbytes()); + assertEquals(256*GB / 20, configFromMemorySetting(256 + reservedMemoryGb, 0).summary().cache().maxbytes()); } @Test public void require_that_summary_cache_memory_is_reduced_with_combined_cluster() { - assertEquals(combinedFactor * 1*GB / 20, configFromMemorySetting(1 + reservedMemoryGb, true).summary().cache().maxbytes(), 1000); - assertEquals(combinedFactor * 256*GB / 20, configFromMemorySetting(256 + reservedMemoryGb, true).summary().cache().maxbytes(), 1000); + assertEquals(combinedFactor * 1*GB / 20, configFromMemorySetting(1 + reservedMemoryGb, ApplicationContainerCluster.heapSizePercentageOfTotalNodeMemoryWhenCombinedCluster*0.01).summary().cache().maxbytes(), 1000); + assertEquals(combinedFactor * 256*GB / 20, configFromMemorySetting(256 + reservedMemoryGb, ApplicationContainerCluster.heapSizePercentageOfTotalNodeMemoryWhenCombinedCluster*0.01).summary().cache().maxbytes(), 1000); } @Test @@ -169,16 +172,16 @@ public class NodeResourcesTuningTest { } private static void assertDocumentStoreMaxFileSize(long expFileSizeBytes, int wantedMemoryGb) { - assertEquals(expFileSizeBytes, configFromMemorySetting(wantedMemoryGb + reservedMemoryGb, false).summary().log().maxfilesize()); + assertEquals(expFileSizeBytes, configFromMemorySetting(wantedMemoryGb + reservedMemoryGb, 0).summary().log().maxfilesize()); } private static void assertFlushStrategyMemory(long expMemoryBytes, int wantedMemoryGb) { - assertEquals(expMemoryBytes, configFromMemorySetting(wantedMemoryGb + reservedMemoryGb, false).flush().memory().maxmemory()); - assertEquals(expMemoryBytes, configFromMemorySetting(wantedMemoryGb + reservedMemoryGb, false).flush().memory().each().maxmemory()); + assertEquals(expMemoryBytes, configFromMemorySetting(wantedMemoryGb + reservedMemoryGb, 0).flush().memory().maxmemory()); + assertEquals(expMemoryBytes, configFromMemorySetting(wantedMemoryGb + reservedMemoryGb, 0).flush().memory().each().maxmemory()); } - private static void assertFlushStrategyTlsSize(long expTlsSizeBytes, int diskGb) { - assertEquals(expTlsSizeBytes, configFromDiskSetting(diskGb).flush().memory().maxtlssize()); + private static void assertFlushStrategyTlsSize(long expTlsSizeBytes, int diskGb, double tlsSizeFraction) { + assertEquals(expTlsSizeBytes, configFromDiskSetting(diskGb, tlsSizeFraction).flush().memory().maxtlssize()); } private static void assertSummaryReadIo(ProtonConfig.Summary.Read.Io.Enum expValue, boolean fastDisk) { @@ -194,55 +197,67 @@ public class NodeResourcesTuningTest { } private static void assertWriteFilter(double expMemoryLimit, int memoryGb) { - assertEquals(expMemoryLimit, configFromMemorySetting(memoryGb, false).writefilter().memorylimit(), delta); + assertEquals(expMemoryLimit, configFromMemorySetting(memoryGb, 0).writefilter().memorylimit(), delta); } private static ProtonConfig configFromDiskSetting(boolean fastDisk) { - return getConfig(new FlavorsConfig.Flavor.Builder().fastDisk(fastDisk), false); + return getConfig(new FlavorsConfig.Flavor.Builder().fastDisk(fastDisk)); } private static ProtonConfig configFromDiskSetting(int diskGb) { - return getConfig(new FlavorsConfig.Flavor.Builder().minDiskAvailableGb(diskGb), false); + return configFromDiskSetting(diskGb, 0.07); + } + private static ProtonConfig configFromDiskSetting(int diskGb, double tlsSizeFraction) { + return getConfig(new FlavorsConfig.Flavor.Builder().minDiskAvailableGb(diskGb), 0, tlsSizeFraction); } - private static ProtonConfig configFromMemorySetting(double memoryGb, boolean combined) { - return getConfig(new FlavorsConfig.Flavor.Builder().minMainMemoryAvailableGb(memoryGb), combined); + private static ProtonConfig configFromMemorySetting(double memoryGb, double fractionOfMemoryReserved) { + return getConfig(new FlavorsConfig.Flavor.Builder().minMainMemoryAvailableGb(memoryGb), fractionOfMemoryReserved, 0.07); } private static ProtonConfig configFromMemorySetting(double memoryGb, ProtonConfig.Builder builder) { return getConfig(new FlavorsConfig.Flavor.Builder() - .minMainMemoryAvailableGb(memoryGb), builder, false); + .minMainMemoryAvailableGb(memoryGb), builder); } private static ProtonConfig configFromNumCoresSetting(double numCores) { - return getConfig(new FlavorsConfig.Flavor.Builder().minCpuCores(numCores), false); + return getConfig(new FlavorsConfig.Flavor.Builder().minCpuCores(numCores)); } private static ProtonConfig configFromNumCoresSetting(double numCores, int numThreadsPerSearch) { return getConfig(new FlavorsConfig.Flavor.Builder().minCpuCores(numCores), - new ProtonConfig.Builder(), numThreadsPerSearch, false); + new ProtonConfig.Builder(), numThreadsPerSearch); } private static ProtonConfig configFromEnvironmentType(boolean docker) { String environment = (docker ? "DOCKER_CONTAINER" : "undefined"); - return getConfig(new FlavorsConfig.Flavor.Builder().environment(environment), false); + return getConfig(new FlavorsConfig.Flavor.Builder().environment(environment)); } - private static ProtonConfig getConfig(FlavorsConfig.Flavor.Builder flavorBuilder, boolean combined) { - return getConfig(flavorBuilder, new ProtonConfig.Builder(), combined); + private static ProtonConfig getConfig(FlavorsConfig.Flavor.Builder flavorBuilder) { + return getConfig(flavorBuilder, new ProtonConfig.Builder()); } - private static ProtonConfig getConfig(FlavorsConfig.Flavor.Builder flavorBuilder, ProtonConfig.Builder protonBuilder, boolean combined) { - flavorBuilder.name("my_flavor"); - NodeResourcesTuning tuning = new NodeResourcesTuning(new Flavor(new FlavorsConfig.Flavor(flavorBuilder)).resources(), 1, combined); - tuning.getConfig(protonBuilder); - return new ProtonConfig(protonBuilder); + private static ProtonConfig getConfig(FlavorsConfig.Flavor.Builder flavorBuilder, double fractionOfMemoryReserved, double tlsSizeFraction) { + return getConfig(flavorBuilder, new ProtonConfig.Builder(), fractionOfMemoryReserved, tlsSizeFraction); + } + + private static ProtonConfig getConfig(FlavorsConfig.Flavor.Builder flavorBuilder, ProtonConfig.Builder protonBuilder) { + return getConfig(flavorBuilder, protonBuilder,1); + } + private static ProtonConfig getConfig(FlavorsConfig.Flavor.Builder flavorBuilder, ProtonConfig.Builder protonBuilder, double fractionOfMemoryReserved, double tlsSizeFraction) { + return getConfig(flavorBuilder, protonBuilder, 1, fractionOfMemoryReserved, tlsSizeFraction); + } + + private static ProtonConfig getConfig(FlavorsConfig.Flavor.Builder flavorBuilder, ProtonConfig.Builder protonBuilder, + int numThreadsPerSearch) { + return getConfig(flavorBuilder, protonBuilder, numThreadsPerSearch, 0, 0.07); } private static ProtonConfig getConfig(FlavorsConfig.Flavor.Builder flavorBuilder, ProtonConfig.Builder protonBuilder, - int numThreadsPerSearch, boolean combined) { + int numThreadsPerSearch, double fractionOfMemoryReserved, double tlsSizeFraction) { flavorBuilder.name("my_flavor"); - NodeResourcesTuning tuning = new NodeResourcesTuning(new Flavor(new FlavorsConfig.Flavor(flavorBuilder)).resources(), numThreadsPerSearch, combined); + NodeResourcesTuning tuning = new NodeResourcesTuning(new Flavor(new FlavorsConfig.Flavor(flavorBuilder)).resources(), numThreadsPerSearch, fractionOfMemoryReserved, tlsSizeFraction); tuning.getConfig(protonBuilder); return new ProtonConfig(protonBuilder); } diff --git a/config-model/src/test/java/com/yahoo/vespa/model/search/test/SchemaNodeTest.java b/config-model/src/test/java/com/yahoo/vespa/model/search/test/SchemaNodeTest.java index cca2d817df4..d000f83d6cd 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/search/test/SchemaNodeTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/search/test/SchemaNodeTest.java @@ -4,7 +4,6 @@ package com.yahoo.vespa.model.search.test; import com.yahoo.config.model.api.ModelContext; import com.yahoo.config.model.deploy.DeployState; import com.yahoo.config.model.deploy.TestProperties; -import com.yahoo.config.model.producer.AbstractConfigProducer; import com.yahoo.config.model.test.MockRoot; import com.yahoo.searchlib.TranslogserverConfig; import com.yahoo.vespa.config.search.core.ProtonConfig; @@ -14,8 +13,6 @@ import com.yahoo.vespa.model.HostResource; import com.yahoo.vespa.model.search.NodeSpec; import com.yahoo.vespa.model.search.SearchNode; import com.yahoo.vespa.model.search.TransactionLogServer; -import org.hamcrest.CoreMatchers; -import org.junit.Assert; import org.junit.Test; import java.util.Optional; @@ -52,12 +49,14 @@ public class SchemaNodeTest { } private static SearchNode createSearchNode(MockRoot root, String name, int distributionKey, - NodeSpec nodeSpec, boolean flushOnShutDown, boolean isHosted, boolean combined) { - return SearchNode.create(root, name, distributionKey, nodeSpec, "mycluster", null, flushOnShutDown, Optional.empty(), Optional.empty(), isHosted, combined); + NodeSpec nodeSpec, boolean flushOnShutDown, boolean isHosted) { + return SearchNode.create(root, name, distributionKey, nodeSpec, "mycluster", null, flushOnShutDown, + Optional.empty(), Optional.empty(), isHosted, 0.0, + root.getDeployState().featureFlags().tlsSizeFraction()); } private static SearchNode createSearchNode(MockRoot root) { - return createSearchNode(root, "mynode", 3, new NodeSpec(7, 5), true, true, false); + return createSearchNode(root, "mynode", 3, new NodeSpec(7, 5), true, true); } @Test @@ -70,7 +69,7 @@ public class SchemaNodeTest { @Test public void requireThatBasedirIsCorrectForElasticMode() { MockRoot root = new MockRoot(""); - SearchNode node = createSearchNode(root, "mynode", 3, new NodeSpec(7, 5), false, root.getDeployState().isHosted(), false); + SearchNode node = createSearchNode(root, "mynode", 3, new NodeSpec(7, 5), false, root.getDeployState().isHosted()); prepare(root, node, true); assertBaseDir(Defaults.getDefaults().underVespaHome("var/db/vespa/search/cluster.mycluster/n3"), node); } @@ -78,7 +77,7 @@ public class SchemaNodeTest { @Test public void requireThatPreShutdownCommandIsEmptyWhenNotActivated() { MockRoot root = new MockRoot(""); - SearchNode node = createSearchNode(root, "mynode", 3, new NodeSpec(7, 5), false, root.getDeployState().isHosted(), false); + SearchNode node = createSearchNode(root, "mynode", 3, new NodeSpec(7, 5), false, root.getDeployState().isHosted()); node.setHostResource(new HostResource(new Host(node, "mynbode"))); node.initService(root.deployLogger()); assertFalse(node.getPreShutdownCommand().isPresent()); @@ -87,12 +86,11 @@ public class SchemaNodeTest { @Test public void requireThatPreShutdownCommandUsesPrepareRestartWhenActivated() { MockRoot root = new MockRoot(""); - SearchNode node = createSearchNode(root, "mynode2", 4, new NodeSpec(7, 5), true, root.getDeployState().isHosted(), false); + SearchNode node = createSearchNode(root, "mynode2", 4, new NodeSpec(7, 5), true, root.getDeployState().isHosted()); node.setHostResource(new HostResource(new Host(node, "mynbode2"))); node.initService(root.deployLogger()); assertTrue(node.getPreShutdownCommand().isPresent()); - Assert.assertThat(node.getPreShutdownCommand().get(), - CoreMatchers.containsString("vespa-proton-cmd " + node.getRpcPort() + " prepareRestart")); + assertTrue(node.getPreShutdownCommand().get().contains("vespa-proton-cmd " + node.getRpcPort() + " prepareRestart")); } private MockRoot createRoot(ModelContext.Properties properties) { diff --git a/config-model/src/test/java/com/yahoo/vespa/model/storage/test/StorageModelTestCase.java b/config-model/src/test/java/com/yahoo/vespa/model/storage/test/StorageModelTestCase.java index a2aa330efbe..72d6a80e711 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/storage/test/StorageModelTestCase.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/storage/test/StorageModelTestCase.java @@ -8,10 +8,8 @@ import com.yahoo.vespa.model.content.cluster.ContentCluster; import com.yahoo.vespa.model.test.utils.VespaModelCreatorWithFilePkg; import org.junit.Test; -import static org.hamcrest.core.Is.is; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNotNull; -import static org.junit.Assert.assertThat; /** * Tests storage model @@ -50,7 +48,7 @@ public class StorageModelTestCase { MetricsmanagerConfig.Builder builder = new MetricsmanagerConfig.Builder(); contentCluster.getConfig(builder); MetricsmanagerConfig config = new MetricsmanagerConfig(builder); - assertThat(config.snapshot().periods(0), is(60)); + assertEquals(60, config.snapshot().periods(0)); } } diff --git a/config-model/src/test/java/com/yahoo/vespa/model/test/VespaModelTestCase.java b/config-model/src/test/java/com/yahoo/vespa/model/test/VespaModelTestCase.java index 9fe1c8c40d3..bf4d066e454 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/test/VespaModelTestCase.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/test/VespaModelTestCase.java @@ -39,18 +39,15 @@ import java.io.File; import java.io.IOException; import java.io.StringReader; import java.util.ArrayList; -import java.util.Arrays; import java.util.List; import java.util.Map; import java.util.Optional; import java.util.Set; import java.util.logging.Level; -import static org.hamcrest.CoreMatchers.is; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertNotNull; -import static org.junit.Assert.assertThat; import static org.junit.Assert.assertTrue; /** @@ -258,13 +255,13 @@ public class VespaModelTestCase { "</services>") .create(); Admin admin = model.getAdmin(); - assertThat(admin.getSlobroks().size(), is(1)); - assertThat(admin.getConfigservers().size(), is(1)); + assertEquals(1, admin.getSlobroks().size()); + assertEquals(1, admin.getConfigservers().size()); Set<HostInfo> hosts = model.getHosts(); - assertThat(hosts.size(), is(1)); + assertEquals(1, hosts.size()); //logd, config proxy, sentinel, config server, slobrok, log server HostInfo host = hosts.iterator().next(); - assertThat(host.getServices().size(), is(7)); + assertEquals(7, host.getServices().size()); new LogdConfig((LogdConfig.Builder) model.getConfig(new LogdConfig.Builder(), "admin/model")); } @@ -278,7 +275,7 @@ public class VespaModelTestCase { .applicationPackage(applicationPackage) .modelHostProvisioner(new InMemoryProvisioner(true, false, "host1.yahoo.com")) .properties(new TestProperties() - .setConfigServerSpecs(Arrays.asList(new TestProperties.Spec("cfghost", 1234, 1236))) + .setConfigServerSpecs(List.of(new TestProperties.Spec("cfghost", 1234, 1236))) .setMultitenant(true)) .build(); VespaModel model = new VespaModel(new NullConfigModelRegistry(), deployState); @@ -291,8 +288,8 @@ public class VespaModelTestCase { VespaModel model = new VespaModel(new MockApplicationPackage.Builder() .withServices("<services version='1.0'><container version='1.0'><search /></container></services>") .build()); - assertThat(model.hostSystem().getHosts().size(), is(1)); - assertThat(model.getContainerClusters().size(), is(1)); + assertEquals(1, model.hostSystem().getHosts().size()); + assertEquals(1, model.getContainerClusters().size()); } @Test @@ -300,9 +297,9 @@ public class VespaModelTestCase { ApplicationPackage app = MockApplicationPackage.createEmpty(); DeployState.Builder builder = new DeployState.Builder().applicationPackage(app); VespaModel model = new VespaModel(new NullConfigModelRegistry(), builder.build()); - assertThat(model.getContainerClusters().size(), is(0)); + assertTrue(model.getContainerClusters().isEmpty()); model = new VespaModel(new NullConfigModelRegistry(), builder.permanentApplicationPackage(Optional.of(FilesApplicationPackage.fromFile(new File(TESTDIR, "app_permanent")))).build()); - assertThat(model.getContainerClusters().size(), is(1)); + assertEquals(1, model.getContainerClusters().size()); } @Test diff --git a/config-model/src/test/java/com/yahoo/vespa/model/utils/FileSenderTest.java b/config-model/src/test/java/com/yahoo/vespa/model/utils/FileSenderTest.java index e9527bae7a2..96815646a88 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/utils/FileSenderTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/utils/FileSenderTest.java @@ -23,8 +23,7 @@ import java.util.HashMap; import java.util.List; import java.util.Map; -import static org.hamcrest.core.Is.is; -import static org.junit.Assert.assertThat; +import static org.junit.Assert.assertEquals; /** * @author Ulf Lilleengen @@ -89,8 +88,8 @@ public class FileSenderTest { builder.setField("stringVal", "foo.txt"); fileRegistry.pathToRef.put("foo.txt", new FileNode("fooshash").value()); fileSender().sendUserConfiguredFiles(producer); - assertThat(builder.getObject("fileVal").getValue(), is("fooshash")); - assertThat(builder.getObject("stringVal").getValue(), is("foo.txt")); + assertEquals("fooshash", builder.getObject("fileVal").getValue()); + assertEquals("foo.txt", builder.getObject("stringVal").getValue()); } @Test @@ -101,8 +100,8 @@ public class FileSenderTest { builder.setField("stringVal", "foo.txt"); fileRegistry.pathToRef.put("foo.txt", new FileNode("fooshash").value()); fileSender().sendUserConfiguredFiles(producer); - assertThat(builder.getObject("fileVal").getValue(), is("fooshash")); - assertThat(builder.getObject("stringVal").getValue(), is("foo.txt")); + assertEquals("fooshash", builder.getObject("fileVal").getValue()); + assertEquals("foo.txt", builder.getObject("stringVal").getValue()); } @Test @@ -114,8 +113,8 @@ public class FileSenderTest { inner.setField("stringVal", "bar.txt"); fileRegistry.pathToRef.put("bar.txt", new FileNode("barhash").value()); fileSender().sendUserConfiguredFiles(producer); - assertThat(builder.getArray("inner").get(0).getObject("fileVal").getValue(), is("barhash")); - assertThat(builder.getArray("inner").get(0).getObject("stringVal").getValue(), is("bar.txt")); + assertEquals("barhash", builder.getArray("inner").get(0).getObject("fileVal").getValue()); + assertEquals("bar.txt", builder.getArray("inner").get(0).getObject("stringVal").getValue()); } @Test @@ -131,10 +130,10 @@ public class FileSenderTest { fileRegistry.pathToRef.put("bar.txt", new FileNode("barhash").value()); fileRegistry.pathToRef.put("path.txt", new FileNode("pathhash").value()); fileSender().sendUserConfiguredFiles(producer); - assertThat(builder.getArray("fileArray").get(0).getValue(), is("foohash")); - assertThat(builder.getArray("fileArray").get(1).getValue(), is("barhash")); - assertThat(builder.getArray("pathArray").get(0).getValue(), is("pathhash")); - assertThat(builder.getArray("stringArray").get(0).getValue(), is("foo.txt")); + assertEquals("foohash", builder.getArray("fileArray").get(0).getValue()); + assertEquals("barhash", builder.getArray("fileArray").get(1).getValue()); + assertEquals("pathhash", builder.getArray("pathArray").get(0).getValue()); + assertEquals("foo.txt", builder.getArray("stringArray").get(0).getValue()); } @Test @@ -145,8 +144,8 @@ public class FileSenderTest { builder.getObject("struct").setField("stringVal", "foo.txt"); fileRegistry.pathToRef.put("foo.txt", new FileNode("foohash").value()); fileSender().sendUserConfiguredFiles(producer); - assertThat(builder.getObject("struct").getObject("fileVal").getValue(), is("foohash")); - assertThat(builder.getObject("struct").getObject("stringVal").getValue(), is("foo.txt")); + assertEquals("foohash", builder.getObject("struct").getObject("fileVal").getValue()); + assertEquals("foo.txt", builder.getObject("struct").getObject("stringVal").getValue()); } @Test @@ -162,10 +161,10 @@ public class FileSenderTest { fileRegistry.pathToRef.put("bar.txt", new FileNode("barhash").value()); fileRegistry.pathToRef.put("path.txt", new FileNode("pathhash").value()); fileSender().sendUserConfiguredFiles(producer); - assertThat(builder.getMap("fileMap").get("foo").getValue(), is("foohash")); - assertThat(builder.getMap("fileMap").get("bar").getValue(), is("barhash")); - assertThat(builder.getMap("pathMap").get("path").getValue(), is("pathhash")); - assertThat(builder.getMap("stringMap").get("bar").getValue(), is("bar.txt")); + assertEquals("foohash", builder.getMap("fileMap").get("foo").getValue()); + assertEquals("barhash", builder.getMap("fileMap").get("bar").getValue()); + assertEquals("pathhash", builder.getMap("pathMap").get("path").getValue()); + assertEquals("bar.txt", builder.getMap("stringMap").get("bar").getValue()); } @Test @@ -177,8 +176,8 @@ public class FileSenderTest { inner.setField("stringVal", "bar.txt"); fileRegistry.pathToRef.put("bar.txt", new FileNode("barhash").value()); fileSender().sendUserConfiguredFiles(producer); - assertThat(builder.getMap("inner").get("foo").getObject("fileVal").getValue(), is("barhash")); - assertThat(builder.getMap("inner").get("foo").getObject("stringVal").getValue(), is("bar.txt")); + assertEquals("barhash", builder.getMap("inner").get("foo").getObject("fileVal").getValue()); + assertEquals("bar.txt", builder.getMap("inner").get("foo").getObject("stringVal").getValue()); } @Test(expected = IllegalArgumentException.class) diff --git a/config-model/src/test/java/com/yahoo/vespa/model/utils/internal/ReflectionUtilTest.java b/config-model/src/test/java/com/yahoo/vespa/model/utils/internal/ReflectionUtilTest.java index 2909031f380..fb1afb75e10 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/utils/internal/ReflectionUtilTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/utils/internal/ReflectionUtilTest.java @@ -12,10 +12,8 @@ import org.junit.Test; import java.util.Set; import static com.yahoo.vespa.model.utils.internal.ReflectionUtil.getAllConfigsProduced; -import static org.hamcrest.Matchers.is; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertThat; import static org.junit.Assert.assertTrue; /** @@ -71,14 +69,14 @@ public class ReflectionUtilTest { @Test public void getAllConfigsProduced_includes_configs_produced_by_super_class() { Set<ConfigKey<?>> configs = getAllConfigsProduced(ConcreteProducer.class, "foo"); - assertThat(configs.size(), is(1)); + assertEquals(1, configs.size()); assertTrue(configs.contains(new ConfigKey<>(SimpletypesConfig.CONFIG_DEF_NAME, "foo", SimpletypesConfig.CONFIG_DEF_NAMESPACE))); } @Test public void getAllConfigsProduced_includes_configs_produced_by_implemented_interface() { Set<ConfigKey<?>> configs = getAllConfigsProduced(InterfaceImplementingProducer.class, "foo"); - assertThat(configs.size(), is(2)); + assertEquals(2, configs.size()); assertTrue(configs.contains(new ConfigKey<>(SimpletypesConfig.CONFIG_DEF_NAME, "foo", SimpletypesConfig.CONFIG_DEF_NAMESPACE))); assertTrue(configs.contains(new ConfigKey<>(ArraytypesConfig.CONFIG_DEF_NAME, "foo", ArraytypesConfig.CONFIG_DEF_NAMESPACE))); } @@ -86,7 +84,7 @@ public class ReflectionUtilTest { @Test public void getAllConfigsProduced_includes_configs_directly_implemented_by_producer() { Set<ConfigKey<?>> configs = getAllConfigsProduced(SimpleProducer.class, "foo"); - assertThat(configs.size(), is(1)); + assertEquals(1, configs.size()); assertTrue(configs.contains(new ConfigKey<>(SimpletypesConfig.CONFIG_DEF_NAME, "foo", SimpletypesConfig.CONFIG_DEF_NAMESPACE))); } diff --git a/config-provisioning/pom.xml b/config-provisioning/pom.xml index 12b31bd6370..5f0c93ceec5 100644 --- a/config-provisioning/pom.xml +++ b/config-provisioning/pom.xml @@ -64,11 +64,6 @@ Provisioning APIs. <scope>test</scope> </dependency> <dependency> - <groupId>org.hamcrest</groupId> - <artifactId>hamcrest-all</artifactId> - <scope>test</scope> - </dependency> - <dependency> <groupId>junit</groupId> <artifactId>junit</artifactId> <scope>test</scope> diff --git a/config-provisioning/src/test/java/com/yahoo/config/provision/ApplicationIdTest.java b/config-provisioning/src/test/java/com/yahoo/config/provision/ApplicationIdTest.java index 27b9a7213cb..c82230f7edf 100644 --- a/config-provisioning/src/test/java/com/yahoo/config/provision/ApplicationIdTest.java +++ b/config-provisioning/src/test/java/com/yahoo/config/provision/ApplicationIdTest.java @@ -7,8 +7,6 @@ import com.yahoo.cloud.config.ApplicationIdConfig; import com.yahoo.test.TotalOrderTester; import org.junit.Test; import com.google.common.testing.EqualsTester; -import static org.junit.Assert.assertThat; -import static org.hamcrest.Matchers.*; /** * @author Ulf Lilleengen @@ -53,9 +51,9 @@ public class ApplicationIdTest { ApplicationId id1 = applicationId("foo"); ApplicationId id2 = applicationId("bar"); ApplicationId id3 = idFrom("tenant", "baz", "bim"); - assertThat(id1.serializedForm(), is("default:foo:default")); - assertThat(id2.serializedForm(), is("default:bar:default")); - assertThat(id3.serializedForm(), is("tenant:baz:bim")); + assertEquals("default:foo:default", id1.serializedForm()); + assertEquals("default:bar:default", id2.serializedForm()); + assertEquals("tenant:baz:bim", id3.serializedForm()); } @Test @@ -63,20 +61,20 @@ public class ApplicationIdTest { ApplicationId id1 = applicationId("foo"); ApplicationId id2 = idFrom("bar", "baz", "default"); ApplicationId id3 = idFrom("tenant", "baz", "bim"); - assertThat(id1.toShortString(), is("default.foo")); - assertThat(id1.toFullString(), is("default.foo.default")); - assertThat(id2.toShortString(), is("bar.baz")); - assertThat(id2.toFullString(), is("bar.baz.default")); - assertThat(id3.toShortString(), is("tenant.baz.bim")); - assertThat(id3.toFullString(), is("tenant.baz.bim")); + assertEquals("default.foo", id1.toShortString()); + assertEquals("default.foo.default", id1.toFullString()); + assertEquals("bar.baz", id2.toShortString()); + assertEquals("bar.baz.default", id2.toFullString()); + assertEquals("tenant.baz.bim", id3.toShortString()); + assertEquals("tenant.baz.bim", id3.toFullString()); } @Test public void require_that_idstring_can_be_parsed() { ApplicationId id = ApplicationId.fromSerializedForm("ten:foo:bim"); - assertThat(id.tenant().value(), is("ten")); - assertThat(id.application().value(), is("foo")); - assertThat(id.instance().value(), is("bim")); + assertEquals("ten", id.tenant().value()); + assertEquals("foo", id.application().value()); + assertEquals("bim", id.instance().value()); } @Test(expected = IllegalArgumentException.class) @@ -87,8 +85,8 @@ public class ApplicationIdTest { @Test public void require_that_defaults_are_given() { ApplicationId id1 = applicationId("foo"); - assertThat(id1.tenant().value(), is("default")); - assertThat(id1.instance().value(), is("default")); + assertEquals("default", id1.tenant().value()); + assertEquals("default", id1.instance().value()); } @Test diff --git a/config-provisioning/src/test/java/com/yahoo/config/provision/IdentifierTestBase.java b/config-provisioning/src/test/java/com/yahoo/config/provision/IdentifierTestBase.java index 370bcd6cdf1..d4edb7a14eb 100644 --- a/config-provisioning/src/test/java/com/yahoo/config/provision/IdentifierTestBase.java +++ b/config-provisioning/src/test/java/com/yahoo/config/provision/IdentifierTestBase.java @@ -4,10 +4,9 @@ package com.yahoo.config.provision; import com.google.common.testing.EqualsTester; import org.junit.Test; -import static org.hamcrest.CoreMatchers.is; -import static org.hamcrest.CoreMatchers.not; +import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertThat; +import static org.junit.Assert.assertNotEquals; import static org.junit.Assert.assertTrue; /** @@ -29,8 +28,8 @@ public abstract class IdentifierTestBase<ID_TYPE> { assertTrue(isDefault(def)); assertTrue(isDefault(def2)); assertFalse(isDefault(notdef)); - assertThat(def, is(def2)); - assertThat(def2, is(not(notdef))); + assertEquals(def, def2); + assertNotEquals(def2, notdef); } @Test diff --git a/config-provisioning/src/test/java/com/yahoo/config/provision/TenantTest.java b/config-provisioning/src/test/java/com/yahoo/config/provision/TenantTest.java index 8475c30c1f6..2bfcd183c85 100644 --- a/config-provisioning/src/test/java/com/yahoo/config/provision/TenantTest.java +++ b/config-provisioning/src/test/java/com/yahoo/config/provision/TenantTest.java @@ -4,8 +4,7 @@ package com.yahoo.config.provision; import com.yahoo.test.TotalOrderTester; import org.junit.Test; -import static org.hamcrest.CoreMatchers.is; -import static org.junit.Assert.assertThat; +import static org.junit.Assert.assertEquals; /** * @author Ulf Lilleengen @@ -29,7 +28,7 @@ public class TenantTest extends IdentifierTestBase<TenantName> { @Test public void testComparator() { - assertThat(TenantName.defaultName().compareTo(TenantName.defaultName()), is(0)); + assertEquals(0, TenantName.defaultName().compareTo(TenantName.defaultName())); new TotalOrderTester<TenantName>() .theseObjects(TenantName.from("a"), TenantName.from("a")) diff --git a/config/src/main/java/com/yahoo/config/subscription/impl/JRTConfigRequester.java b/config/src/main/java/com/yahoo/config/subscription/impl/JRTConfigRequester.java index fdfaf8b72fd..604c85555db 100644 --- a/config/src/main/java/com/yahoo/config/subscription/impl/JRTConfigRequester.java +++ b/config/src/main/java/com/yahoo/config/subscription/impl/JRTConfigRequester.java @@ -19,12 +19,10 @@ import java.time.Duration; import java.time.Instant; import java.util.concurrent.ScheduledThreadPoolExecutor; import java.util.concurrent.TimeUnit; -import java.util.logging.Level; import java.util.logging.Logger; import static java.util.logging.Level.FINE; import static java.util.logging.Level.FINEST; -import static java.util.logging.Level.INFO; import static java.util.logging.Level.SEVERE; import static java.util.logging.Level.WARNING; @@ -69,8 +67,8 @@ public class JRTConfigRequester implements RequestWaiter { this.scheduler = scheduler; this.connectionPool = connectionPool; this.timingValues = timingValues; - // Adjust so that we wait 1 second with logging warning in case there are some errors just when starting up - timeForLastLogWarning = Instant.now().minus(delayBetweenWarnings.plus(Duration.ofSeconds(5))); + // Adjust so that we wait 5 seconds with logging warning in case there are some errors just when starting up + timeForLastLogWarning = Instant.now().minus(delayBetweenWarnings).plus(Duration.ofSeconds(5)); } /** @@ -170,15 +168,11 @@ public class JRTConfigRequester implements RequestWaiter { private void handleFailedRequest(JRTClientConfigRequest jrtReq, JRTConfigSubscription<ConfigInstance> sub, Connection connection) { logError(jrtReq, connection); - log.log(INFO, "Failure of config subscription to " + connection.getAddress() + - ", clients will keep existing config until resolved: " + sub); connectionPool.switchConnection(connection); if (failures < 10) failures++; long delay = calculateFailedRequestDelay(failures, timingValues); - // The logging depends on whether we are configured or not. - Level logLevel = sub.getConfigState().getConfig() == null ? Level.FINE : Level.INFO; - log.log(logLevel, () -> "Request for config " + jrtReq.getShortDescription() + "' failed with error code " + + log.log(FINE, () -> "Request for config " + jrtReq.getShortDescription() + "' failed with error code " + jrtReq.errorCode() + " (" + jrtReq.errorMessage() + "), scheduling new request " + " in " + delay + " ms"); scheduleNextRequest(jrtReq, sub, delay, calculateErrorTimeout()); diff --git a/config/src/test/java/com/yahoo/config/subscription/ConfigGetterTest.java b/config/src/test/java/com/yahoo/config/subscription/ConfigGetterTest.java index ffbc1896aab..68cadb33ecd 100644 --- a/config/src/test/java/com/yahoo/config/subscription/ConfigGetterTest.java +++ b/config/src/test/java/com/yahoo/config/subscription/ConfigGetterTest.java @@ -7,8 +7,7 @@ import org.junit.Test; import java.io.File; -import static org.hamcrest.CoreMatchers.is; -import static org.junit.Assert.assertThat; +import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; /** @@ -28,15 +27,15 @@ public class ConfigGetterTest { ConfigGetter<AppConfig> getter = new ConfigGetter<>(AppConfig.class); AppConfig config = getter.getConfig(configId); - assertThat(config.times(), is(times)); - assertThat(config.message(), is(message)); - assertThat(config.a().size(), is(1)); - assertThat(config.a(0).name(), is(a0)); + assertEquals(times, config.times()); + assertEquals(message, config.message()); + assertEquals(1, config.a().size()); + assertEquals(a0, config.a(0).name()); AppService service = new AppService(configId, sourceSet); AppConfig serviceConfig = service.getConfig(); assertTrue(service.isConfigured()); - assertThat(config, is(serviceConfig)); + assertEquals(config, serviceConfig); service.cancelSubscription(); } @@ -45,16 +44,16 @@ public class ConfigGetterTest { public void testGetFromRawSource() { ConfigGetter<AppConfig> getter = new ConfigGetter<>(new RawSource("message \"one\""), AppConfig.class); AppConfig config = getter.getConfig("test"); - assertThat(config.message(), is("one")); + assertEquals("one", config.message()); } @Test public void testGetTwice() { ConfigGetter<AppConfig> getter = new ConfigGetter<>(AppConfig.class); AppConfig config = getter.getConfig("raw:message \"one\""); - assertThat(config.message(), is("one")); + assertEquals("one", config.message()); config = getter.getConfig("raw:message \"two\""); - assertThat(config.message(), is("two")); + assertEquals("two", config.message()); } @Test @@ -85,11 +84,11 @@ public class ConfigGetterTest { } private void verifyFooValues(AppConfig config) { - assertThat(config.message(), is("msg1")); - assertThat(config.times(), is(3)); - assertThat(config.a(0).name(), is("a0")); - assertThat(config.a(1).name(), is("a1")); - assertThat(config.a(2).name(), is("a2")); + assertEquals("msg1", config.message()); + assertEquals(3, config.times()); + assertEquals("a0", config.a(0).name()); + assertEquals("a1", config.a(1).name()); + assertEquals("a2", config.a(2).name()); } @Test @@ -100,15 +99,15 @@ public class ConfigGetterTest { String configId = "raw:times " + times + "\nmessage " + message + "\na[1]\na[0].name " + a0; AppConfig config = ConfigGetter.getConfig(AppConfig.class, configId); - assertThat(config.times(), is(times)); - assertThat(config.message(), is(message)); - assertThat(config.a().size(), is(1)); - assertThat(config.a(0).name(), is(a0)); + assertEquals(times, config.times()); + assertEquals(message, config.message()); + assertEquals(1, config.a().size()); + assertEquals(a0, config.a(0).name()); AppService service = new AppService(configId, sourceSet); AppConfig serviceConfig = service.getConfig(); assertTrue(service.isConfigured()); - assertThat(config, is(serviceConfig)); + assertEquals(config, serviceConfig); service.cancelSubscription(); } diff --git a/config/src/test/java/com/yahoo/config/subscription/ConfigInstancePayloadTest.java b/config/src/test/java/com/yahoo/config/subscription/ConfigInstancePayloadTest.java index 3dc0f7188e3..c656bfe1a60 100644 --- a/config/src/test/java/com/yahoo/config/subscription/ConfigInstancePayloadTest.java +++ b/config/src/test/java/com/yahoo/config/subscription/ConfigInstancePayloadTest.java @@ -15,9 +15,8 @@ import java.util.Arrays; import java.util.List; import static com.yahoo.foo.FunctionTestConfig.*; -import static org.hamcrest.CoreMatchers.is; import static org.junit.Assert.assertNotNull; -import static org.junit.Assert.assertThat; +import static org.junit.Assert.assertEquals; import static org.junit.Assert.fail; /** @@ -131,8 +130,8 @@ public class ConfigInstancePayloadTest { try { System.out.println(payload.toString(false)); FunctionTestConfig config2 = new FunctionTestConfig((FunctionTestConfig.Builder)new ConfigTransformer<>(FunctionTestConfig.class).toConfigBuilder(payload)); - assertThat(config2, is(expected)); - assertThat(ConfigInstance.serialize(config2), is(ConfigInstance.serialize(expected))); + assertEquals(expected, config2); + assertEquals(ConfigInstance.serialize(expected), ConfigInstance.serialize(config2)); } catch (Exception e) { e.printStackTrace(); fail(); @@ -156,14 +155,14 @@ public class ConfigInstancePayloadTest { System.out.println(payload.toString()); MaptypesConfig config = ConfigInstanceUtil.getNewInstance(MaptypesConfig.class, "foo", payload); System.out.println(config); - assertThat(config.intmap().size(), is(1)); - assertThat(config.intmap("foo"), is(1337)); + assertEquals(1, config.intmap().size()); + assertEquals(1337, config.intmap("foo")); assertNotNull(config.innermap("bar")); - assertThat(config.innermap("bar").foo(), is(93)); - assertThat(config.nestedmap().size(), is(1)); + assertEquals(93, config.innermap("bar").foo()); + assertEquals(1, config.nestedmap().size()); assertNotNull(config.nestedmap("baz")); - assertThat(config.nestedmap("baz").inner("foo"), is(1)); - assertThat(config.nestedmap("baz").inner("bar"), is(2)); + assertEquals(1, config.nestedmap("baz").inner("foo")); + assertEquals(2, config.nestedmap("baz").inner("bar")); } private MaptypesConfig createMapTypesConfig() { diff --git a/config/src/test/java/com/yahoo/config/subscription/ConfigInstanceSerializationTest.java b/config/src/test/java/com/yahoo/config/subscription/ConfigInstanceSerializationTest.java index 57ceee9b1bb..831e645c763 100644 --- a/config/src/test/java/com/yahoo/config/subscription/ConfigInstanceSerializationTest.java +++ b/config/src/test/java/com/yahoo/config/subscription/ConfigInstanceSerializationTest.java @@ -8,8 +8,7 @@ import org.junit.Test; import java.util.List; -import static org.hamcrest.CoreMatchers.is; -import static org.junit.Assert.assertThat; +import static org.junit.Assert.assertEquals; /** * @author gjoranv @@ -27,8 +26,8 @@ public class ConfigInstanceSerializationTest { ConfigPayload payload = new CfgConfigPayloadBuilder().deserialize(lines); FunctionTestConfig config2 = ConfigInstanceUtil.getNewInstance(FunctionTestConfig.class, ":parent:", payload); - assertThat(config, is(config2)); - assertThat(ConfigInstance.serialize(config), is(ConfigInstance.serialize(config2))); + assertEquals(config, config2); + assertEquals(ConfigInstance.serialize(config), ConfigInstance.serialize(config2)); } } diff --git a/config/src/test/java/com/yahoo/config/subscription/ConfigInstanceUtilTest.java b/config/src/test/java/com/yahoo/config/subscription/ConfigInstanceUtilTest.java index aaf6782a6ff..091494955fb 100644 --- a/config/src/test/java/com/yahoo/config/subscription/ConfigInstanceUtilTest.java +++ b/config/src/test/java/com/yahoo/config/subscription/ConfigInstanceUtilTest.java @@ -13,11 +13,9 @@ import org.junit.Test; import java.io.File; import java.util.Arrays; -import static org.hamcrest.CoreMatchers.is; - import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertThat; import static com.yahoo.foo.FunctionTestConfig.*; +import static org.junit.Assert.assertTrue; /** * @author Ulf Lilleengen @@ -43,26 +41,26 @@ public class ConfigInstanceUtilTest { ConfigInstanceUtil.setValues(destination, source); FunctionTestConfig result = new FunctionTestConfig(destination); - assertThat(result.int_val(), is(-1)); - assertThat(result.string_val(), is("foo")); - assertThat(result.intarr().size(), is(1)); - assertThat(result.intarr(0), is(0)); - assertThat(result.longarr().size(), is(2)); - assertThat(result.doublearr().size(), is(3)); + assertEquals(-1, result.int_val()); + assertEquals("foo", result.string_val()); + assertEquals(1, result.intarr().size()); + assertEquals(0, result.intarr(0)); + assertEquals(2, result.longarr().size()); + assertEquals(3, result.doublearr().size()); assertEquals(2344.0, result.doublearr(0), 0.01); assertEquals(123.0, result.doublearr(1), 0.01); assertEquals(0.0, result.doublearr(2), 0.01); - assertThat(result.basicStruct().bar(), is(-1)); - assertThat(result.basicStruct().foo(), is("basicFoo")); - assertThat(result.basicStruct().intArr().size(), is(3)); - assertThat(result.basicStruct().intArr(0), is(310)); - assertThat(result.basicStruct().intArr(1), is(311)); - assertThat(result.basicStruct().intArr(2), is(0)); - assertThat(result.myarray().size(), is(3)); - assertThat(result.myarray(2).intval(), is(-1)); - assertThat(result.myarray(2).refval(), is("")); - assertThat(result.myarray(2).fileVal().value(), is("")); - assertThat(result.myarray(2).myStruct().a(), is(0)); + assertEquals(-1, result.basicStruct().bar()); + assertEquals("basicFoo", result.basicStruct().foo()); + assertEquals(3, result.basicStruct().intArr().size()); + assertEquals(310, result.basicStruct().intArr(0)); + assertEquals(311, result.basicStruct().intArr(1)); + assertEquals(0, result.basicStruct().intArr(2)); + assertEquals(3, result.myarray().size()); + assertEquals(-1, result.myarray(2).intval()); + assertTrue(result.myarray(2).refval().isEmpty()); + assertTrue(result.myarray(2).fileVal().value().isEmpty()); + assertEquals(0, result.myarray(2).myStruct().a()); } diff --git a/config/src/test/java/com/yahoo/config/subscription/ConfigInterruptedExceptionTest.java b/config/src/test/java/com/yahoo/config/subscription/ConfigInterruptedExceptionTest.java index dbc47b7e24d..3f050c449c7 100644 --- a/config/src/test/java/com/yahoo/config/subscription/ConfigInterruptedExceptionTest.java +++ b/config/src/test/java/com/yahoo/config/subscription/ConfigInterruptedExceptionTest.java @@ -3,8 +3,7 @@ package com.yahoo.config.subscription; import org.junit.Test; -import static org.hamcrest.CoreMatchers.is; -import static org.junit.Assert.assertThat; +import static org.junit.Assert.assertEquals; /** * @author Ulf Lilleengen @@ -14,6 +13,6 @@ public class ConfigInterruptedExceptionTest { @Test public void require_that_throwable_is_preserved() { ConfigInterruptedException e = new ConfigInterruptedException(new RuntimeException("foo")); - assertThat(e.getCause().getMessage(), is("foo")); + assertEquals("foo", e.getCause().getMessage()); } } diff --git a/config/src/test/java/com/yahoo/config/subscription/ConfigSourceSetTest.java b/config/src/test/java/com/yahoo/config/subscription/ConfigSourceSetTest.java index 452b51bf408..2e46623ef9a 100755 --- a/config/src/test/java/com/yahoo/config/subscription/ConfigSourceSetTest.java +++ b/config/src/test/java/com/yahoo/config/subscription/ConfigSourceSetTest.java @@ -6,8 +6,9 @@ import org.junit.After; import java.util.Set; -import static org.hamcrest.CoreMatchers.is; -import static org.junit.Assert.*; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotEquals; +import static org.junit.Assert.assertTrue; /** * @author <a href="gv@yahoo-inc.com">G. Voldengen</a> @@ -54,7 +55,7 @@ public class ConfigSourceSetTest { // TODO: Unable to set environment, so only able to test property usage for now System.setProperty("configsources", "foo:123,bar:345,tcp/baz:333,quux"); ConfigSourceSet set = ConfigSourceSet.createDefault(); - assertThat(set.getSources().size(), is(4)); + assertEquals(4, set.getSources().size()); assertTrue(set.getSources().contains("tcp/foo:123")); assertTrue(set.getSources().contains("tcp/bar:345")); assertTrue(set.getSources().contains("tcp/baz:333")); diff --git a/config/src/test/java/com/yahoo/config/subscription/ConfigURITest.java b/config/src/test/java/com/yahoo/config/subscription/ConfigURITest.java index 3c2750c55cb..5c6c09b8b18 100644 --- a/config/src/test/java/com/yahoo/config/subscription/ConfigURITest.java +++ b/config/src/test/java/com/yahoo/config/subscription/ConfigURITest.java @@ -6,8 +6,7 @@ import org.junit.Test; import java.io.File; import java.io.IOException; -import static org.hamcrest.core.Is.is; -import static org.junit.Assert.assertThat; +import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; /** @@ -18,7 +17,7 @@ public class ConfigURITest { @Test public void testDefaultUri() { ConfigURI uri = ConfigURI.createFromId("foo"); - assertThat(uri.getConfigId(), is("foo")); + assertEquals("foo", uri.getConfigId()); assertTrue(uri.getSource() instanceof ConfigSourceSet); } @@ -26,21 +25,21 @@ public class ConfigURITest { public void testFileUri() throws IOException { File file = File.createTempFile("foo", ".cfg"); ConfigURI uri = ConfigURI.createFromId("file:" + file.getAbsolutePath()); - assertThat(uri.getConfigId(), is("")); + assertTrue(uri.getConfigId().isEmpty()); assertTrue(uri.getSource() instanceof FileSource); } @Test public void testDirUri() throws IOException { ConfigURI uri = ConfigURI.createFromId("dir:."); - assertThat(uri.getConfigId(), is("")); + assertTrue(uri.getConfigId().isEmpty()); assertTrue(uri.getSource() instanceof DirSource); } @Test public void testCustomUri() { ConfigURI uri = ConfigURI.createFromIdAndSource("foo", new ConfigSet()); - assertThat(uri.getConfigId(), is("foo")); + assertEquals("foo", uri.getConfigId()); assertTrue(uri.getSource() instanceof ConfigSet); } } diff --git a/config/src/test/java/com/yahoo/config/subscription/FunctionTest.java b/config/src/test/java/com/yahoo/config/subscription/FunctionTest.java index 97a682d3b9a..42c2c599899 100644 --- a/config/src/test/java/com/yahoo/config/subscription/FunctionTest.java +++ b/config/src/test/java/com/yahoo/config/subscription/FunctionTest.java @@ -214,7 +214,7 @@ public class FunctionTest { assertEquals(0, config.intarr().size()); assertEquals(2, config.longarr().size()); assertEquals(Long.MAX_VALUE, config.longarr(0)); - assertThat(config.longarr().get(1), is(Long.MIN_VALUE)); + assertEquals(Long.MIN_VALUE, config.longarr().get(1).longValue()); assertEquals(2, config.doublearr().size()); assertEquals(1, config.stringarr().size()); assertEquals(1, config.enumarr().size()); @@ -231,8 +231,8 @@ public class FunctionTest { assertEquals("basicFoo", config.basicStruct().foo()); assertEquals(3, config.basicStruct().bar()); // new List api assertEquals(2, config.basicStruct().intArr().size()); - assertThat(config.basicStruct().intArr().get(0), is(310)); // new List api - assertThat(config.basicStruct().intArr().get(1), is(311)); // new List api + assertEquals(310, config.basicStruct().intArr().get(0).intValue()); // new List api + assertEquals(311, config.basicStruct().intArr().get(1).intValue()); // new List api assertEquals(310, config.basicStruct().intArr(0)); // short-hand assertEquals("inner0", config.rootStruct().inner0().name()); // new List api assertEquals(11, config.rootStruct().inner0().index()); diff --git a/config/src/test/java/com/yahoo/config/subscription/NamespaceTest.java b/config/src/test/java/com/yahoo/config/subscription/NamespaceTest.java index 4edf931a7c6..963f3de5e43 100644 --- a/config/src/test/java/com/yahoo/config/subscription/NamespaceTest.java +++ b/config/src/test/java/com/yahoo/config/subscription/NamespaceTest.java @@ -4,8 +4,7 @@ package com.yahoo.config.subscription; import com.yahoo.myproject.config.NamespaceConfig; import org.junit.Test; -import static org.hamcrest.CoreMatchers.is; -import static org.junit.Assert.assertThat; +import static org.junit.Assert.assertEquals; /** * @author gjoranv @@ -15,6 +14,6 @@ public class NamespaceTest { @Test public void verifyConfigClassWithExplicitNamespace() { NamespaceConfig config = new ConfigGetter<>(NamespaceConfig.class).getConfig("raw: a 0\n"); - assertThat(config.a(), is(0)); + assertEquals(0, config.a()); } } diff --git a/config/src/test/java/com/yahoo/config/subscription/impl/FileConfigSubscriptionTest.java b/config/src/test/java/com/yahoo/config/subscription/impl/FileConfigSubscriptionTest.java index 74af35e39dc..e37a5d7b36b 100644 --- a/config/src/test/java/com/yahoo/config/subscription/impl/FileConfigSubscriptionTest.java +++ b/config/src/test/java/com/yahoo/config/subscription/impl/FileConfigSubscriptionTest.java @@ -14,11 +14,10 @@ import java.io.FileWriter; import java.io.IOException; import java.nio.file.Files; -import static org.hamcrest.CoreMatchers.is; +import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertNotSame; import static org.junit.Assert.assertSame; -import static org.junit.Assert.assertThat; import static org.junit.Assert.assertTrue; /** @@ -45,11 +44,11 @@ public class FileConfigSubscriptionTest { new ConfigKey<>(SimpletypesConfig.class, ""), TEST_TYPES_FILE); assertTrue(sub.nextConfig(1000)); - assertThat(sub.getConfigState().getConfig().intval(), is(23)); + assertEquals(23, sub.getConfigState().getConfig().intval()); Thread.sleep(1000); writeConfig("intval", "33"); assertTrue(sub.nextConfig(1000)); - assertThat(sub.getConfigState().getConfig().intval(), is(33)); + assertEquals(33, sub.getConfigState().getConfig().intval()); } @Test @@ -59,12 +58,12 @@ public class FileConfigSubscriptionTest { new ConfigKey<>(SimpletypesConfig.class, ""), TEST_TYPES_FILE); assertTrue(sub.nextConfig(1000)); - assertThat(sub.getConfigState().getConfig().intval(), is(23)); + assertEquals(23, sub.getConfigState().getConfig().intval()); writeConfig("intval", "33"); sub.reload(1); assertTrue(sub.nextConfig(1000)); ConfigSubscription.ConfigState<SimpletypesConfig> configState = sub.getConfigState(); - assertThat(configState.getConfig().intval(), is(33)); + assertEquals(33, configState.getConfig().intval()); assertTrue(configState.isConfigChanged()); assertTrue(configState.isGenerationChanged()); @@ -81,7 +80,7 @@ public class FileConfigSubscriptionTest { sub.reload(2); assertTrue(sub.nextConfig(1000)); configState = sub.getConfigState(); - assertThat(configState.getConfig().intval(), is(33)); + assertEquals(33, configState.getConfig().intval()); assertFalse(configState.isConfigChanged()); assertTrue(configState.isGenerationChanged()); @@ -102,7 +101,7 @@ public class FileConfigSubscriptionTest { new DirSource(new File(cfgDir)), new TimingValues()); assertTrue(sub.nextConfig(1000)); - assertThat(sub.getConfigState().getConfig().configId(), is(cfgId)); + assertEquals(cfgId, sub.getConfigState().getConfig().configId()); } @Test(expected = IllegalArgumentException.class) diff --git a/config/src/test/java/com/yahoo/vespa/config/ConfigBuilderMergeTest.java b/config/src/test/java/com/yahoo/vespa/config/ConfigBuilderMergeTest.java index 4800a05b114..93139d2c92e 100644 --- a/config/src/test/java/com/yahoo/vespa/config/ConfigBuilderMergeTest.java +++ b/config/src/test/java/com/yahoo/vespa/config/ConfigBuilderMergeTest.java @@ -12,7 +12,7 @@ import java.util.Arrays; import static com.yahoo.foo.MaptypesConfig.Innermap; import static org.hamcrest.CoreMatchers.is; -import static org.junit.Assert.assertThat; +import static org.hamcrest.MatcherAssert.assertThat; /** * SEO keywords: test override() on builders. overrideTest, testOverride diff --git a/config/src/test/java/com/yahoo/vespa/config/ConfigCacheKeyTest.java b/config/src/test/java/com/yahoo/vespa/config/ConfigCacheKeyTest.java index 6788a2f9c36..7d6174c2fbc 100755 --- a/config/src/test/java/com/yahoo/vespa/config/ConfigCacheKeyTest.java +++ b/config/src/test/java/com/yahoo/vespa/config/ConfigCacheKeyTest.java @@ -3,9 +3,8 @@ package com.yahoo.vespa.config; import org.junit.Test; -import static org.hamcrest.CoreMatchers.is; -import static org.hamcrest.CoreMatchers.not; -import static org.junit.Assert.*; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotEquals; /** * @@ -30,10 +29,10 @@ public class ConfigCacheKeyTest { assertEquals(k1, k2); assertNotEquals(k3, k2); assertNotEquals(k4, k1); - assertThat(k1.hashCode(), is(k2.hashCode())); - assertThat(k1.getDefMd5(), is(defMd5)); - assertThat(k1.toString(), is(configKey.toString() + "," + defMd5)); - assertThat(k5.hashCode(), is(not(k1.hashCode()))); + assertEquals(k2.hashCode(), k1.hashCode()); + assertEquals(defMd5, k1.getDefMd5()); + assertEquals(configKey + "," + defMd5, k1.toString()); + assertNotEquals(k1.hashCode(), k5.hashCode()); } } diff --git a/config/src/test/java/com/yahoo/vespa/config/ConfigDefinitionBuilderTest.java b/config/src/test/java/com/yahoo/vespa/config/ConfigDefinitionBuilderTest.java index 68b25266de8..a9f09951d7e 100644 --- a/config/src/test/java/com/yahoo/vespa/config/ConfigDefinitionBuilderTest.java +++ b/config/src/test/java/com/yahoo/vespa/config/ConfigDefinitionBuilderTest.java @@ -10,7 +10,11 @@ import java.io.FileReader; import java.io.IOException; import static org.hamcrest.CoreMatchers.is; -import static org.junit.Assert.*; +import static org.hamcrest.MatcherAssert.assertThat; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertTrue; /** * Unit tests for ConfigDefinitionBuilder. diff --git a/config/src/test/java/com/yahoo/vespa/config/ConfigDefinitionTest.java b/config/src/test/java/com/yahoo/vespa/config/ConfigDefinitionTest.java index db12145776b..fa85f582e99 100755 --- a/config/src/test/java/com/yahoo/vespa/config/ConfigDefinitionTest.java +++ b/config/src/test/java/com/yahoo/vespa/config/ConfigDefinitionTest.java @@ -11,8 +11,8 @@ import java.util.Collections; import java.util.Comparator; import java.util.List; -import static org.hamcrest.CoreMatchers.is; -import static org.junit.Assert.*; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNull; /** * Unit tests for ConfigDefinition. @@ -56,14 +56,14 @@ public class ConfigDefinitionTest { def.addIntDef("xyzzy", 2, 0, null); assertNull(def.getIntDefs().get("foo").getDefVal()); - assertThat(def.getIntDefs().get("foo").getMin(), is(ConfigDefinition.INT_MIN)); - assertThat(def.getIntDefs().get("foo").getMax(), is(ConfigDefinition.INT_MAX)); - assertThat(def.getIntDefs().get("bar").getDefVal(), is(0)); - assertThat(def.getIntDefs().get("baz").getDefVal(), is(1)); - - assertThat(def.getIntDefs().get("xyzzy").getDefVal(), is(2)); - assertThat(def.getIntDefs().get("xyzzy").getMin(), is(0)); - assertThat(def.getIntDefs().get("xyzzy").getMax(), is(ConfigDefinition.INT_MAX)); + assertEquals(ConfigDefinition.INT_MIN, def.getIntDefs().get("foo").getMin()); + assertEquals(ConfigDefinition.INT_MAX, def.getIntDefs().get("foo").getMax()); + assertEquals(0, def.getIntDefs().get("bar").getDefVal().longValue()); + assertEquals(1, def.getIntDefs().get("baz").getDefVal().longValue()); + + assertEquals(2, def.getIntDefs().get("xyzzy").getDefVal().longValue()); + assertEquals(0, def.getIntDefs().get("xyzzy").getMin().longValue()); + assertEquals(ConfigDefinition.INT_MAX, def.getIntDefs().get("xyzzy").getMax()); } @Test @@ -75,17 +75,16 @@ public class ConfigDefinitionTest { def.addLongDef("xyzzy", 2L, 0L, null); assertNull(def.getLongDefs().get("foo").getDefVal()); - assertThat(def.getLongDefs().get("foo").getMin(), is(ConfigDefinition.LONG_MIN)); - assertThat(def.getLongDefs().get("foo").getMax(), is(ConfigDefinition.LONG_MAX)); - assertThat(def.getLongDefs().get("bar").getDefVal(), is(1234567890123L)); + assertEquals(ConfigDefinition.LONG_MIN, def.getLongDefs().get("foo").getMin()); + assertEquals(ConfigDefinition.LONG_MAX, def.getLongDefs().get("foo").getMax()); + assertEquals(1234567890123L, def.getLongDefs().get("bar").getDefVal().longValue()); - assertThat(def.getLongDefs().get("xyzzy").getDefVal(), is(2L)); - assertThat(def.getLongDefs().get("xyzzy").getMin(), is(0L)); - assertThat(def.getLongDefs().get("xyzzy").getMax(), is(ConfigDefinition.LONG_MAX)); + assertEquals(2L, def.getLongDefs().get("xyzzy").getDefVal().longValue()); + assertEquals(0L, def.getLongDefs().get("xyzzy").getMin().longValue()); + assertEquals(ConfigDefinition.LONG_MAX, def.getLongDefs().get("xyzzy").getMax()); } @Test - @SuppressWarnings("serial") public void testDefaultsPayloadMap() { ConfigDefinition def = new ConfigDefinition("foo", "namespace1"); def.addStringDef("mystring"); @@ -122,7 +121,7 @@ public class ConfigDefinitionTest { def.addIntDef("intval"); def.addLongDef("longval"); def.addDoubleDef("doubleval"); - def.addEnumDef("enumval", new EnumDef(Arrays.asList("FOO"), "FOO")); + def.addEnumDef("enumval", new EnumDef(List.of("FOO"), "FOO")); def.addReferenceDef("refval"); def.addFileDef("fileval"); def.addInnerArrayDef("innerarr"); diff --git a/config/src/test/java/com/yahoo/vespa/config/ConfigFileFormatterTest.java b/config/src/test/java/com/yahoo/vespa/config/ConfigFileFormatterTest.java index c86d54d7471..ab07b669bd0 100644 --- a/config/src/test/java/com/yahoo/vespa/config/ConfigFileFormatterTest.java +++ b/config/src/test/java/com/yahoo/vespa/config/ConfigFileFormatterTest.java @@ -19,15 +19,15 @@ import java.io.IOException; import java.io.StringReader; import java.nio.charset.StandardCharsets; -import static org.hamcrest.core.Is.is; -import static org.junit.Assert.assertThat; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; /** * @author Ulf Lilleengen */ public class ConfigFileFormatterTest { - private String expected_simpletypes = "stringval \"foo\"\n" + + private final String expected_simpletypes = "stringval \"foo\"\n" + "intval 324234\n" + "longval 324\n" + "doubleval 3.455\n" + @@ -66,7 +66,7 @@ public class ConfigFileFormatterTest { ByteArrayOutputStream baos = new ByteArrayOutputStream(); InnerCNode def = new DefParser("simpletypes", new StringReader(StringUtilities.implode(SimpletypesConfig.CONFIG_DEF_SCHEMA, "\n"))).getTree(); new ConfigFileFormat(def).encode(baos, slime); - assertThat(baos.toString(), is(expected_simpletypes)); + assertEquals(expected_simpletypes, baos.toString()); } @Test @@ -77,7 +77,7 @@ public class ConfigFileFormatterTest { InnerCNode def = new DefParser("simpletypes", new StringReader(StringUtilities.implode(SimpletypesConfig.CONFIG_DEF_SCHEMA, "\n"))).getTree(); ByteArrayOutputStream baos = new ByteArrayOutputStream(); new ConfigFileFormat(def).encode(baos, slime); - assertThat(baos.toString(), is("")); + assertTrue(baos.toString().isEmpty()); } // TODO: Reenable this when we can reenable typechecking. @@ -121,7 +121,7 @@ public class ConfigFileFormatterTest { InnerCNode def = new DefParser("simpletypes", new StringReader(StringUtilities.implode(SimpletypesConfig.CONFIG_DEF_SCHEMA, "\n"))).getTree(); ByteArrayOutputStream baos = new ByteArrayOutputStream(); new ConfigFileFormat(def).encode(baos, slime); - assertThat(baos.toString(), is("boolval false\n")); + assertEquals("boolval false\n", baos.toString()); } // TODO: Remove this when we can reenable typechecking. @@ -137,7 +137,8 @@ public class ConfigFileFormatterTest { InnerCNode def = new DefParser("simpletypes", new StringReader(StringUtilities.implode(SimpletypesConfig.CONFIG_DEF_SCHEMA, "\n"))).getTree(); ByteArrayOutputStream baos = new ByteArrayOutputStream(); new ConfigFileFormat(def).encode(baos, slime); - assertThat(baos.toString(StandardCharsets.UTF_8), is("enumval null\nintval null\nlongval null\nboolval false\ndoubleval null\n")); + assertEquals("enumval null\nintval null\nlongval null\nboolval false\ndoubleval null\n", + baos.toString(StandardCharsets.UTF_8)); } // TODO: Reenable this when we can reenable typechecking. @@ -160,7 +161,7 @@ public class ConfigFileFormatterTest { ByteArrayOutputStream baos = new ByteArrayOutputStream(); InnerCNode def = new DefParser("simpletypes", new StringReader(StringUtilities.implode(SimpletypesConfig.CONFIG_DEF_SCHEMA, "\n"))).getTree(); new ConfigFileFormat(def).encode(baos, slime); - assertThat(baos.toString(StandardCharsets.UTF_8), is("stringval \"" + value + "\"\n")); + assertEquals("stringval \"" + value + "\"\n", baos.toString(StandardCharsets.UTF_8)); } @Test @@ -189,7 +190,7 @@ public class ConfigFileFormatterTest { ByteArrayOutputStream baos = new ByteArrayOutputStream(); InnerCNode def = new DefParser("arraytypes", new StringReader(StringUtilities.implode(ArraytypesConfig.CONFIG_DEF_SCHEMA, "\n"))).getTree(); new ConfigFileFormat(def).encode(baos, slime); - assertThat(baos.toString(), is( + assertEquals( "boolarr[0] true\n" + "boolarr[1] false\n" + "doublearr[0] 3.14\n" + @@ -201,7 +202,8 @@ public class ConfigFileFormatterTest { "longarr[0] 55\n" + "longarr[1] 66\n" + "stringarr[0] \"foo\"\n" + - "stringarr[1] \"bar\"\n")); + "stringarr[1] \"bar\"\n", + baos.toString()); } @Test @@ -221,7 +223,7 @@ public class ConfigFileFormatterTest { ByteArrayOutputStream baos = new ByteArrayOutputStream(); InnerCNode def = new DefParser("maptypes", new StringReader(StringUtilities.implode(MaptypesConfig.CONFIG_DEF_SCHEMA, "\n"))).getTree(); new ConfigFileFormat(def).encode(baos, slime); - assertThat(baos.toString(), is( + assertEquals( "boolmap{\"foo\"} true\n" + "boolmap{\"bar\"} false\n" + "intmap{\"foo\"} 1234\n" + @@ -229,7 +231,8 @@ public class ConfigFileFormatterTest { "doublemap{\"foo\"} 3.14\n" + "stringmap{\"foo\"} \"bar\"\n" + "innermap{\"bar\"}.foo 1234\n" + - "nestedmap{\"baz\"}.inner{\"foo\"} 1234\n")); + "nestedmap{\"baz\"}.inner{\"foo\"} 1234\n", + baos.toString()); } @Test @@ -246,12 +249,12 @@ public class ConfigFileFormatterTest { ByteArrayOutputStream baos = new ByteArrayOutputStream(); InnerCNode def = new DefParser("structtypes", new StringReader(StringUtilities.implode(StructtypesConfig.CONFIG_DEF_SCHEMA, "\n"))).getTree(); new ConfigFileFormat(def).encode(baos, slime); - assertThat(baos.toString(), is( + assertEquals( "simple.name \"myname\"\n" + "simple.gender FEMALE\n" + "simple.emails[0] \"foo@bar.com\"\n" + - "simple.emails[1] \"bar@baz.net\"\n" - )); + "simple.emails[1] \"bar@baz.net\"\n", + baos.toString()); } @Test @@ -286,8 +289,7 @@ public class ConfigFileFormatterTest { ByteArrayOutputStream baos = new ByteArrayOutputStream(); InnerCNode def = new DefParser("structtypes", new StringReader(StringUtilities.implode(StructtypesConfig.CONFIG_DEF_SCHEMA, "\n"))).getTree(); new ConfigFileFormat(def).encode(baos, slime); - assertThat(baos.toString(), is( - "nested.inner.name \"baz\"\n" + + assertEquals("nested.inner.name \"baz\"\n" + "nested.inner.gender FEMALE\n" + "nested.inner.emails[0] \"foo\"\n" + "nested.inner.emails[1] \"bar\"\n" + @@ -296,8 +298,8 @@ public class ConfigFileFormatterTest { "nestedarr[0].inner.emails[0] \"foo@bar\"\n" + "nestedarr[0].inner.emails[1] \"bar@foo\"\n" + "complexarr[0].innerarr[0].name \"bar\"\n" + - "complexarr[0].innerarr[0].gender MALE\n" - )); + "complexarr[0].innerarr[0].gender MALE\n", + baos.toString()); } @Test @@ -308,7 +310,7 @@ public class ConfigFileFormatterTest { InnerCNode def = new DefParser("simpletypes", new StringReader(StringUtilities.implode(SimpletypesConfig.CONFIG_DEF_SCHEMA, "\n"))).getTree(); ByteArrayOutputStream baos = new ByteArrayOutputStream(); new ConfigFileFormat(def).encode(baos, slime); - assertThat(baos.toString(), is("stringval \"some\\\"quotes\\\\\\\"instring\"\n")); + assertEquals("stringval \"some\\\"quotes\\\\\\\"instring\"\n", baos.toString()); } @Test @@ -323,7 +325,7 @@ public class ConfigFileFormatterTest { ByteArrayOutputStream baos = new ByteArrayOutputStream(); new ConfigFileFormat(def).encode(baos, slime); System.out.println(bytesToHexString(baos.toByteArray())); - assertThat(Utf8.toString(baos.toByteArray()), is("stringval \"" + input + "\"\n")); + assertEquals(Utf8.toString(baos.toByteArray()), "stringval \"" + input + "\"\n"); } private static String bytesToHexString(byte[] bytes){ diff --git a/config/src/test/java/com/yahoo/vespa/config/ConfigPayloadBuilderTest.java b/config/src/test/java/com/yahoo/vespa/config/ConfigPayloadBuilderTest.java index 3e5cf22ef37..cc5ec7de1ad 100644 --- a/config/src/test/java/com/yahoo/vespa/config/ConfigPayloadBuilderTest.java +++ b/config/src/test/java/com/yahoo/vespa/config/ConfigPayloadBuilderTest.java @@ -10,10 +10,8 @@ import org.junit.Test; import java.io.ByteArrayOutputStream; import java.io.IOException; -import static org.hamcrest.CoreMatchers.is; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNotNull; -import static org.junit.Assert.assertThat; /** * @author Ulf Lilleengen @@ -89,12 +87,12 @@ public class ConfigPayloadBuilderTest { map.put("barkey", "barvalue"); map.put("bazkey", "bazvalue"); map.put("fookey", "lolvalue"); - assertThat(map.getElements().size(), is(3)); + assertEquals(3, map.getElements().size()); Cursor root = createSlime(builder); Cursor a = root.field("foo"); - assertThat(a.field("barkey").asString(), is("barvalue")); - assertThat(a.field("bazkey").asString(), is("bazvalue")); - assertThat(a.field("fookey").asString(), is("lolvalue")); + assertEquals("barvalue", a.field("barkey").asString()); + assertEquals("bazvalue", a.field("bazkey").asString()); + assertEquals("lolvalue", a.field("fookey").asString()); } @Test @@ -111,7 +109,7 @@ public class ConfigPayloadBuilderTest { array.append("bar"); array.append("baz"); array.append("bim"); - assertThat(array.getElements().size(), is(3)); + assertEquals(3, array.getElements().size()); Cursor root = createSlime(builder); Cursor a = root.field("foo"); assertEquals("bar", a.entry(0).asString()); @@ -174,16 +172,16 @@ public class ConfigPayloadBuilderTest { ConfigPayloadBuilder b2 = array.get(1); ConfigPayloadBuilder b3 = array.get(0); ConfigPayloadBuilder b4 = array.get(1); - assertThat(b1, is(b3)); - assertThat(b2, is(b4)); + assertEquals(b1, b3); + assertEquals(b2, b4); ConfigPayloadBuilder.Array array_indexed = builder.getArray("bar"); ConfigPayloadBuilder bi3 = array_indexed.set(3); ConfigPayloadBuilder bi1 = array_indexed.set(1); ConfigPayloadBuilder bi32 = array_indexed.get(3); ConfigPayloadBuilder bi12 = array_indexed.get(1); - assertThat(bi12, is(bi1)); - assertThat(bi32, is(bi3)); + assertEquals(bi12, bi1); + assertEquals(bi32, bi3); } @Test @@ -204,30 +202,30 @@ public class ConfigPayloadBuilderTest { b3.getArray("eee").append("lll"); b3.setField("uuu", "vvv"); - assertThat(b1.override(b2), is(b1)); - assertThat(b1.override(b3), is(b1)); + assertEquals(b1, b1.override(b2)); + assertEquals(b1, b1.override(b3)); Cursor b1root = createSlime(b1); Cursor b2root = createSlime(b2); Cursor b3root = createSlime(b3); - assertThat(b3root.field("aaa").asString(), is("c")); - assertThat(b3root.field("bbb").field("ccc").asString(), is("fff")); - assertThat(b3root.field("eee").children(), is(1)); - assertThat(b3root.field("eee").entry(0).asString(), is("lll")); - assertThat(b3root.field("uuu").asString(), is("vvv")); - - assertThat(b2root.field("aaa").asString(), is("b")); - assertThat(b2root.field("bbb").field("ccc").asString(), is("eee")); - assertThat(b2root.field("eee").children(), is(1)); - assertThat(b2root.field("eee").entry(0).asString(), is("kkk")); - assertThat(b2root.field("uuu").asString(), is("ttt")); - - assertThat(b1root.field("aaa").asString(), is("c")); - assertThat(b1root.field("bbb").field("ccc").asString(), is("fff")); - assertThat(b1root.field("eee").children(), is(2)); - assertThat(b1root.field("eee").entry(0).asString(), is("kkk")); - assertThat(b1root.field("eee").entry(1).asString(), is("lll")); - assertThat(b1root.field("uuu").asString(), is("vvv")); + assertEquals("c", b3root.field("aaa").asString()); + assertEquals("fff", b3root.field("bbb").field("ccc").asString()); + assertEquals(1, b3root.field("eee").children()); + assertEquals("lll", b3root.field("eee").entry(0).asString()); + assertEquals("vvv", b3root.field("uuu").asString()); + + assertEquals("b", b2root.field("aaa").asString()); + assertEquals("eee", b2root.field("bbb").field("ccc").asString()); + assertEquals(1, b2root.field("eee").children()); + assertEquals("kkk", b2root.field("eee").entry(0).asString()); + assertEquals("ttt", b2root.field("uuu").asString()); + + assertEquals("c", b1root.field("aaa").asString()); + assertEquals("fff", b1root.field("bbb").field("ccc").asString()); + assertEquals(2, b1root.field("eee").children()); + assertEquals("kkk", b1root.field("eee").entry(0).asString()); + assertEquals("lll", b1root.field("eee").entry(1).asString()); + assertEquals("vvv", b1root.field("uuu").asString()); } @Test(expected=IllegalStateException.class) @@ -266,7 +264,8 @@ public class ConfigPayloadBuilderTest { ConfigPayloadBuilder builder = new ConfigPayloadBuilder(new ConfigPayload(slime)); ByteArrayOutputStream baos = new ByteArrayOutputStream(); ConfigPayload.fromBuilder(builder).serialize(baos, new JsonFormat(true)); - assertThat(baos.toString(), is("{\"foo\":\"bar\",\"foorio\":{\"bar\":\"bam\",\"bario\":{\"bim\":\"bul\"},\"blim\":[{\"fim\":\"fam\"},{\"blim\":\"blam\"}]},\"arrio\":[\"himbio\"]}")); + assertEquals("{\"foo\":\"bar\",\"foorio\":{\"bar\":\"bam\",\"bario\":{\"bim\":\"bul\"},\"blim\":[{\"fim\":\"fam\"},{\"blim\":\"blam\"}]},\"arrio\":[\"himbio\"]}", + baos.toString()); } @Test(expected=IllegalArgumentException.class) diff --git a/config/src/test/java/com/yahoo/vespa/config/ConfigPayloadTest.java b/config/src/test/java/com/yahoo/vespa/config/ConfigPayloadTest.java index 50f4233e13a..46b710cdcf9 100644 --- a/config/src/test/java/com/yahoo/vespa/config/ConfigPayloadTest.java +++ b/config/src/test/java/com/yahoo/vespa/config/ConfigPayloadTest.java @@ -17,10 +17,10 @@ import org.junit.Test; import java.io.StringReader; import static org.hamcrest.CoreMatchers.is; +import static org.hamcrest.MatcherAssert.assertThat; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertNotNull; -import static org.junit.Assert.assertThat; import static org.junit.Assert.assertTrue; /** diff --git a/config/src/test/java/com/yahoo/vespa/config/DefaultValueApplierTest.java b/config/src/test/java/com/yahoo/vespa/config/DefaultValueApplierTest.java index 0acefe5cd18..199d5f14b9f 100644 --- a/config/src/test/java/com/yahoo/vespa/config/DefaultValueApplierTest.java +++ b/config/src/test/java/com/yahoo/vespa/config/DefaultValueApplierTest.java @@ -10,8 +10,7 @@ import org.junit.Test; import java.io.StringReader; -import static org.hamcrest.CoreMatchers.is; -import static org.junit.Assert.assertThat; +import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; /** @@ -43,9 +42,9 @@ public class DefaultValueApplierTest { public void require_that_simple_defaults_are_applied() { Slime slime = apply("strdef string default=\"foo\""); assertTrue(slime.get().field("str").valid()); - assertThat(slime.get().field("str").asString(), is("myvalue")); + assertEquals("myvalue", slime.get().field("str").asString()); assertTrue(slime.get().field("strdef").valid()); - assertThat(slime.get().field("strdef").asString(), is("foo")); + assertEquals("foo", slime.get().field("strdef").asString()); } @@ -54,7 +53,7 @@ public class DefaultValueApplierTest { Slime slime = apply("nested.str string default=\"bar\""); assertTrue(slime.get().field("nested").valid()); assertTrue(slime.get().field("nested").field("str").valid()); - assertThat(slime.get().field("nested").field("str").asString(), is("bar")); + assertEquals("bar", slime.get().field("nested").field("str").asString()); } @Test @@ -65,11 +64,11 @@ public class DefaultValueApplierTest { Slime slime = apply(payload, "nestedarr[].foo string", "nestedarr[].bar string default=\"bim\""); assertTrue(slime.get().field("nestedarr").valid()); - assertThat(slime.get().field("nestedarr").entries(), is(1)); + assertEquals(1, slime.get().field("nestedarr").entries()); assertTrue(slime.get().field("nestedarr").entry(0).field("foo").valid()); - assertThat(slime.get().field("nestedarr").entry(0).field("foo").asString(), is("myfoo")); + assertEquals("myfoo", slime.get().field("nestedarr").entry(0).field("foo").asString()); assertTrue(slime.get().field("nestedarr").entry(0).field("bar").valid()); - assertThat(slime.get().field("nestedarr").entry(0).field("bar").asString(), is("bim")); + assertEquals("bim", slime.get().field("nestedarr").entry(0).field("bar").asString()); } @Test @@ -79,8 +78,8 @@ public class DefaultValueApplierTest { Slime slime = apply(payload, "nestedarr[].foo string", "nestedarr[].bar string default=\"bim\""); assertTrue(slime.get().field("nestedarr").valid()); - assertThat(slime.get().field("nestedarr").entries(), is(0)); - assertThat(slime.get().field("nestedarr").type(), is(Type.ARRAY)); + assertEquals(0, slime.get().field("nestedarr").entries()); + assertEquals(Type.ARRAY, slime.get().field("nestedarr").type()); } @Test @@ -91,10 +90,10 @@ public class DefaultValueApplierTest { Slime slime = apply(payload, "nestedmap{}.foo string", "nestedmap{}.bar string default=\"bim\""); assertTrue(slime.get().field("nestedmap").valid()); - assertThat(slime.get().field("nestedmap").fields(), is(1)); + assertEquals(1, slime.get().field("nestedmap").fields()); assertTrue(slime.get().field("nestedmap").field("mykey").field("foo").valid()); - assertThat(slime.get().field("nestedmap").field("mykey").field("foo").asString(), is("myfoo")); + assertEquals("myfoo", slime.get().field("nestedmap").field("mykey").field("foo").asString()); assertTrue(slime.get().field("nestedmap").field("mykey").field("bar").valid()); - assertThat(slime.get().field("nestedmap").field("mykey").field("bar").asString(), is("bim")); + assertEquals("bim", slime.get().field("nestedmap").field("mykey").field("bar").asString()); } } diff --git a/config/src/test/java/com/yahoo/vespa/config/ErrorCodeTest.java b/config/src/test/java/com/yahoo/vespa/config/ErrorCodeTest.java index 4ae8bc676d7..3bbdd6f0aa2 100644 --- a/config/src/test/java/com/yahoo/vespa/config/ErrorCodeTest.java +++ b/config/src/test/java/com/yahoo/vespa/config/ErrorCodeTest.java @@ -4,8 +4,7 @@ package com.yahoo.vespa.config; import org.junit.Test; import static org.hamcrest.CoreMatchers.is; -import static org.junit.Assert.assertThat; - +import static org.hamcrest.MatcherAssert.assertThat; /** * @author hmusum */ diff --git a/config/src/test/java/com/yahoo/vespa/config/GenericConfigBuilderTest.java b/config/src/test/java/com/yahoo/vespa/config/GenericConfigBuilderTest.java index bb612bb3245..ff801e5cf12 100644 --- a/config/src/test/java/com/yahoo/vespa/config/GenericConfigBuilderTest.java +++ b/config/src/test/java/com/yahoo/vespa/config/GenericConfigBuilderTest.java @@ -9,8 +9,7 @@ import java.io.ByteArrayOutputStream; import java.io.IOException; import static org.hamcrest.core.Is.is; -import static org.junit.Assert.assertThat; - +import static org.hamcrest.MatcherAssert.assertThat; /** * @author Ulf Lilleengen */ diff --git a/config/src/test/java/com/yahoo/vespa/config/JRTConnectionPoolTest.java b/config/src/test/java/com/yahoo/vespa/config/JRTConnectionPoolTest.java index 43f9a87454d..8186347f998 100644 --- a/config/src/test/java/com/yahoo/vespa/config/JRTConnectionPoolTest.java +++ b/config/src/test/java/com/yahoo/vespa/config/JRTConnectionPoolTest.java @@ -11,12 +11,9 @@ import java.util.List; import java.util.Map; import java.util.stream.Collectors; -import static org.hamcrest.CoreMatchers.is; -import static org.hamcrest.CoreMatchers.not; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNotEquals; import static org.junit.Assert.assertNotNull; -import static org.junit.Assert.assertThat; import static org.junit.Assert.assertTrue; /** @@ -100,7 +97,7 @@ public class JRTConnectionPoolTest { // Update to the same set, should be equal sourcePool.updateSources(twoSources); - assertThat(sourcesBefore, is(sourcePool.getSourceSet())); + assertEquals(sourcePool.getSourceSet(), sourcesBefore); // Update to new set List<String> newSources = new ArrayList<>(); @@ -109,8 +106,8 @@ public class JRTConnectionPoolTest { sourcePool.updateSources(newSources); ConfigSourceSet newSourceSet = sourcePool.getSourceSet(); assertNotNull(newSourceSet); - assertThat(newSourceSet.getSources().size(), is(2)); - assertThat(newSourceSet, is(not(sourcesBefore))); + assertEquals(2, newSourceSet.getSources().size()); + assertNotEquals(sourcesBefore, newSourceSet); assertTrue(newSourceSet.getSources().contains("host2")); assertTrue(newSourceSet.getSources().contains("host3")); @@ -120,8 +117,8 @@ public class JRTConnectionPoolTest { sourcePool.updateSources(newSources2); ConfigSourceSet newSourceSet2 = sourcePool.getSourceSet(); assertNotNull(newSourceSet2); - assertThat(newSourceSet2.getSources().size(), is(1)); - assertThat(newSourceSet2, is(not(newSourceSet))); + assertEquals(1, newSourceSet2.getSources().size()); + assertNotEquals(newSourceSet, newSourceSet2); assertTrue(newSourceSet2.getSources().contains("host4")); sourcePool.close(); diff --git a/config/src/test/java/com/yahoo/vespa/config/LZ4PayloadCompressorTest.java b/config/src/test/java/com/yahoo/vespa/config/LZ4PayloadCompressorTest.java index 56bcdda20a0..c35dd016b39 100644 --- a/config/src/test/java/com/yahoo/vespa/config/LZ4PayloadCompressorTest.java +++ b/config/src/test/java/com/yahoo/vespa/config/LZ4PayloadCompressorTest.java @@ -4,8 +4,7 @@ package com.yahoo.vespa.config; import com.yahoo.text.Utf8; import org.junit.Test; -import static org.hamcrest.core.Is.is; -import static org.junit.Assert.assertThat; +import static org.junit.Assert.assertArrayEquals; /** * @author Ulf Lilleengen @@ -24,6 +23,6 @@ public class LZ4PayloadCompressorTest { byte[] data = Utf8.toBytes(input); byte[] compressed = compressor.compress(data); byte[] output = compressor.decompress(compressed, data.length); - assertThat(data, is(output)); + assertArrayEquals(output, data); } } diff --git a/config/src/test/java/com/yahoo/vespa/config/RawConfigTest.java b/config/src/test/java/com/yahoo/vespa/config/RawConfigTest.java index 12c1d2f8069..7d49d15dc47 100644 --- a/config/src/test/java/com/yahoo/vespa/config/RawConfigTest.java +++ b/config/src/test/java/com/yahoo/vespa/config/RawConfigTest.java @@ -16,11 +16,11 @@ import static com.yahoo.vespa.config.PayloadChecksum.Type.MD5; import static com.yahoo.vespa.config.PayloadChecksum.Type.XXHASH64; import static org.hamcrest.CoreMatchers.is; import static org.hamcrest.CoreMatchers.not; +import static org.hamcrest.MatcherAssert.assertThat; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertNotEquals; import static org.junit.Assert.assertNull; -import static org.junit.Assert.assertThat; /** * @author hmusum diff --git a/config/src/test/java/com/yahoo/vespa/config/protocol/JRTConfigRequestV3Test.java b/config/src/test/java/com/yahoo/vespa/config/protocol/JRTConfigRequestV3Test.java index dabd87e1eec..8d160076db9 100644 --- a/config/src/test/java/com/yahoo/vespa/config/protocol/JRTConfigRequestV3Test.java +++ b/config/src/test/java/com/yahoo/vespa/config/protocol/JRTConfigRequestV3Test.java @@ -28,10 +28,10 @@ import java.util.Optional; import static com.yahoo.vespa.config.PayloadChecksum.Type.MD5; import static com.yahoo.vespa.config.PayloadChecksum.Type.XXHASH64; import static org.hamcrest.CoreMatchers.is; +import static org.hamcrest.MatcherAssert.assertThat; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertNotNull; -import static org.junit.Assert.assertThat; import static org.junit.Assert.assertTrue; /** diff --git a/config/src/test/java/com/yahoo/vespa/config/protocol/PayloadTest.java b/config/src/test/java/com/yahoo/vespa/config/protocol/PayloadTest.java index 8d895d0e2b8..af6aefb26e1 100644 --- a/config/src/test/java/com/yahoo/vespa/config/protocol/PayloadTest.java +++ b/config/src/test/java/com/yahoo/vespa/config/protocol/PayloadTest.java @@ -10,8 +10,7 @@ import org.junit.Test; import java.nio.charset.StandardCharsets; -import static org.hamcrest.CoreMatchers.is; -import static org.junit.Assert.assertThat; +import static org.junit.Assert.assertEquals; /** * @author Ulf Lilleengen @@ -23,13 +22,13 @@ public class PayloadTest { String json = "{\"foo\":13}"; ConfigPayload configPayload = ConfigPayload.fromString(json); Payload payload = Payload.from(configPayload); - assertThat(payload.getData().toString(), is(json)); + assertEquals(json, payload.getData().toString()); payload = Payload.from(payload.getData(), CompressionInfo.create(CompressionType.UNCOMPRESSED, 0)); Payload compressed = payload.withCompression(CompressionType.LZ4); Payload uncompressed = compressed.withCompression(CompressionType.UNCOMPRESSED); - assertThat(uncompressed.getData().toString(), is(json)); - assertThat(compressed.toString(), is(json)); - assertThat(uncompressed.toString(), is(json)); + assertEquals(json, uncompressed.getData().toString()); + assertEquals(json, compressed.toString()); + assertEquals(json, uncompressed.toString()); } @Test @@ -52,10 +51,7 @@ public class PayloadTest { slime.setString("foo 2"); Payload f = Payload.from(new ConfigPayload(slime)); - Payload g = null; - Payload h = null; - Payload i = null; - Payload j = null; + Payload g, h, i, j; g = Payload.from(new Utf8Array(foo1.getBytes(StandardCharsets.UTF_8)), CompressionInfo.uncompressed()); h = Payload.from(new Utf8Array(foo1.getBytes(StandardCharsets.UTF_8)), CompressionInfo.uncompressed()); diff --git a/config/src/test/java/com/yahoo/vespa/config/protocol/SlimeTraceSerializerTest.java b/config/src/test/java/com/yahoo/vespa/config/protocol/SlimeTraceSerializerTest.java index 3525809a447..8b1de561a1e 100644 --- a/config/src/test/java/com/yahoo/vespa/config/protocol/SlimeTraceSerializerTest.java +++ b/config/src/test/java/com/yahoo/vespa/config/protocol/SlimeTraceSerializerTest.java @@ -15,8 +15,8 @@ import java.util.Iterator; import java.util.Map; import static org.hamcrest.CoreMatchers.is; +import static org.hamcrest.MatcherAssert.assertThat; import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertThat; import static org.junit.Assert.assertTrue; /** diff --git a/configgen/src/test/java/com/yahoo/config/codegen/DefLineParsingTest.java b/configgen/src/test/java/com/yahoo/config/codegen/DefLineParsingTest.java index 05e2e7c6ff6..9cf363376cc 100644 --- a/configgen/src/test/java/com/yahoo/config/codegen/DefLineParsingTest.java +++ b/configgen/src/test/java/com/yahoo/config/codegen/DefLineParsingTest.java @@ -1,11 +1,13 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.config.codegen; -import static org.hamcrest.CoreMatchers.is; -import static org.junit.Assert.*; - import org.junit.Test; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertTrue; + /** * Tests parsing of a single line of a .def file * @@ -225,9 +227,9 @@ public class DefLineParsingTest { DefLine i = new DefLine("i int range=[0, 100]"); DefLine l = new DefLine("l long range=[-1e10, 0]"); DefLine d = new DefLine("d double range=[0, 1.0]"); - assertThat(i.getRange(), is("[0, 100]")); - assertThat(l.getRange(), is("[-1e10, 0]")); - assertThat(d.getRange(), is("[0, 1.0]")); + assertEquals("[0, 100]", i.getRange()); + assertEquals("[-1e10, 0]", l.getRange()); + assertEquals("[0, 1.0]", d.getRange()); } @Test diff --git a/configgen/src/test/java/com/yahoo/config/codegen/DefParserNamespaceTest.java b/configgen/src/test/java/com/yahoo/config/codegen/DefParserNamespaceTest.java index 97ee17cb6d5..7921a3aecbe 100644 --- a/configgen/src/test/java/com/yahoo/config/codegen/DefParserNamespaceTest.java +++ b/configgen/src/test/java/com/yahoo/config/codegen/DefParserNamespaceTest.java @@ -8,9 +8,8 @@ import java.io.IOException; import static com.yahoo.config.codegen.DefParserTest.assertLineFails; import static com.yahoo.config.codegen.DefParserTest.createDefTemplate; import static com.yahoo.config.codegen.DefParserTest.createParser; -import static org.hamcrest.CoreMatchers.is; -import static org.hamcrest.CoreMatchers.not; -import static org.junit.Assert.assertThat; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotEquals; /** * @author gjoranv @@ -22,7 +21,7 @@ public class DefParserNamespaceTest { public void namespace_is_set_on_root_node() { DefParser parser = createParser("version=1\nnamespace=myproject.config\n"); CNode root = parser.getTree(); - assertThat(root.getNamespace(), is("myproject.config")); + assertEquals("myproject.config", root.getNamespace()); } @Test @@ -30,7 +29,7 @@ public class DefParserNamespaceTest { String PACKAGE = "com.github.myproject"; DefParser parser = createParser("package=" + PACKAGE + "\n"); CNode root = parser.getTree(); - assertThat(root.getNamespace(), is(PACKAGE)); + assertEquals(PACKAGE, root.getNamespace()); } @Test(expected = CodegenRuntimeException.class) @@ -48,7 +47,7 @@ public class DefParserNamespaceTest { public void spaces_are_allowed_around_equals_sign() { DefParser parser = createParser("version=1\nnamespace = myproject.config\n"); CNode root = parser.getTree(); - assertThat(root.getNamespace(), is("myproject.config")); + assertEquals("myproject.config", root.getNamespace()); } @Test @@ -69,7 +68,7 @@ public class DefParserNamespaceTest { parser = createParser("version=1\nnamespace=myproject.config\n"); CNode namespaceRoot = parser.getTree(); - assertThat(root.defMd5, not(namespaceRoot.defMd5)); + assertNotEquals(root.defMd5, namespaceRoot.defMd5); } @@ -82,12 +81,12 @@ public class DefParserNamespaceTest { } @Test - public void number_is_not_allowed_as_namespace_start_char() throws IOException, DefParser.DefParserException { + public void number_is_not_allowed_as_namespace_start_char() { assertLineFails("namespace=2.a.b"); } @Test - public void number_is_not_allowed_as_leading_char_in_namespace_token() throws IOException, DefParser.DefParserException { + public void number_is_not_allowed_as_leading_char_in_namespace_token() { assertLineFails("namespace=a.b.2c"); } diff --git a/configgen/src/test/java/com/yahoo/config/codegen/DefParserPackageTest.java b/configgen/src/test/java/com/yahoo/config/codegen/DefParserPackageTest.java index d995ab63886..cd7ca1a89b9 100644 --- a/configgen/src/test/java/com/yahoo/config/codegen/DefParserPackageTest.java +++ b/configgen/src/test/java/com/yahoo/config/codegen/DefParserPackageTest.java @@ -5,14 +5,12 @@ import org.junit.Test; import java.io.IOException; -import static com.yahoo.config.codegen.DefParser.DEFAULT_PACKAGE_PREFIX; import static com.yahoo.config.codegen.DefParserTest.assertLineFails; import static com.yahoo.config.codegen.DefParserTest.createDefTemplate; import static com.yahoo.config.codegen.DefParserTest.createParser; -import static org.hamcrest.CoreMatchers.is; -import static org.hamcrest.CoreMatchers.not; -import static org.hamcrest.CoreMatchers.nullValue; -import static org.junit.Assert.assertThat; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotEquals; +import static org.junit.Assert.assertNull; /** * Tests setting explicit java package in the def file. @@ -26,7 +24,7 @@ public class DefParserPackageTest { public void package_is_set_on_root_node() { DefParser parser = createParser("package=" + PACKAGE + "\n"); CNode root = parser.getTree(); - assertThat(root.getPackage(), is(PACKAGE)); + assertEquals(PACKAGE, root.getPackage()); } @Test @@ -35,8 +33,8 @@ public class DefParserPackageTest { DefParser parser = createParser("package=" + PACKAGE + "\nnamespace=" + namespace +"\n"); CNode root = parser.getTree(); - assertThat(root.getPackage(), is(PACKAGE)); - assertThat(root.getNamespace(), is(namespace)); + assertEquals(PACKAGE, root.getPackage()); + assertEquals(namespace, root.getNamespace()); } // Required by JavaClassBuilder ctor. @@ -45,7 +43,7 @@ public class DefParserPackageTest { String namespace = "test.namespace"; DefParser parser = createParser("namespace=" + namespace + "\n"); CNode root = parser.getTree(); - assertThat(root.getPackage(), nullValue()); + assertNull(root.getPackage()); } @Test(expected = CodegenRuntimeException.class) @@ -57,7 +55,7 @@ public class DefParserPackageTest { public void spaces_are_allowed_around_equals_sign() { DefParser parser = createParser("package = " + PACKAGE + "\n"); CNode root = parser.getTree(); - assertThat(root.getPackage(), is(PACKAGE)); + assertEquals(PACKAGE, root.getPackage()); } @Test @@ -78,7 +76,7 @@ public class DefParserPackageTest { parser = createParser("package=" + PACKAGE + "\na string\n"); CNode rootWithPackage = parser.getTree(); - assertThat(root.defMd5, not(rootWithPackage.defMd5)); + assertNotEquals(root.defMd5, rootWithPackage.defMd5); } @@ -91,12 +89,12 @@ public class DefParserPackageTest { } @Test - public void number_is_not_allowed_as_package_start_char() throws IOException, DefParser.DefParserException { + public void number_is_not_allowed_as_package_start_char() { assertLineFails("package=2.a.b"); } @Test - public void number_is_not_allowed_as_leading_char_in_package_token() throws IOException, DefParser.DefParserException { + public void number_is_not_allowed_as_leading_char_in_package_token() { assertLineFails("package=a.b.2c"); } diff --git a/configgen/src/test/java/com/yahoo/config/codegen/DefParserTest.java b/configgen/src/test/java/com/yahoo/config/codegen/DefParserTest.java index 22ccf4105c1..e4d9f6cb3bd 100644 --- a/configgen/src/test/java/com/yahoo/config/codegen/DefParserTest.java +++ b/configgen/src/test/java/com/yahoo/config/codegen/DefParserTest.java @@ -1,15 +1,18 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.config.codegen; -import static org.junit.Assert.*; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; import org.junit.Test; import org.junit.Ignore; -import static org.hamcrest.CoreMatchers.is; - -import java.io.*; +import java.io.File; +import java.io.FileReader; +import java.io.IOException; +import java.io.StringReader; /** * Unit tests for DefParser. @@ -28,7 +31,7 @@ public class DefParserTest { CNode root = new DefParser("test", new FileReader(defFile)).getTree(); assertNotNull(root); CNode[] children = root.getChildren(); - assertThat(children.length, is(34)); + assertEquals(34, children.length); int numGrandChildren = 0; int numGreatGrandChildren = 0; @@ -39,45 +42,45 @@ public class DefParserTest { numGreatGrandChildren += grandChild.getChildren().length; } } - assertThat(numGrandChildren, is(14)); - assertThat(numGreatGrandChildren, is(6)); + assertEquals(14, numGrandChildren); + assertEquals(6, numGreatGrandChildren); // Verify that each array creates a sub-tree, and that defaults for leafs are handled correctly. CNode myArray = root.getChild("myArray"); - assertThat(myArray.getChildren().length, is(5)); + assertEquals(5, myArray.getChildren().length); // int within array LeafCNode myArrayInt = (LeafCNode) myArray.getChild("intVal"); - assertThat(myArrayInt.getDefaultValue().getValue(), is("14")); + assertEquals("14", myArrayInt.getDefaultValue().getValue()); // enum within array LeafCNode myArrayEnum = (LeafCNode) myArray.getChild("enumVal"); - assertThat(myArrayEnum.getDefaultValue().getValue(), is("TYPE")); + assertEquals("TYPE", myArrayEnum.getDefaultValue().getValue()); // Verify array within array and a default value for a leaf in the inner array. CNode anotherArray = myArray.getChild("anotherArray"); - assertThat(anotherArray.getChildren().length, is(1)); + assertEquals(1, anotherArray.getChildren().length); LeafCNode foo = (LeafCNode) anotherArray.getChild("foo"); - assertThat(foo.getDefaultValue().getValue(), is("-4")); + assertEquals("-4", foo.getDefaultValue().getValue()); } @Test public void testFileWithNamespaceInFilename() throws IOException { File defFile = new File(TEST_DIR + "baz.bar.foo.def"); CNode root = new DefParser("test", new FileReader(defFile)).getTree(); - assertThat(root.defMd5, is("31a0f9bda0e5ff929762a29569575a7e")); + assertEquals("31a0f9bda0e5ff929762a29569575a7e", root.defMd5); } @Test public void testMd5Sum() throws IOException { File defFile = new File(DEF_NAME); CNode root = new DefParser("test", new FileReader(defFile)).getTree(); - assertThat(root.defMd5, is("f901bdc5c96e7005130399c63f247823")); + assertEquals("f901bdc5c96e7005130399c63f247823", root.defMd5); } @Test public void testMd5Sum2() { String def = "version=1\na string\n"; CNode root = new DefParser("testMd5Sum2", new StringReader(def)).getTree(); - assertThat(root.defMd5, is("a5e5fdbb2b27e56ba7d5e60e335c598b")); + assertEquals("a5e5fdbb2b27e56ba7d5e60e335c598b", root.defMd5); } @Test @@ -104,7 +107,7 @@ public class DefParserTest { private void testExpectedVersion(String versionLine, String expectedVersion) { InnerCNode root = createParser(versionLine).getTree(); - assertThat(root.defVersion, is(expectedVersion)); + assertEquals(expectedVersion, root.defVersion); } @Test @@ -146,7 +149,7 @@ public class DefParserTest { @Test(expected = CodegenRuntimeException.class) @Ignore("Not implemented yet") - public void testInvalidEnum() throws DefParser.DefParserException { + public void testInvalidEnum() { DefParser parser = createParser("version=1\nanEnum enum {A, B, A}\n"); //parser.validateDef(def); } @@ -171,7 +174,7 @@ public class DefParserTest { CNode root = parser.getTree(); LeafCNode node = (LeafCNode) root.getChild("enum1"); assertNotNull(node); - assertThat(node.getDefaultValue().getStringRepresentation(), is("A")); + assertEquals("A", node.getDefaultValue().getStringRepresentation()); } @Test(expected = DefParser.DefParserException.class) diff --git a/configgen/src/test/java/com/yahoo/config/codegen/JavaClassBuilderTest.java b/configgen/src/test/java/com/yahoo/config/codegen/JavaClassBuilderTest.java index cce2d66b5f0..d69617ec5da 100644 --- a/configgen/src/test/java/com/yahoo/config/codegen/JavaClassBuilderTest.java +++ b/configgen/src/test/java/com/yahoo/config/codegen/JavaClassBuilderTest.java @@ -12,9 +12,7 @@ import java.util.List; import static com.yahoo.config.codegen.ConfiggenUtil.createClassName; import static com.yahoo.config.codegen.JavaClassBuilder.createUniqueSymbol; -import static org.hamcrest.CoreMatchers.is; import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertThat; import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; @@ -82,8 +80,8 @@ public class JavaClassBuilderTest { "n int\n"; InnerCNode root = new DefParser("test", new StringReader(testDefinition)).getTree(); - assertThat(createUniqueSymbol(root, "foo"), is("f")); - assertThat(createUniqueSymbol(root, "name"), is("na")); + assertEquals("f", createUniqueSymbol(root, "foo")); + assertEquals("na", createUniqueSymbol(root, "name")); assertTrue(createUniqueSymbol(root, "m").startsWith(ReservedWords.INTERNAL_PREFIX + "m")); // The basis string is not a legal return value, even if unique, to avoid @@ -93,12 +91,12 @@ public class JavaClassBuilderTest { @Test public void testCreateClassName() { - assertThat(createClassName("simple"), is("SimpleConfig")); - assertThat(createClassName("a"), is("AConfig")); - assertThat(createClassName("a-b-c"), is("ABCConfig")); - assertThat(createClassName("a-1-2b"), is("A12bConfig")); - assertThat(createClassName("my-app"), is("MyAppConfig")); - assertThat(createClassName("MyApp"), is("MyAppConfig")); + assertEquals("SimpleConfig", createClassName("simple")); + assertEquals("AConfig", createClassName("a")); + assertEquals("ABCConfig", createClassName("a-b-c")); + assertEquals("A12bConfig", createClassName("a-1-2b")); + assertEquals("MyAppConfig", createClassName("my-app")); + assertEquals("MyAppConfig", createClassName("MyApp")); } @Test(expected = CodegenRuntimeException.class) diff --git a/configgen/src/test/java/com/yahoo/config/codegen/NormalizedDefinitionTest.java b/configgen/src/test/java/com/yahoo/config/codegen/NormalizedDefinitionTest.java index 3c328721b6c..411ac0fe58a 100644 --- a/configgen/src/test/java/com/yahoo/config/codegen/NormalizedDefinitionTest.java +++ b/configgen/src/test/java/com/yahoo/config/codegen/NormalizedDefinitionTest.java @@ -1,14 +1,18 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.config.codegen; -import static org.junit.Assert.*; -import java.io.*; +import java.io.BufferedReader; +import java.io.FileNotFoundException; +import java.io.FileReader; +import java.io.IOException; +import java.io.StringReader; import java.util.List; import org.junit.Test; -import static org.hamcrest.CoreMatchers.is; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; /** @@ -38,13 +42,13 @@ public class NormalizedDefinitionTest { } assertNotNull(out); - assertThat(out.size(), is(6)); - assertThat(out.get(0), is ("version=1\n")); - assertThat(out.get(1), is ("aString string\n")); - assertThat(out.get(2), is ("anInt int\n")); - assertThat(out.get(3), is ("aStringCommentCharacterAfter string default=\"ab\"\n")); - assertThat(out.get(4), is ("aStringWithCommentCharacter string default=\"a#b\"\n")); - assertThat(out.get(5), is ("aStringWithEscapedQuote string default=\"a\"b\"\n")); + assertEquals(6, out.size()); + assertEquals("version=1\n", out.get(0)); + assertEquals("aString string\n", out.get(1)); + assertEquals("anInt int\n", out.get(2)); + assertEquals("aStringCommentCharacterAfter string default=\"ab\"\n", out.get(3)); + assertEquals("aStringWithCommentCharacter string default=\"a#b\"\n", out.get(4)); + assertEquals("aStringWithEscapedQuote string default=\"a\"b\"\n", out.get(5)); reader.close(); } @@ -68,7 +72,7 @@ public class NormalizedDefinitionTest { } assertNotNull(out); - assertThat(out.size(), is(72)); + assertEquals(72, out.size()); assertNotNull(fileReader); fileReader.close(); diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java index 1b55d17fd36..063603fe8a8 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java @@ -202,6 +202,9 @@ public class ModelContextImpl implements ModelContext { private final boolean unorderedMergeChaining; private final boolean useV8GeoPositions; private final boolean useV8DocManagerCfg; + private final int maxCompactBuffers; + private final boolean failDeploymentWithInvalidJvmOptions; + private final double tlsSizeFraction; public FeatureFlags(FlagSource source, ApplicationId appId) { this.defaultTermwiseLimit = flagValue(source, appId, Flags.DEFAULT_TERM_WISE_LIMIT); @@ -242,6 +245,9 @@ public class ModelContextImpl implements ModelContext { this.unorderedMergeChaining = flagValue(source, appId, Flags.UNORDERED_MERGE_CHAINING); this.useV8GeoPositions = flagValue(source, appId, Flags.USE_V8_GEO_POSITIONS); this.useV8DocManagerCfg = flagValue(source, appId, Flags.USE_V8_DOC_MANAGER_CFG); + this.maxCompactBuffers = flagValue(source, appId, Flags.MAX_COMPACT_BUFFERS); + this.failDeploymentWithInvalidJvmOptions = flagValue(source, appId, Flags.FAIL_DEPLOYMENT_WITH_INVALID_JVM_OPTIONS); + this.tlsSizeFraction = flagValue(source, appId, Flags.TLS_SIZE_FRACTION); } @Override public double defaultTermwiseLimit() { return defaultTermwiseLimit; } @@ -284,6 +290,9 @@ public class ModelContextImpl implements ModelContext { @Override public boolean unorderedMergeChaining() { return unorderedMergeChaining; } @Override public boolean useV8GeoPositions() { return useV8GeoPositions; } @Override public boolean useV8DocManagerCfg() { return useV8DocManagerCfg; } + @Override public boolean failDeploymentWithInvalidJvmOptions() { return failDeploymentWithInvalidJvmOptions; } + @Override public int maxCompactBuffers() { return maxCompactBuffers; } + @Override public double tlsSizeFraction() { return tlsSizeFraction; } private static <V> V flagValue(FlagSource source, ApplicationId appId, UnboundFlag<? extends V, ?, ?> flag) { return flag.bindTo(source) diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/rpc/RpcServer.java b/configserver/src/main/java/com/yahoo/vespa/config/server/rpc/RpcServer.java index 99ffff6403b..6d16520d77b 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/rpc/RpcServer.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/rpc/RpcServer.java @@ -558,7 +558,7 @@ public class RpcServer implements Runnable, ReloadListener, TenantListener { private void invokeRpcIfValidConnection(Request request) { if (target.isValid()) { - target.invokeSync(request, 600); + target.invokeSync(request, 60); } else { throw new RuntimeException("Connection to " + target + " is invalid", target.getConnectionLostReason()); } diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/ModelContextImplTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/ModelContextImplTest.java index b0046a201ab..0e8c69327e2 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/ModelContextImplTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/ModelContextImplTest.java @@ -102,6 +102,7 @@ public class ModelContextImplTest { assertEquals(1.0, context.properties().featureFlags().defaultTermwiseLimit(), 0.0); assertFalse(context.properties().featureFlags().useAsyncMessageHandlingOnSchedule()); assertEquals(0.5, context.properties().featureFlags().feedConcurrency(), 0.0); + assertEquals(1, context.properties().featureFlags().maxCompactBuffers()); } } diff --git a/container-core/abi-spec.json b/container-core/abi-spec.json index a6783d1e5f5..174febca9df 100644 --- a/container-core/abi-spec.json +++ b/container-core/abi-spec.json @@ -875,6 +875,22 @@ ], "fields": [] }, + "com.yahoo.container.jdisc.ThreadedHttpRequestHandler$Context": { + "superClass": "java.lang.Object", + "interfaces": [], + "attributes": [ + "public" + ], + "methods": [ + "public void <init>(java.util.concurrent.Executor, com.yahoo.container.logging.AccessLog, com.yahoo.jdisc.Metric)", + "public void <init>(java.util.concurrent.Executor, com.yahoo.jdisc.Metric)", + "public void <init>(com.yahoo.container.jdisc.ThreadedHttpRequestHandler$Context)", + "public java.util.concurrent.Executor getExecutor()", + "public com.yahoo.container.logging.AccessLog getAccessLog()", + "public com.yahoo.jdisc.Metric getMetric()" + ], + "fields": [] + }, "com.yahoo.container.jdisc.ThreadedHttpRequestHandler$LazyContentChannel": { "superClass": "java.lang.Object", "interfaces": [ @@ -903,13 +919,15 @@ "methods": [ "public void <init>(java.util.concurrent.Executor)", "public void <init>(java.util.concurrent.Executor, com.yahoo.jdisc.Metric)", + "public void <init>(com.yahoo.container.jdisc.ThreadedHttpRequestHandler$Context)", "public void <init>(java.util.concurrent.Executor, com.yahoo.jdisc.Metric, boolean)", "public abstract com.yahoo.container.jdisc.HttpResponse handle(com.yahoo.container.jdisc.HttpRequest)", "public com.yahoo.container.jdisc.HttpResponse handle(com.yahoo.container.jdisc.HttpRequest, com.yahoo.jdisc.handler.ContentChannel)", "public final void handleRequest(com.yahoo.jdisc.Request, com.yahoo.jdisc.handler.BufferedContentChannel, com.yahoo.jdisc.handler.ResponseHandler)", "protected void addDateHeader(com.yahoo.container.jdisc.HttpResponse, long)", "protected com.yahoo.container.jdisc.LoggingCompletionHandler createLoggingCompletionHandler(long, long, com.yahoo.container.jdisc.HttpResponse, com.yahoo.container.jdisc.HttpRequest, com.yahoo.container.jdisc.ContentChannelOutputStream)", - "protected com.yahoo.jdisc.http.HttpRequest asHttpRequest(com.yahoo.jdisc.Request)" + "protected com.yahoo.jdisc.http.HttpRequest asHttpRequest(com.yahoo.jdisc.Request)", + "public static com.yahoo.container.jdisc.ThreadedHttpRequestHandler$Context testContext()" ], "fields": [ "public static final java.lang.String CONTENT_TYPE", @@ -1990,11 +2008,14 @@ "public com.yahoo.jdisc.http.ServerConfig$Metric$Builder monitoringHandlerPaths(java.util.Collection)", "public com.yahoo.jdisc.http.ServerConfig$Metric$Builder searchHandlerPaths(java.lang.String)", "public com.yahoo.jdisc.http.ServerConfig$Metric$Builder searchHandlerPaths(java.util.Collection)", + "public com.yahoo.jdisc.http.ServerConfig$Metric$Builder ignoredUserAgents(java.lang.String)", + "public com.yahoo.jdisc.http.ServerConfig$Metric$Builder ignoredUserAgents(java.util.Collection)", "public com.yahoo.jdisc.http.ServerConfig$Metric build()" ], "fields": [ "public java.util.List monitoringHandlerPaths", - "public java.util.List searchHandlerPaths" + "public java.util.List searchHandlerPaths", + "public java.util.List ignoredUserAgents" ] }, "com.yahoo.jdisc.http.ServerConfig$Metric": { @@ -2009,7 +2030,9 @@ "public java.util.List monitoringHandlerPaths()", "public java.lang.String monitoringHandlerPaths(int)", "public java.util.List searchHandlerPaths()", - "public java.lang.String searchHandlerPaths(int)" + "public java.lang.String searchHandlerPaths(int)", + "public java.util.List ignoredUserAgents()", + "public java.lang.String ignoredUserAgents(int)" ], "fields": [] }, diff --git a/container-core/src/main/java/com/yahoo/container/jdisc/LoggingRequestHandler.java b/container-core/src/main/java/com/yahoo/container/jdisc/LoggingRequestHandler.java index 717bde1eb5e..b6e370dd911 100644 --- a/container-core/src/main/java/com/yahoo/container/jdisc/LoggingRequestHandler.java +++ b/container-core/src/main/java/com/yahoo/container/jdisc/LoggingRequestHandler.java @@ -29,6 +29,7 @@ import java.util.logging.Level; // TODO Vespa 8: Remove deprecated constructors public abstract class LoggingRequestHandler extends ThreadedHttpRequestHandler { + // TODO: Deprecate public static class Context { final Executor executor; @@ -57,6 +58,7 @@ public abstract class LoggingRequestHandler extends ThreadedHttpRequestHandler { } + // TODO: Deprecate public static Context testOnlyContext() { return new Context(new Executor() { @Override diff --git a/container-core/src/main/java/com/yahoo/container/jdisc/ThreadedHttpRequestHandler.java b/container-core/src/main/java/com/yahoo/container/jdisc/ThreadedHttpRequestHandler.java index 0c3c1e2120b..722f1850013 100644 --- a/container-core/src/main/java/com/yahoo/container/jdisc/ThreadedHttpRequestHandler.java +++ b/container-core/src/main/java/com/yahoo/container/jdisc/ThreadedHttpRequestHandler.java @@ -2,6 +2,7 @@ package com.yahoo.container.jdisc; import com.google.inject.Inject; +import com.yahoo.container.logging.AccessLog; import com.yahoo.jdisc.Metric; import com.yahoo.jdisc.Request; import com.yahoo.jdisc.handler.BufferedContentChannel; @@ -47,6 +48,11 @@ public abstract class ThreadedHttpRequestHandler extends ThreadedRequestHandler this(executor, metric, false); } + // TODO: move Inject annotation here! + public ThreadedHttpRequestHandler(Context context) { + this(context.executor, context.metric); + } + public ThreadedHttpRequestHandler(Executor executor, Metric metric, boolean allowAsyncResponse) { super(executor, metric, allowAsyncResponse); log = Logger.getLogger(this.getClass().getName()); @@ -249,5 +255,36 @@ public abstract class ThreadedHttpRequestHandler extends ThreadedRequestHandler return (com.yahoo.jdisc.http.HttpRequest) request; } + public static Context testContext() { + return new Context(Runnable::run, null); + } + + public static class Context { + + final Executor executor; + final Metric metric; + + /** @deprecated Use {@link #Context(Executor, Metric)} instead */ + @Deprecated(forRemoval = true, since = "7") + public Context(Executor executor, AccessLog ignored, Metric metric) { + this(executor, metric); + } + + @Inject + public Context(Executor executor, Metric metric) { + this.executor = executor; + this.metric = metric; + } + + public Context(Context other) { + this.executor = other.executor; + this.metric = other.metric; + } + + public Executor getExecutor() { return executor; } + @Deprecated(forRemoval = true, since = "7") public AccessLog getAccessLog() { return null; } + public Metric getMetric() { return metric; } + + } } diff --git a/container-core/src/main/java/com/yahoo/jdisc/http/server/jetty/HttpResponseStatisticsCollector.java b/container-core/src/main/java/com/yahoo/jdisc/http/server/jetty/HttpResponseStatisticsCollector.java index 0aa2820f959..631f4080c7e 100644 --- a/container-core/src/main/java/com/yahoo/jdisc/http/server/jetty/HttpResponseStatisticsCollector.java +++ b/container-core/src/main/java/com/yahoo/jdisc/http/server/jetty/HttpResponseStatisticsCollector.java @@ -3,6 +3,7 @@ package com.yahoo.jdisc.http.server.jetty; import com.yahoo.jdisc.Metric; import com.yahoo.jdisc.http.HttpRequest; +import org.eclipse.jetty.http.HttpHeader; import org.eclipse.jetty.http.HttpStatus; import org.eclipse.jetty.server.AsyncContextEvent; import org.eclipse.jetty.server.Handler; @@ -21,6 +22,7 @@ import java.io.IOException; import java.util.ArrayList; import java.util.Collection; import java.util.HashMap; +import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Objects; @@ -106,12 +108,25 @@ class HttpResponseStatisticsCollector extends HandlerWrapper implements Graceful } } - private void observeEndOfRequest(Request request, HttpServletResponse flushableResponse) throws IOException { - var metrics = StatusCodeMetric.of(request, monitoringHandlerPaths, searchHandlerPaths); - metrics.forEach(metric -> - statistics.computeIfAbsent(metric, __ -> new LongAdder()) - .increment()); + void ignoreUserAgent(String agentName) { + ignoredUserAgents.add(agentName); + } + private Set<String> ignoredUserAgents = new HashSet<>(); + + private boolean shouldLogMetricsFor(Request request) { + String agent = request.getHeader(HttpHeader.USER_AGENT.toString()); + if (agent == null) return true; + return ! ignoredUserAgents.contains(agent); + } + + private void observeEndOfRequest(Request request, HttpServletResponse flushableResponse) throws IOException { + if (shouldLogMetricsFor(request)) { + var metrics = StatusCodeMetric.of(request, monitoringHandlerPaths, searchHandlerPaths); + metrics.forEach(metric -> + statistics.computeIfAbsent(metric, __ -> new LongAdder()) + .increment()); + } long live = inFlight.decrementAndGet(); FutureCallback shutdownCb = shutdown.get(); if (shutdownCb != null) { diff --git a/container-core/src/main/java/com/yahoo/jdisc/http/server/jetty/JettyHttpServer.java b/container-core/src/main/java/com/yahoo/jdisc/http/server/jetty/JettyHttpServer.java index fca34f3bbd7..006d6cc84da 100644 --- a/container-core/src/main/java/com/yahoo/jdisc/http/server/jetty/JettyHttpServer.java +++ b/container-core/src/main/java/com/yahoo/jdisc/http/server/jetty/JettyHttpServer.java @@ -159,6 +159,9 @@ public class JettyHttpServer extends AbstractServerProvider { new HttpResponseStatisticsCollector(serverConfig.metric().monitoringHandlerPaths(), serverConfig.metric().searchHandlerPaths()); statisticsCollector.setHandler(gzipHandler); + for (String agent : serverConfig.metric().ignoredUserAgents()) { + statisticsCollector.ignoreUserAgent(agent); + } StatisticsHandler statisticsHandler = newStatisticsHandler(); statisticsHandler.setHandler(statisticsCollector); diff --git a/container-core/src/main/java/com/yahoo/language/provider/DefaultLinguisticsProvider.java b/container-core/src/main/java/com/yahoo/language/provider/DefaultLinguisticsProvider.java index 563c9a8bdff..a38f39559f5 100644 --- a/container-core/src/main/java/com/yahoo/language/provider/DefaultLinguisticsProvider.java +++ b/container-core/src/main/java/com/yahoo/language/provider/DefaultLinguisticsProvider.java @@ -18,10 +18,12 @@ import com.yahoo.language.opennlp.OpenNlpLinguistics; public class DefaultLinguisticsProvider implements Provider<Linguistics> { // Use lazy initialization to avoid expensive (memory-wise) instantiation - private final Supplier<Linguistics> linguisticsSupplier = Suppliers.memoize(OpenNlpLinguistics::new); + private final Supplier<Linguistics> linguisticsSupplier; @Inject - public DefaultLinguisticsProvider() { } + public DefaultLinguisticsProvider() { + linguisticsSupplier = Suppliers.memoize(OpenNlpLinguistics::new); + } @Override public Linguistics get() { return linguisticsSupplier.get(); } diff --git a/container-core/src/main/resources/configdefinitions/jdisc.http.jdisc.http.server.def b/container-core/src/main/resources/configdefinitions/jdisc.http.jdisc.http.server.def index db7322c76f2..7e1404bbc5e 100644 --- a/container-core/src/main/resources/configdefinitions/jdisc.http.jdisc.http.server.def +++ b/container-core/src/main/resources/configdefinitions/jdisc.http.jdisc.http.server.def @@ -57,6 +57,9 @@ metric.monitoringHandlerPaths[] string # Paths that should be reported with search dimensions where applicable metric.searchHandlerPaths[] string +# User-agent names to ignore wrt statistics (crawlers etc) +metric.ignoredUserAgents[] string + # HTTP request headers that contain remote address accessLog.remoteAddressHeaders[] string diff --git a/container-core/src/test/java/com/yahoo/container/di/ConfigRetrieverTest.java b/container-core/src/test/java/com/yahoo/container/di/ConfigRetrieverTest.java index 99ea6a4ee58..695efbf7c17 100644 --- a/container-core/src/test/java/com/yahoo/container/di/ConfigRetrieverTest.java +++ b/container-core/src/test/java/com/yahoo/container/di/ConfigRetrieverTest.java @@ -9,21 +9,17 @@ import com.yahoo.container.di.ConfigRetriever.BootstrapConfigs; import com.yahoo.container.di.ConfigRetriever.ComponentsConfigs; import com.yahoo.container.di.ConfigRetriever.ConfigSnapshot; import com.yahoo.vespa.config.ConfigKey; -import org.hamcrest.Matchers; import org.junit.After; import org.junit.Before; import org.junit.Ignore; -import org.junit.Rule; import org.junit.Test; -import org.junit.rules.ExpectedException; import java.util.Collections; import java.util.HashSet; import java.util.Set; -import static org.hamcrest.CoreMatchers.instanceOf; -import static org.hamcrest.CoreMatchers.is; -import static org.junit.Assert.assertThat; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; /** @@ -52,7 +48,7 @@ public class ConfigRetrieverTest { ConfigRetriever retriever = createConfigRetriever(); ConfigSnapshot bootstrapConfigs = retriever.getConfigs(Collections.emptySet(), 0, true); - assertThat(bootstrapConfigs, Matchers.instanceOf(BootstrapConfigs.class)); + assertTrue(bootstrapConfigs instanceof BootstrapConfigs); } @Test @@ -66,19 +62,15 @@ public class ConfigRetrieverTest { ConfigSnapshot componentsConfigs = retriever.getConfigs(Collections.singleton(testConfigKey), 0, true); if (componentsConfigs instanceof ComponentsConfigs) { - assertThat(componentsConfigs.size(), is(3)); + assertEquals(3, componentsConfigs.size()); } else { fail("ComponentsConfigs has unexpected type: " + componentsConfigs); } } - @Rule - public ExpectedException expectedException = ExpectedException.none(); - @Ignore @SuppressWarnings("unused") public void require_exception_upon_modified_components_keys_without_bootstrap() { - expectedException.expect(IllegalArgumentException.class); writeConfigs(); ConfigRetriever retriever = createConfigRetriever(); @@ -88,15 +80,20 @@ public class ConfigRetrieverTest { Set<ConfigKey<? extends ConfigInstance>> keys = new HashSet<>(); keys.add(testConfigKey); keys.add(new ConfigKey<>(TestConfig.class, "")); - retriever.getConfigs(keys, 0, true); + try { + retriever.getConfigs(keys, 0, true); + fail(); + } catch (IllegalArgumentException e) { + assertEquals("", e.getMessage()); + } } @Test public void require_that_empty_components_keys_after_bootstrap_returns_components_configs() { writeConfigs(); ConfigRetriever retriever = createConfigRetriever(); - assertThat(retriever.getConfigs(Collections.emptySet(), 0, true), instanceOf(BootstrapConfigs.class)); - assertThat(retriever.getConfigs(Collections.emptySet(), 0, true), instanceOf(ComponentsConfigs.class)); + assertTrue(retriever.getConfigs(Collections.emptySet(), 0, true) instanceof BootstrapConfigs); + assertTrue(retriever.getConfigs(Collections.emptySet(), 0, true) instanceof ComponentsConfigs); } public void writeConfigs() { diff --git a/container-core/src/test/java/com/yahoo/container/di/componentgraph/core/ComponentGraphTest.java b/container-core/src/test/java/com/yahoo/container/di/componentgraph/core/ComponentGraphTest.java index 1ded443a3eb..4d44281658c 100644 --- a/container-core/src/test/java/com/yahoo/container/di/componentgraph/core/ComponentGraphTest.java +++ b/container-core/src/test/java/com/yahoo/container/di/componentgraph/core/ComponentGraphTest.java @@ -30,16 +30,11 @@ import java.util.concurrent.Executors; import java.util.function.Supplier; import static com.yahoo.container.di.componentgraph.core.ComponentGraph.isBindingAnnotation; -import static org.hamcrest.CoreMatchers.containsString; -import static org.hamcrest.CoreMatchers.equalTo; -import static org.hamcrest.CoreMatchers.is; -import static org.hamcrest.CoreMatchers.not; -import static org.hamcrest.CoreMatchers.notNullValue; -import static org.hamcrest.CoreMatchers.sameInstance; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertNotNull; -import static org.junit.Assert.assertThat; +import static org.junit.Assert.assertNotSame; +import static org.junit.Assert.assertSame; import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; @@ -80,7 +75,7 @@ public class ComponentGraphTest { ComponentTakingConfig instance = componentGraph.getInstance(ComponentTakingConfig.class); assertNotNull(instance); - assertThat(instance.config.stringVal(), is("test-value")); + assertEquals("test-value", instance.config.stringVal()); } @Test @@ -154,7 +149,7 @@ public class ComponentGraphTest { componentGraph.complete(); ComponentTakingAllSimpleComponents instance = componentGraph.getInstance(ComponentTakingAllSimpleComponents.class); - assertThat(instance.simpleComponents.allComponents().size(), is(3)); + assertEquals(3, instance.simpleComponents.allComponents().size()); } @Test @@ -164,7 +159,7 @@ public class ComponentGraphTest { componentGraph.complete(); ComponentTakingAllSimpleComponents instance = componentGraph.getInstance(ComponentTakingAllSimpleComponents.class); - assertThat(instance.simpleComponents.allComponents().size(), is(0)); + assertTrue(instance.simpleComponents.allComponents().isEmpty()); } @Test @@ -177,7 +172,7 @@ public class ComponentGraphTest { ComponentTakingAllSimpleComponentsUpperBound instance = componentGraph .getInstance(ComponentTakingAllSimpleComponentsUpperBound.class); - assertThat(instance.simpleComponents.allComponents().size(), is(2)); + assertEquals(2, instance.simpleComponents.allComponents().size()); } @Test(expected = RuntimeException.class) @@ -202,7 +197,7 @@ public class ComponentGraphTest { SimpleComponent instance1 = componentGraph.getInstance(SimpleComponent.class); SimpleComponent instance2 = componentGraph.getInstance(SimpleComponent.class); - assertThat(instance1, sameInstance(instance2)); + assertSame(instance1, instance2); } @Test @@ -218,7 +213,7 @@ public class ComponentGraphTest { ComponentTakingComponent instance = componentGraph.getInstance(ComponentTakingComponent.class); ComponentTakingConfig injected = (ComponentTakingConfig) instance.injectedComponent; - assertThat(injected.config.stringVal(), is("test-value")); + assertEquals("test-value", injected.config.stringVal()); } @Test(expected = RuntimeException.class) @@ -248,11 +243,11 @@ public class ComponentGraphTest { componentGraph.complete(); Set<ConfigKey<? extends ConfigInstance>> configKeys = componentGraph.configKeys(); - assertThat(configKeys.size(), is(2)); + assertEquals(2, configKeys.size()); configKeys.forEach(key -> { - assertThat(key.getConfigClass(), equalTo(TestConfig.class)); - assertThat(key.getConfigId(), containsString("component")); + assertEquals(key.getConfigClass(), TestConfig.class); + assertTrue(key.getConfigId().contains("component")); }); } @@ -362,8 +357,8 @@ public class ComponentGraphTest { componentGraph.complete(); fail("Cycle exception expected."); } catch (Throwable e) { - assertThat(e.getMessage(), containsString("cycle")); - assertThat(e.getMessage(), containsString("ComponentCausingCycle")); + assertTrue(e.getMessage().contains("cycle")); + assertTrue(e.getMessage().contains("ComponentCausingCycle")); } } @@ -411,8 +406,8 @@ public class ComponentGraphTest { Executor executorA = graph.getSecond().getFirst(); Executor executorB = graph.getSecond().getSecond(); - assertThat(graphSize, is(4)); - assertThat(executorA, not(sameInstance(executorB))); + assertEquals(4, graphSize); + assertNotSame(executorA, executorB); } @Test @@ -423,8 +418,8 @@ public class ComponentGraphTest { Executor executorA = graph.getSecond().getFirst(); Executor executorB = graph.getSecond().getSecond(); - assertThat(graphSize, is(3)); - assertThat(executorA, sameInstance(executorB)); + assertEquals(3, graphSize); + assertSame(executorA, executorB); } private Pair<Integer, Pair<Executor, Executor>> buildGraphWithChildInjector(Supplier<Executor> executorProvider) { @@ -459,7 +454,7 @@ public class ComponentGraphTest { newGraph.reuseNodes(oldGraph); Executor newExecutor = newGraph.getInstance(Executor.class); - assertThat(executor, sameInstance(newExecutor)); + assertSame(executor, newExecutor); } private ComponentGraph createReusingGraph() { @@ -478,7 +473,7 @@ public class ComponentGraphTest { componentGraph.add(mockComponentNodeWithId(ComponentTakingComponentId.class, componentId)); componentGraph.complete(); - assertThat(componentGraph.getInstance(ComponentTakingComponentId.class).componentId, is(ComponentId.fromString(componentId))); + assertEquals(ComponentId.fromString(componentId), componentGraph.getInstance(ComponentTakingComponentId.class).componentId); } @@ -512,7 +507,7 @@ public class ComponentGraphTest { private final TestConfig config; public ComponentTakingConfig(TestConfig config) { - assertThat(config, notNullValue()); + assertNotNull(config); this.config = config; } } @@ -521,7 +516,7 @@ public class ComponentGraphTest { private final SimpleComponent injectedComponent; public ComponentTakingComponent(SimpleComponent injectedComponent) { - assertThat(injectedComponent, notNullValue()); + assertNotNull(injectedComponent); this.injectedComponent = injectedComponent; } } @@ -530,7 +525,7 @@ public class ComponentGraphTest { private final ComponentTakingComponent injectedComponent; public ComponentTakingComponentTakingComponent(ComponentTakingComponent injectedComponent) { - assertThat(injectedComponent, notNullValue()); + assertNotNull(injectedComponent); this.injectedComponent = injectedComponent; } } @@ -541,8 +536,8 @@ public class ComponentGraphTest { private final SimpleComponent simpleComponent; public ComponentTakingConfigAndComponent(TestConfig config, SimpleComponent injectedComponent) { - assertThat(config, notNullValue()); - assertThat(injectedComponent, notNullValue()); + assertNotNull(config); + assertNotNull(injectedComponent); this.config = config; this.simpleComponent = injectedComponent; } @@ -552,7 +547,7 @@ public class ComponentGraphTest { public final ComponentRegistry<SimpleComponent> simpleComponents; public ComponentTakingAllSimpleComponents(ComponentRegistry<SimpleComponent> simpleComponents) { - assertThat(simpleComponents, notNullValue()); + assertNotNull(simpleComponents); this.simpleComponents = simpleComponents; } } @@ -561,20 +556,20 @@ public class ComponentGraphTest { private final ComponentRegistry<? extends SimpleComponent> simpleComponents; public ComponentTakingAllSimpleComponentsUpperBound(ComponentRegistry<? extends SimpleComponent> simpleComponents) { - assertThat(simpleComponents, notNullValue()); + assertNotNull(simpleComponents); this.simpleComponents = simpleComponents; } } public static class ComponentTakingAllComponentsWithTypeVariable<COMPONENT extends AbstractComponent> extends AbstractComponent { public ComponentTakingAllComponentsWithTypeVariable(ComponentRegistry<COMPONENT> simpleComponents) { - assertThat(simpleComponents, notNullValue()); + assertNotNull(simpleComponents); } } public static class ComponentTakingNamedComponent extends AbstractComponent { public ComponentTakingNamedComponent(@Named("named-test") SimpleComponent injectedComponent) { - assertThat(injectedComponent, notNullValue()); + assertNotNull(injectedComponent); } } @@ -633,7 +628,7 @@ public class ComponentGraphTest { private final Executor executor; public ComponentTakingExecutor(Executor executor) { - assertThat(executor, notNullValue()); + assertNotNull(executor); this.executor = executor; } } diff --git a/container-core/src/test/java/com/yahoo/container/di/componentgraph/core/FallbackToGuiceInjectorTest.java b/container-core/src/test/java/com/yahoo/container/di/componentgraph/core/FallbackToGuiceInjectorTest.java index 188e04d8497..1e3d67ed463 100644 --- a/container-core/src/test/java/com/yahoo/container/di/componentgraph/core/FallbackToGuiceInjectorTest.java +++ b/container-core/src/test/java/com/yahoo/container/di/componentgraph/core/FallbackToGuiceInjectorTest.java @@ -10,23 +10,19 @@ import com.google.inject.name.Names; import com.yahoo.component.AbstractComponent; import com.yahoo.component.ComponentId; import com.yahoo.config.ConfigInstance; -import com.yahoo.container.di.componentgraph.core.ComponentGraph; -import com.yahoo.container.di.componentgraph.core.ComponentNode; -import com.yahoo.container.di.componentgraph.core.Node; import com.yahoo.vespa.config.ConfigKey; import org.junit.Before; -import org.junit.Rule; import org.junit.Test; -import org.junit.rules.ExpectedException; import java.util.HashMap; import java.util.Map; import java.util.concurrent.Executor; import java.util.concurrent.Executors; -import static org.hamcrest.CoreMatchers.is; -import static org.junit.Assert.assertThat; +import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; /** * @author Tony Vaagenes @@ -37,11 +33,8 @@ public class FallbackToGuiceInjectorTest { private ComponentGraph componentGraph; private Injector injector; - private Map<ConfigKey<? extends ConfigInstance>, ConfigInstance> configs = - new HashMap<>(); + private final Map<ConfigKey<? extends ConfigInstance>, ConfigInstance> configs = new HashMap<>(); - @Rule - public final ExpectedException exception = ExpectedException.none(); @Before public void createGraph() { @@ -91,7 +84,7 @@ public class FallbackToGuiceInjectorTest { complete(); MyComponent component = getInstance(MyComponent.class); - assertThat(component.url, is("http://yahoo.com")); + assertEquals("http://yahoo.com", component.url); assertNotNull(component.executor); } @@ -102,17 +95,19 @@ public class FallbackToGuiceInjectorTest { complete(); ComponentTakingDefaultString component = getInstance(ComponentTakingDefaultString.class); - assertThat(component.injectedString, is("")); + assertTrue(component.injectedString.isEmpty()); } @Test public void guice_injector_fails_when_no_explicit_binding_exists_and_class_has_no_default_ctor() { setInjector(emptyGuiceInjector()); register(ComponentThatCannotBeConstructed.class); - - exception.expect(RuntimeException.class); - exception.expectMessage("When resolving dependencies of 'com.yahoo.container.di.componentgraph.core.FallbackToGuiceInjectorTest$ComponentThatCannotBeConstructed'"); - complete(); + try { + complete(); + fail(); + } catch (RuntimeException e) { + assertEquals("When resolving dependencies of 'com.yahoo.container.di.componentgraph.core.FallbackToGuiceInjectorTest$ComponentThatCannotBeConstructed'", e.getMessage()); + } } public void register(Class<?> componentClass) { @@ -123,9 +118,8 @@ public class FallbackToGuiceInjectorTest { return ComponentId.fromString(componentClass.getName()); } - @SuppressWarnings("unchecked") private Node mockComponentNode(Class<?> componentClass) { - return new ComponentNode(toId(componentClass), toId(componentClass).toString(), (Class<Object>)componentClass, null); + return new ComponentNode(toId(componentClass), toId(componentClass).toString(), componentClass, null); } public <T> T getInstance(Class<T> componentClass) { diff --git a/container-core/src/test/java/com/yahoo/container/di/componentgraph/core/ReuseComponentsTest.java b/container-core/src/test/java/com/yahoo/container/di/componentgraph/core/ReuseComponentsTest.java index 537140d6f3f..29452f7babe 100644 --- a/container-core/src/test/java/com/yahoo/container/di/componentgraph/core/ReuseComponentsTest.java +++ b/container-core/src/test/java/com/yahoo/container/di/componentgraph/core/ReuseComponentsTest.java @@ -21,11 +21,9 @@ import java.util.concurrent.Executor; import java.util.function.Function; import java.util.function.Supplier; -import static org.hamcrest.CoreMatchers.equalTo; -import static org.hamcrest.CoreMatchers.is; -import static org.hamcrest.CoreMatchers.not; -import static org.hamcrest.CoreMatchers.sameInstance; -import static org.junit.Assert.assertThat; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotSame; +import static org.junit.Assert.assertSame; /** * @author gjoranv @@ -47,7 +45,7 @@ public class ReuseComponentsTest { newGraph.reuseNodes(graph); T instance2 = getComponent(newGraph, classToLookup); - assertThat(instance2, sameInstance(instance)); + assertSame(instance2, instance); } @Test(expected = IllegalStateException.class) @@ -59,7 +57,7 @@ public class ReuseComponentsTest { newGraph.reuseNodes(graph); SimpleComponent2 instance2 = getComponent(newGraph, SimpleComponent2.class); - assertThat(instance2.getId(), is(instance.getId())); + assertEquals(instance2.getId(),instance.getId()); @SuppressWarnings("unused") SimpleComponent throwsException = getComponent(newGraph, SimpleComponent.class); } @@ -79,7 +77,7 @@ public class ReuseComponentsTest { newGraph.reuseNodes(graph); ComponentTakingConfig instance2 = getComponent(newGraph, componentClass); - assertThat(instance2, not(sameInstance(instance))); + assertNotSame(instance2, instance); } @Test @@ -111,7 +109,7 @@ public class ReuseComponentsTest { newGraph.reuseNodes(oldGraph); ComponentTakingComponent newInstance = getComponent(newGraph, ComponentTakingComponent.class); - assertThat(newInstance, not(sameInstance(oldInstance))); + assertNotSame(newInstance, oldInstance); } @Test @@ -143,7 +141,7 @@ public class ReuseComponentsTest { newGraph.reuseNodes(oldGraph); ComponentRegistry<SimpleComponent> newSimpleComponentRegistry = getComponent(newGraph, ComponentTakingAllSimpleComponents.class).simpleComponents; - assertThat(newSimpleComponentRegistry, not(sameInstance(oldSimpleComponentRegistry))); + assertNotSame(newSimpleComponentRegistry, oldSimpleComponentRegistry); } @Test @@ -177,8 +175,8 @@ public class ReuseComponentsTest { SimpleComponent newInjectedComponent = getComponent(newGraph, SimpleComponent.class); ComponentTakingConfigAndComponent newDependentComponent = getComponent(newGraph, ComponentTakingConfigAndComponent.class); - assertThat(newDependentComponent, not(sameInstance(oldDependentComponent))); - assertThat(newInjectedComponent, sameInstance(oldInjectedComponent)); + assertNotSame(newDependentComponent, oldDependentComponent); + assertSame(newInjectedComponent, oldInjectedComponent); } @Test @@ -197,7 +195,7 @@ public class ReuseComponentsTest { componentRetriever.apply(oldGraph); // Ensure creation of GuiceNode ComponentGraph newGraph = makeGraph.get(); newGraph.reuseNodes(oldGraph); - assertThat(componentRetriever.apply(oldGraph), sameInstance(componentRetriever.apply(newGraph))); + assertSame(componentRetriever.apply(oldGraph), componentRetriever.apply(newGraph)); } @Test @@ -218,7 +216,7 @@ public class ReuseComponentsTest { Node targetNode1 = createNodeWithInjectedNodeWithInjectedNode.apply("indirectlyInjected_1"); Node targetNode2 = createNodeWithInjectedNodeWithInjectedNode.apply("indirectlyInjected_2"); - assertThat(targetNode1, equalTo(targetNode2)); + assertEquals(targetNode1, targetNode2); } private void completeNode(ComponentNode node) { diff --git a/container-core/src/test/java/com/yahoo/container/di/componentgraph/cycle/CycleFinderTest.java b/container-core/src/test/java/com/yahoo/container/di/componentgraph/cycle/CycleFinderTest.java index d1d3914a5ff..65fbdc578aa 100644 --- a/container-core/src/test/java/com/yahoo/container/di/componentgraph/cycle/CycleFinderTest.java +++ b/container-core/src/test/java/com/yahoo/container/di/componentgraph/cycle/CycleFinderTest.java @@ -4,13 +4,13 @@ package com.yahoo.container.di.componentgraph.cycle; import org.junit.Test; +import java.util.List; + import static com.yahoo.container.di.componentgraph.cycle.CycleFinderTest.Vertices.A; import static com.yahoo.container.di.componentgraph.cycle.CycleFinderTest.Vertices.B; import static com.yahoo.container.di.componentgraph.cycle.CycleFinderTest.Vertices.C; import static com.yahoo.container.di.componentgraph.cycle.CycleFinderTest.Vertices.D; -import static org.hamcrest.Matchers.contains; -import static org.hamcrest.Matchers.empty; -import static org.junit.Assert.assertThat; +import static org.junit.Assert.assertTrue; /** * @author gjoranv @@ -28,7 +28,7 @@ public class CycleFinderTest { graph.edge(D, A); var cycleFinder = new CycleFinder<>(graph); - assertThat(cycleFinder.findCycle(), empty()); + assertTrue(cycleFinder.findCycle().isEmpty()); } @Test @@ -39,7 +39,7 @@ public class CycleFinderTest { graph.edge(C, A); var cycleFinder = new CycleFinder<>(graph); - assertThat(cycleFinder.findCycle(), contains(A, B, C, A)); + assertTrue(cycleFinder.findCycle().containsAll(List.of(A, B, C, A))); } @Test @@ -48,7 +48,7 @@ public class CycleFinderTest { graph.edge(A, A); var cycleFinder = new CycleFinder<>(graph); - assertThat(cycleFinder.findCycle(), contains(A, A)); + assertTrue(cycleFinder.findCycle().containsAll(List.of(A, A))); } @Test @@ -59,7 +59,7 @@ public class CycleFinderTest { graph.edge(C, B); var cycleFinder = new CycleFinder<>(graph); - assertThat(cycleFinder.findCycle(), contains(B, C, B)); + assertTrue(cycleFinder.findCycle().containsAll(List.of(B, C, B))); } @Test @@ -68,8 +68,8 @@ public class CycleFinderTest { graph.edge(A, A); var cycleFinder = new CycleFinder<>(graph); - assertThat(cycleFinder.findCycle(), contains(A, A)); - assertThat(cycleFinder.findCycle(), contains(A, A)); + assertTrue(cycleFinder.findCycle().containsAll(List.of(A, A))); + assertTrue(cycleFinder.findCycle().containsAll(List.of(A, A))); } @Test @@ -78,8 +78,8 @@ public class CycleFinderTest { graph.edge(A, B); var cycleFinder = new CycleFinder<>(graph); - assertThat(cycleFinder.findCycle(), empty()); - assertThat(cycleFinder.findCycle(), empty()); + assertTrue(cycleFinder.findCycle().isEmpty()); + assertTrue(cycleFinder.findCycle().isEmpty()); } } diff --git a/container-core/src/test/java/com/yahoo/container/di/componentgraph/cycle/GraphTest.java b/container-core/src/test/java/com/yahoo/container/di/componentgraph/cycle/GraphTest.java index c5c6e97180c..526e683ad46 100644 --- a/container-core/src/test/java/com/yahoo/container/di/componentgraph/cycle/GraphTest.java +++ b/container-core/src/test/java/com/yahoo/container/di/componentgraph/cycle/GraphTest.java @@ -2,18 +2,16 @@ package com.yahoo.container.di.componentgraph.cycle; -import org.junit.Rule; import org.junit.Test; -import org.junit.rules.ExpectedException; + +import java.util.List; import static com.yahoo.container.di.componentgraph.cycle.GraphTest.Vertices.A; import static com.yahoo.container.di.componentgraph.cycle.GraphTest.Vertices.B; import static com.yahoo.container.di.componentgraph.cycle.GraphTest.Vertices.C; -import static org.hamcrest.CoreMatchers.is; -import static org.hamcrest.Matchers.contains; -import static org.hamcrest.Matchers.containsInAnyOrder; -import static org.hamcrest.Matchers.empty; -import static org.junit.Assert.assertThat; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; /** * @author gjoranv @@ -22,9 +20,6 @@ public class GraphTest { enum Vertices {A, B, C} - @Rule - public ExpectedException expectedException = ExpectedException.none(); - @Test public void vertices_and_edges_are_added_and_can_be_retrieved() { var graph = new Graph<Vertices>(); @@ -32,19 +27,22 @@ public class GraphTest { graph.edge(B, C); graph.edge(A, C); - assertThat(graph.getVertices().size(), is(3)); - assertThat(graph.getAdjacent(A), containsInAnyOrder(B, C)); - assertThat(graph.getAdjacent(B), containsInAnyOrder(C)); - assertThat(graph.getAdjacent(C), empty()); + assertEquals(3, graph.getVertices().size()); + assertTrue(graph.getAdjacent(A).containsAll(List.of(B, C))); + assertTrue(graph.getAdjacent(B).contains(C)); + assertTrue(graph.getAdjacent(C).isEmpty()); } @Test public void null_vertices_are_not_allowed() { var graph = new Graph<Vertices>(); - expectedException.expect(IllegalArgumentException.class); - expectedException.expectMessage("Null vertices are not allowed"); - graph.edge(A, null); + try { + graph.edge(A, null); + fail(); + } catch (IllegalArgumentException e) { + assertEquals("Null vertices are not allowed, edge: A->null", e.getMessage()); + } } @Test @@ -53,7 +51,7 @@ public class GraphTest { graph.edge(A, B); graph.edge(A, B); - assertThat(graph.getAdjacent(A).size(), is(1)); + assertEquals(1, graph.getAdjacent(A).size()); } @Test @@ -61,7 +59,7 @@ public class GraphTest { var graph = new Graph<Vertices>(); graph.edge(A, A); - assertThat(graph.getAdjacent(A), contains(A)); + assertTrue(graph.getAdjacent(A).contains(A)); } } diff --git a/container-core/src/test/java/com/yahoo/container/handler/AccessLogRequestHandlerTest.java b/container-core/src/test/java/com/yahoo/container/handler/AccessLogRequestHandlerTest.java index aad064bc82e..2eba4144b45 100644 --- a/container-core/src/test/java/com/yahoo/container/handler/AccessLogRequestHandlerTest.java +++ b/container-core/src/test/java/com/yahoo/container/handler/AccessLogRequestHandlerTest.java @@ -9,8 +9,7 @@ import java.io.ByteArrayOutputStream; import java.io.IOException; import java.util.concurrent.Executor; -import static org.hamcrest.CoreMatchers.is; -import static org.junit.Assert.assertThat; +import static org.junit.Assert.assertEquals; import static org.mockito.Mockito.mock; public class AccessLogRequestHandlerTest { @@ -25,14 +24,14 @@ public class AccessLogRequestHandlerTest { keeper.addUri("foo"); HttpResponse response = handler.handle(null); response.render(out); - assertThat(out.toString(), is("{\"entries\":[{\"url\":\"foo\"}]}")); + assertEquals("{\"entries\":[{\"url\":\"foo\"}]}", out.toString()); } @Test public void testEmpty() throws IOException { HttpResponse response = handler.handle(null); response.render(out); - assertThat(out.toString(), is("{\"entries\":[]}")); + assertEquals("{\"entries\":[]}", out.toString()); } @Test @@ -41,7 +40,7 @@ public class AccessLogRequestHandlerTest { keeper.addUri("foo"); HttpResponse response = handler.handle(null); response.render(out); - assertThat(out.toString(), is("{\"entries\":[{\"url\":\"foo\"},{\"url\":\"foo\"}]}")); + assertEquals("{\"entries\":[{\"url\":\"foo\"},{\"url\":\"foo\"}]}", out.toString()); } } diff --git a/container-core/src/test/java/com/yahoo/container/handler/test/MockServiceTest.java b/container-core/src/test/java/com/yahoo/container/handler/test/MockServiceTest.java index 604fe9494d7..2efa717800d 100644 --- a/container-core/src/test/java/com/yahoo/container/handler/test/MockServiceTest.java +++ b/container-core/src/test/java/com/yahoo/container/handler/test/MockServiceTest.java @@ -12,8 +12,7 @@ import java.io.FileNotFoundException; import java.io.IOException; import java.util.concurrent.Executor; -import static org.hamcrest.core.Is.is; -import static org.junit.Assert.assertThat; +import static org.junit.Assert.assertEquals; /** * @author Ulf Lilleengen @@ -37,7 +36,7 @@ public class MockServiceTest { @Test public void testNoHandlerFound() throws InterruptedException, IOException { HttpResponse response = runHandler(com.yahoo.jdisc.http.HttpRequest.Method.DELETE, "/foo/bar"); - assertThat(response.getStatus(), is(404)); + assertEquals(404, response.getStatus()); assertResponseContents(response, "DELETE:/foo/bar was not found"); } @@ -52,19 +51,14 @@ public class MockServiceTest { } private void assertResponse(HttpResponse response, int expectedCode, String expectedMessage) throws IOException { - assertThat(response.getStatus(), is(expectedCode)); + assertEquals(expectedCode, response.getStatus()); assertResponseContents(response, expectedMessage); } private void assertResponseContents(HttpResponse response, String expected) throws IOException { ByteArrayOutputStream baos = new ByteArrayOutputStream(); response.render(baos); - assertThat(baos.toString(), is(expected)); - } - - private void assertResponseOk(HttpResponse response) { - assertThat(response.getStatus(), is(200)); - assertThat(response.getContentType(), is("text/plain")); + assertEquals(expected, baos.toString()); } private HttpResponse runHandler(com.yahoo.jdisc.http.HttpRequest.Method method, String path) throws InterruptedException, IOException { diff --git a/container-core/src/test/java/com/yahoo/container/logging/CircularArrayAccessLogKeeperTest.java b/container-core/src/test/java/com/yahoo/container/logging/CircularArrayAccessLogKeeperTest.java index c69e293750c..451e6dc42bb 100644 --- a/container-core/src/test/java/com/yahoo/container/logging/CircularArrayAccessLogKeeperTest.java +++ b/container-core/src/test/java/com/yahoo/container/logging/CircularArrayAccessLogKeeperTest.java @@ -8,7 +8,7 @@ import static org.hamcrest.Matchers.contains; import static org.hamcrest.core.Is.is; import static org.hamcrest.core.IsCollectionContaining.hasItem; import static org.hamcrest.core.IsNot.not; -import static org.junit.Assert.assertThat; +import static org.hamcrest.MatcherAssert.assertThat; public class CircularArrayAccessLogKeeperTest { private CircularArrayAccessLogKeeper circularArrayAccessLogKeeper = new CircularArrayAccessLogKeeper(); diff --git a/container-core/src/test/java/com/yahoo/osgi/provider/model/ComponentModelTest.java b/container-core/src/test/java/com/yahoo/osgi/provider/model/ComponentModelTest.java index 2b1a8a42149..4433b78c807 100644 --- a/container-core/src/test/java/com/yahoo/osgi/provider/model/ComponentModelTest.java +++ b/container-core/src/test/java/com/yahoo/osgi/provider/model/ComponentModelTest.java @@ -4,8 +4,7 @@ package com.yahoo.osgi.provider.model; import com.yahoo.container.bundle.BundleInstantiationSpecification; import org.junit.Test; -import static org.hamcrest.CoreMatchers.is; -import static org.junit.Assert.assertThat; +import static org.junit.Assert.assertEquals; /** * @author gjoranv @@ -13,7 +12,7 @@ import static org.junit.Assert.assertThat; public class ComponentModelTest { @Test - public void create_from_instantiation_spec() throws Exception { + public void create_from_instantiation_spec() { ComponentModel model = new ComponentModel( BundleInstantiationSpecification.getFromStrings("id", "class", "bundle")); verifyBundleSpec(model); @@ -29,19 +28,19 @@ public class ComponentModelTest { ComponentModel model = new ComponentModel( BundleInstantiationSpecification.getFromStrings("id", "class", "bundle"), "configId"); verifyBundleSpec(model); - assertThat(model.configId, is("configId")); + assertEquals("configId", model.configId); } @Test public void create_from_strings() throws Exception { ComponentModel model = new ComponentModel("id", "class", "bundle", "configId"); verifyBundleSpec(model); - assertThat(model.configId, is("configId")); + assertEquals("configId", model.configId); } private void verifyBundleSpec(ComponentModel model) { - assertThat(model.getComponentId().stringValue(), is("id")); - assertThat(model.getClassId().stringValue(), is("class")); - assertThat(model.bundleInstantiationSpec.bundle.stringValue(), is("bundle")); + assertEquals("id", model.getComponentId().stringValue()); + assertEquals("class", model.getClassId().stringValue()); + assertEquals("bundle", model.bundleInstantiationSpec.bundle.stringValue()); } } diff --git a/container-core/src/test/java/com/yahoo/processing/handler/ProcessingHandlerTestCase.java b/container-core/src/test/java/com/yahoo/processing/handler/ProcessingHandlerTestCase.java index eddaf91986d..3ff3f6e2d27 100644 --- a/container-core/src/test/java/com/yahoo/processing/handler/ProcessingHandlerTestCase.java +++ b/container-core/src/test/java/com/yahoo/processing/handler/ProcessingHandlerTestCase.java @@ -30,7 +30,6 @@ import java.io.InputStream; import java.net.URI; import java.nio.ByteBuffer; import java.util.ArrayList; -import java.util.Arrays; import java.util.Collections; import java.util.ConcurrentModificationException; import java.util.HashMap; @@ -40,13 +39,11 @@ import java.util.concurrent.ExecutionException; import static com.yahoo.jdisc.http.server.jetty.AccessLoggingRequestHandler.CONTEXT_KEY_ACCESS_LOG_ENTRY; import static com.yahoo.processing.test.ProcessorLibrary.MapData; -import static org.hamcrest.CoreMatchers.is; -import static org.hamcrest.CoreMatchers.notNullValue; -import static org.hamcrest.CoreMatchers.sameInstance; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertNull; -import static org.junit.Assert.assertThat; +import static org.junit.Assert.assertSame; import static org.junit.Assert.assertTrue; /** @@ -64,17 +61,17 @@ public class ProcessingHandlerTestCase { private ProcessingTestDriver driver; private final Chain<Processor> defaultChain = - new Chain<Processor>("default", + new Chain<>("default", new ProcessorLibrary.StringDataListAdder("Item1", "Item2"), new ProcessorLibrary.Trace("TraceMessage", 1), new ProcessorLibrary.StringDataAdder("StringData.toString()")); private final Chain<Processor> simpleChain = - new Chain<Processor>("simple", + new Chain<>("simple", new ProcessorLibrary.StringDataAdder("StringData.toString()")); private final Chain<Processor> logValueChain = - new Chain<Processor>("log-value", + new Chain<>("log-value", new ProcessorLibrary.LogValueAdder(LOG_KEY, LOG_VALUE)); @After @@ -94,11 +91,11 @@ public class ProcessingHandlerTestCase { requestContent.close(null); request.release(); responseHandler.readAll(); - assertThat(entry.getKeyValues().get(LOG_KEY), is(List.of(LOG_VALUE))); + assertEquals(List.of(LOG_VALUE), entry.getKeyValues().get(LOG_KEY)); } @Test - public void testProcessingHandlerResolvesChains() throws Exception { + public void testProcessingHandlerResolvesChains() { List<Chain<Processor>> chains = new ArrayList<>(); chains.add(defaultChain); chains.add(simpleChain); @@ -109,16 +106,16 @@ public class ProcessingHandlerTestCase { } @Test - public void testProcessingHandlerPropagatesRequestParametersAndContext() throws InterruptedException { + public void testProcessingHandlerPropagatesRequestParametersAndContext() { List<Chain<Processor>> chains = new ArrayList<>(); - chains.add(new Chain<Processor>("default", new RequestPropertyTracer())); + chains.add(new Chain<>("default", new RequestPropertyTracer())); driver = new ProcessingTestDriver(chains); assertTrue("JDisc request context is propagated to properties()", driver.sendRequest("http://localhost/?chain=default&tracelevel=4").readAll().contains("context.contextVariable: '37'")); } @Test - public void testProcessingHandlerOutputsTrace() throws Exception { + public void testProcessingHandlerOutputsTrace() { List<Chain<Processor>> chains = new ArrayList<>(); chains.add(defaultChain); driver = new ProcessingTestDriver(chains); @@ -131,28 +128,28 @@ public class ProcessingHandlerTestCase { } @Test - public void testProcessingHandlerTransfersErrorsToHttpStatusCodesNoData() throws Exception { + public void testProcessingHandlerTransfersErrorsToHttpStatusCodesNoData() { List<Chain<Processor>> chains = new ArrayList<>(); chains.add(simpleChain); - chains.add(new Chain<Processor>("moved_permanently", new ProcessorLibrary.ErrorAdder(new ErrorMessage(301,"Message")))); - chains.add(new Chain<Processor>("unauthorized", new ProcessorLibrary.ErrorAdder(new ErrorMessage(401,"Message")))); - chains.add(new Chain<Processor>("unauthorized_mapped", new ProcessorLibrary.ErrorAdder(new ErrorMessage(Error.UNAUTHORIZED.code,"Message")))); - chains.add(new Chain<Processor>("forbidden", new ProcessorLibrary.ErrorAdder(new ErrorMessage(403,"Message")))); - chains.add(new Chain<Processor>("forbidden_mapped", new ProcessorLibrary.ErrorAdder(new ErrorMessage(Error.FORBIDDEN.code,"Message")))); - chains.add(new Chain<Processor>("not_found", new ProcessorLibrary.ErrorAdder(new ErrorMessage(404,"Message")))); - chains.add(new Chain<Processor>("not_found_mapped", new ProcessorLibrary.ErrorAdder(new ErrorMessage(Error.NOT_FOUND.code,"Message")))); - chains.add(new Chain<Processor>("too_many_requests", new ProcessorLibrary.ErrorAdder(new ErrorMessage(429,"Message")))); - chains.add(new Chain<Processor>("bad_request", new ProcessorLibrary.ErrorAdder(new ErrorMessage(400,"Message")))); - chains.add(new Chain<Processor>("bad_request_mapped", new ProcessorLibrary.ErrorAdder(new ErrorMessage(Error.BAD_REQUEST.code,"Message")))); - chains.add(new Chain<Processor>("internal_server_error", new ProcessorLibrary.ErrorAdder(new ErrorMessage(500,"Message")))); - chains.add(new Chain<Processor>("internal_server_error_mapped", new ProcessorLibrary.ErrorAdder(new ErrorMessage(Error.INTERNAL_SERVER_ERROR.code,"Message")))); - chains.add(new Chain<Processor>("service_unavailable", new ProcessorLibrary.ErrorAdder(new ErrorMessage(503,"Message")))); - chains.add(new Chain<Processor>("service_unavailable_mapped", new ProcessorLibrary.ErrorAdder(new ErrorMessage(Error.NO_BACKENDS_IN_SERVICE.code,"Message")))); - chains.add(new Chain<Processor>("gateway_timeout", new ProcessorLibrary.ErrorAdder(new ErrorMessage(504,"Message")))); - chains.add(new Chain<Processor>("gateway_timeout_mapped", new ProcessorLibrary.ErrorAdder(new ErrorMessage(Error.TIMEOUT.code,"Message")))); - chains.add(new Chain<Processor>("bad_gateway", new ProcessorLibrary.ErrorAdder(new ErrorMessage(502,"Message")))); - chains.add(new Chain<Processor>("bad_gateway_mapped", new ProcessorLibrary.ErrorAdder(new ErrorMessage(Error.BACKEND_COMMUNICATION_ERROR.code,"Message")))); - chains.add(new Chain<Processor>("unknown_code", new ProcessorLibrary.ErrorAdder(new ErrorMessage(1234567,"Message")))); + chains.add(new Chain<>("moved_permanently", new ProcessorLibrary.ErrorAdder(new ErrorMessage(301,"Message")))); + chains.add(new Chain<>("unauthorized", new ProcessorLibrary.ErrorAdder(new ErrorMessage(401,"Message")))); + chains.add(new Chain<>("unauthorized_mapped", new ProcessorLibrary.ErrorAdder(new ErrorMessage(Error.UNAUTHORIZED.code,"Message")))); + chains.add(new Chain<>("forbidden", new ProcessorLibrary.ErrorAdder(new ErrorMessage(403,"Message")))); + chains.add(new Chain<>("forbidden_mapped", new ProcessorLibrary.ErrorAdder(new ErrorMessage(Error.FORBIDDEN.code,"Message")))); + chains.add(new Chain<>("not_found", new ProcessorLibrary.ErrorAdder(new ErrorMessage(404,"Message")))); + chains.add(new Chain<>("not_found_mapped", new ProcessorLibrary.ErrorAdder(new ErrorMessage(Error.NOT_FOUND.code,"Message")))); + chains.add(new Chain<>("too_many_requests", new ProcessorLibrary.ErrorAdder(new ErrorMessage(429,"Message")))); + chains.add(new Chain<>("bad_request", new ProcessorLibrary.ErrorAdder(new ErrorMessage(400,"Message")))); + chains.add(new Chain<>("bad_request_mapped", new ProcessorLibrary.ErrorAdder(new ErrorMessage(Error.BAD_REQUEST.code,"Message")))); + chains.add(new Chain<>("internal_server_error", new ProcessorLibrary.ErrorAdder(new ErrorMessage(500,"Message")))); + chains.add(new Chain<>("internal_server_error_mapped", new ProcessorLibrary.ErrorAdder(new ErrorMessage(Error.INTERNAL_SERVER_ERROR.code,"Message")))); + chains.add(new Chain<>("service_unavailable", new ProcessorLibrary.ErrorAdder(new ErrorMessage(503,"Message")))); + chains.add(new Chain<>("service_unavailable_mapped", new ProcessorLibrary.ErrorAdder(new ErrorMessage(Error.NO_BACKENDS_IN_SERVICE.code,"Message")))); + chains.add(new Chain<>("gateway_timeout", new ProcessorLibrary.ErrorAdder(new ErrorMessage(504,"Message")))); + chains.add(new Chain<>("gateway_timeout_mapped", new ProcessorLibrary.ErrorAdder(new ErrorMessage(Error.TIMEOUT.code,"Message")))); + chains.add(new Chain<>("bad_gateway", new ProcessorLibrary.ErrorAdder(new ErrorMessage(502,"Message")))); + chains.add(new Chain<>("bad_gateway_mapped", new ProcessorLibrary.ErrorAdder(new ErrorMessage(Error.BACKEND_COMMUNICATION_ERROR.code,"Message")))); + chains.add(new Chain<>("unknown_code", new ProcessorLibrary.ErrorAdder(new ErrorMessage(1234567,"Message")))); driver = new ProcessingTestDriver(chains); assertEqualStatus(200, "http://localhost/?chain=simple"); assertEqualStatus(301, "http://localhost/?chain=moved_permanently"); @@ -177,28 +174,28 @@ public class ProcessingHandlerTestCase { } @Test - public void testProcessingHandlerTransfersErrorsToHttpStatusCodesWithData() throws Exception { + public void testProcessingHandlerTransfersErrorsToHttpStatusCodesWithData() { List<Chain<Processor>> chains = new ArrayList<>(); chains.add(simpleChain); - chains.add(new Chain<Processor>("moved_permanently", new ProcessorLibrary.StringDataAdder("Hello"), new ProcessorLibrary.ErrorAdder(new ErrorMessage(301,"Message")))); - chains.add(new Chain<Processor>("unauthorized", new ProcessorLibrary.StringDataAdder("Hello"), new ProcessorLibrary.ErrorAdder(new ErrorMessage(401,"Message")))); - chains.add(new Chain<Processor>("unauthorized_mapped", new ProcessorLibrary.StringDataAdder("Hello"), new ProcessorLibrary.ErrorAdder(new ErrorMessage(Error.UNAUTHORIZED.code,"Message")))); - chains.add(new Chain<Processor>("forbidden", new ProcessorLibrary.StringDataAdder("Hello"), new ProcessorLibrary.ErrorAdder(new ErrorMessage(403,"Message")))); - chains.add(new Chain<Processor>("forbidden_mapped", new ProcessorLibrary.StringDataAdder("Hello"), new ProcessorLibrary.ErrorAdder(new ErrorMessage(Error.FORBIDDEN.code,"Message")))); - chains.add(new Chain<Processor>("not_found", new ProcessorLibrary.StringDataAdder("Hello"), new ProcessorLibrary.ErrorAdder(new ErrorMessage(404,"Message")))); - chains.add(new Chain<Processor>("not_found_mapped", new ProcessorLibrary.StringDataAdder("Hello"), new ProcessorLibrary.ErrorAdder(new ErrorMessage(Error.NOT_FOUND.code,"Message")))); - chains.add(new Chain<Processor>("too_many_requests", new ProcessorLibrary.StringDataAdder("Hello"), new ProcessorLibrary.ErrorAdder(new ErrorMessage(429,"Message")))); - chains.add(new Chain<Processor>("bad_request", new ProcessorLibrary.StringDataAdder("Hello"), new ProcessorLibrary.ErrorAdder(new ErrorMessage(400,"Message")))); - chains.add(new Chain<Processor>("bad_request_mapped", new ProcessorLibrary.StringDataAdder("Hello"), new ProcessorLibrary.ErrorAdder(new ErrorMessage(Error.BAD_REQUEST.code,"Message")))); - chains.add(new Chain<Processor>("internal_server_error", new ProcessorLibrary.StringDataAdder("Hello"), new ProcessorLibrary.ErrorAdder(new ErrorMessage(500,"Message")))); - chains.add(new Chain<Processor>("internal_server_error_mapped", new ProcessorLibrary.StringDataAdder("Hello"), new ProcessorLibrary.ErrorAdder(new ErrorMessage(Error.INTERNAL_SERVER_ERROR.code,"Message")))); - chains.add(new Chain<Processor>("service_unavailable", new ProcessorLibrary.StringDataAdder("Hello"), new ProcessorLibrary.ErrorAdder(new ErrorMessage(503,"Message")))); - chains.add(new Chain<Processor>("service_unavailable_mapped", new ProcessorLibrary.StringDataAdder("Hello"), new ProcessorLibrary.ErrorAdder(new ErrorMessage(Error.NO_BACKENDS_IN_SERVICE.code,"Message")))); - chains.add(new Chain<Processor>("gateway_timeout", new ProcessorLibrary.StringDataAdder("Hello"), new ProcessorLibrary.ErrorAdder(new ErrorMessage(504,"Message")))); - chains.add(new Chain<Processor>("gateway_timeout_mapped", new ProcessorLibrary.StringDataAdder("Hello"), new ProcessorLibrary.ErrorAdder(new ErrorMessage(Error.TIMEOUT.code,"Message")))); - chains.add(new Chain<Processor>("bad_gateway", new ProcessorLibrary.StringDataAdder("Hello"), new ProcessorLibrary.ErrorAdder(new ErrorMessage(502,"Message")))); - chains.add(new Chain<Processor>("bad_gateway_mapped", new ProcessorLibrary.StringDataAdder("Hello"), new ProcessorLibrary.ErrorAdder(new ErrorMessage(Error.BACKEND_COMMUNICATION_ERROR.code,"Message")))); - chains.add(new Chain<Processor>("unknown_code", new ProcessorLibrary.StringDataAdder("Hello"), new ProcessorLibrary.ErrorAdder(new ErrorMessage(1234567,"Message")))); + chains.add(new Chain<>("moved_permanently", new ProcessorLibrary.StringDataAdder("Hello"), new ProcessorLibrary.ErrorAdder(new ErrorMessage(301,"Message")))); + chains.add(new Chain<>("unauthorized", new ProcessorLibrary.StringDataAdder("Hello"), new ProcessorLibrary.ErrorAdder(new ErrorMessage(401,"Message")))); + chains.add(new Chain<>("unauthorized_mapped", new ProcessorLibrary.StringDataAdder("Hello"), new ProcessorLibrary.ErrorAdder(new ErrorMessage(Error.UNAUTHORIZED.code,"Message")))); + chains.add(new Chain<>("forbidden", new ProcessorLibrary.StringDataAdder("Hello"), new ProcessorLibrary.ErrorAdder(new ErrorMessage(403,"Message")))); + chains.add(new Chain<>("forbidden_mapped", new ProcessorLibrary.StringDataAdder("Hello"), new ProcessorLibrary.ErrorAdder(new ErrorMessage(Error.FORBIDDEN.code,"Message")))); + chains.add(new Chain<>("not_found", new ProcessorLibrary.StringDataAdder("Hello"), new ProcessorLibrary.ErrorAdder(new ErrorMessage(404,"Message")))); + chains.add(new Chain<>("not_found_mapped", new ProcessorLibrary.StringDataAdder("Hello"), new ProcessorLibrary.ErrorAdder(new ErrorMessage(Error.NOT_FOUND.code,"Message")))); + chains.add(new Chain<>("too_many_requests", new ProcessorLibrary.StringDataAdder("Hello"), new ProcessorLibrary.ErrorAdder(new ErrorMessage(429,"Message")))); + chains.add(new Chain<>("bad_request", new ProcessorLibrary.StringDataAdder("Hello"), new ProcessorLibrary.ErrorAdder(new ErrorMessage(400,"Message")))); + chains.add(new Chain<>("bad_request_mapped", new ProcessorLibrary.StringDataAdder("Hello"), new ProcessorLibrary.ErrorAdder(new ErrorMessage(Error.BAD_REQUEST.code,"Message")))); + chains.add(new Chain<>("internal_server_error", new ProcessorLibrary.StringDataAdder("Hello"), new ProcessorLibrary.ErrorAdder(new ErrorMessage(500,"Message")))); + chains.add(new Chain<>("internal_server_error_mapped", new ProcessorLibrary.StringDataAdder("Hello"), new ProcessorLibrary.ErrorAdder(new ErrorMessage(Error.INTERNAL_SERVER_ERROR.code,"Message")))); + chains.add(new Chain<>("service_unavailable", new ProcessorLibrary.StringDataAdder("Hello"), new ProcessorLibrary.ErrorAdder(new ErrorMessage(503,"Message")))); + chains.add(new Chain<>("service_unavailable_mapped", new ProcessorLibrary.StringDataAdder("Hello"), new ProcessorLibrary.ErrorAdder(new ErrorMessage(Error.NO_BACKENDS_IN_SERVICE.code,"Message")))); + chains.add(new Chain<>("gateway_timeout", new ProcessorLibrary.StringDataAdder("Hello"), new ProcessorLibrary.ErrorAdder(new ErrorMessage(504,"Message")))); + chains.add(new Chain<>("gateway_timeout_mapped", new ProcessorLibrary.StringDataAdder("Hello"), new ProcessorLibrary.ErrorAdder(new ErrorMessage(Error.TIMEOUT.code,"Message")))); + chains.add(new Chain<>("bad_gateway", new ProcessorLibrary.StringDataAdder("Hello"), new ProcessorLibrary.ErrorAdder(new ErrorMessage(502,"Message")))); + chains.add(new Chain<>("bad_gateway_mapped", new ProcessorLibrary.StringDataAdder("Hello"), new ProcessorLibrary.ErrorAdder(new ErrorMessage(Error.BACKEND_COMMUNICATION_ERROR.code,"Message")))); + chains.add(new Chain<>("unknown_code", new ProcessorLibrary.StringDataAdder("Hello"), new ProcessorLibrary.ErrorAdder(new ErrorMessage(1234567,"Message")))); driver = new ProcessingTestDriver(chains); assertEqualStatus(200, "http://localhost/?chain=simple"); assertEqualStatus(301, "http://localhost/?chain=moved_permanently"); @@ -227,15 +224,15 @@ public class ProcessingHandlerTestCase { ProcessingTestDriver.MockResponseHandler responseHandler = null; try { Map<String,List<String>> responseHeaders = new HashMap<>(); - responseHeaders.put("foo", Collections.singletonList("fooValue")); - responseHeaders.put("bar", Arrays.asList(new String[] { "barValue", "bazValue"})); + responseHeaders.put("foo", List.of("fooValue")); + responseHeaders.put("bar", List.of("barValue", "bazValue")); Map<String,List<String>> otherResponseHeaders = new HashMap<>(); - otherResponseHeaders.put("foo", Collections.singletonList("fooValue2")); - otherResponseHeaders.put("bax", Collections.singletonList("baxValue")); + otherResponseHeaders.put("foo", List.of("fooValue2")); + otherResponseHeaders.put("bax", List.of("baxValue")); List<Chain<Processor>> chains = new ArrayList<>(); - chains.add(new Chain<Processor>("default",new ResponseHeaderSetter(responseHeaders), + chains.add(new Chain<>("default",new ResponseHeaderSetter(responseHeaders), new ResponseHeaderSetter(otherResponseHeaders))); driver = new ProcessingTestDriver(chains); responseHandler = driver.sendRequest("http://localhost/?chain=default").awaitResponse(); @@ -256,7 +253,7 @@ public class ProcessingHandlerTestCase { ProcessingTestDriver.MockResponseHandler responseHandler = null; try { List<Chain<Processor>> chains = new ArrayList<>(); - chains.add(new Chain<Processor>("default", new ResponseStatusSetter(429))); + chains.add(new Chain<>("default", new ResponseStatusSetter(429))); driver = new ProcessingTestDriver(chains); responseHandler = driver.sendRequest("http://localhost/?chain=default").awaitResponse(); Response response = responseHandler.getResponse(); @@ -275,7 +272,7 @@ public class ProcessingHandlerTestCase { ProcessingTestDriver.MockResponseHandler responseHandler = null; try { List<Chain<Processor>> chains = new ArrayList<>(); - chains.add(new Chain<Processor>("default", new ResponseStatusSetter(200), + chains.add(new Chain<>("default", new ResponseStatusSetter(200), new ProcessorLibrary.StringDataAdder("Hello"), new ProcessorLibrary.ErrorAdder(new ErrorMessage(Error.FORBIDDEN.code,"Message")))); driver = new ProcessingTestDriver(chains); @@ -307,14 +304,14 @@ public class ProcessingHandlerTestCase { @SuppressWarnings("unchecked") @Test - public void testProcessingHandlerSupportsAsyncRendering() throws Exception { + public void testProcessingHandlerSupportsAsyncRendering() { // Set up ProcessorLibrary.FutureDataSource futureDataSource = new ProcessorLibrary.FutureDataSource(); - Chain<Processor> asyncCompletionChain = new Chain<Processor>("asyncCompletion", new ProcessorLibrary.DataCounter("async")); + Chain<Processor> asyncCompletionChain = new Chain<>("asyncCompletion", new ProcessorLibrary.DataCounter("async")); Chain<Processor> chain = - new Chain<Processor>("federation", new ProcessorLibrary.DataCounter("sync"), + new Chain<>("federation", new ProcessorLibrary.DataCounter("sync"), new ProcessorLibrary.Federator(new Chain<Processor>(new ProcessorLibrary.DataSource()), - new Chain<Processor>(new ProcessorLibrary.AsyncDataProcessingInitiator(asyncCompletionChain),futureDataSource))); + new Chain<>(new ProcessorLibrary.AsyncDataProcessingInitiator(asyncCompletionChain),futureDataSource))); List<Chain<Processor>> chains = new ArrayList<>(); chains.add(chain); driver = new ProcessingTestDriver(chains); @@ -350,17 +347,17 @@ public class ProcessingHandlerTestCase { assertEquals(",{\"data\":\"[sync] Data count: 3\"}" + // Async items not counted as they arrive after chain completion "]}", responseHandler.read()); - assertTrue("Transmission completed", null == responseHandler.read()); + assertNull("Transmission completed", responseHandler.read()); } @SuppressWarnings("unchecked") @Test - public void testProcessingHandlerSupportsAsyncUnorderedRendering() throws Exception { + public void testProcessingHandlerSupportsAsyncUnorderedRendering() { // Set up ProcessorLibrary.FutureDataSource futureDataSource1 = new ProcessorLibrary.FutureDataSource(); ProcessorLibrary.FutureDataSource futureDataSource2 = new ProcessorLibrary.FutureDataSource(); Chain<Processor> chain = - new Chain<Processor>("federation", + new Chain<>("federation", new ProcessorLibrary.Federator(false,new Chain<Processor>(futureDataSource1), new Chain<Processor>(futureDataSource2))); List<Chain<Processor>> chains = new ArrayList<>(); @@ -393,7 +390,7 @@ public class ProcessingHandlerTestCase { "]}", responseHandler.read()); - assertTrue("Transmission completed", responseHandler.read()==null); + assertNull("Transmission completed", responseHandler.read()); } @SuppressWarnings("unchecked") @@ -401,7 +398,7 @@ public class ProcessingHandlerTestCase { public void testAsyncOnlyRendering() throws Exception { // Set up ProcessorLibrary.ListenableFutureDataSource futureDataSource = new ProcessorLibrary.ListenableFutureDataSource(); - Chain<Processor> chain = new Chain<>("main", Collections.<Processor>singletonList(futureDataSource)); + Chain<Processor> chain = new Chain<>("main", List.of(futureDataSource)); driver = new ProcessingTestDriver(chain); ProcessingTestDriver.MockResponseHandler responseHandler = driver.sendRequest("http://localhost/?chain=main"); @@ -421,7 +418,7 @@ public class ProcessingHandlerTestCase { responseHandler.read()); assertEquals(200, responseHandler.getStatus()); - assertTrue("Transmission completed", null == responseHandler.read()); + assertNull("Transmission completed", responseHandler.read()); } @SuppressWarnings("unchecked") @@ -429,7 +426,7 @@ public class ProcessingHandlerTestCase { public void testAsyncRenderingWithClientClose() throws Exception { // Set up ProcessorLibrary.ListenableFutureDataSource futureDataSource = new ProcessorLibrary.ListenableFutureDataSource(); - Chain<Processor> chain = new Chain<>("main", Collections.<Processor>singletonList(futureDataSource)); + Chain<Processor> chain = new Chain<>("main", List.of(futureDataSource)); driver = new ProcessingTestDriver(chain); ProcessingTestDriver.MockResponseHandler responseHandler = driver.sendRequest("http://localhost/?chain=main"); @@ -446,7 +443,7 @@ public class ProcessingHandlerTestCase { assertNull(responseHandler.read()); assertEquals(200, responseHandler.getStatus()); - assertTrue("Transmission completed", null == responseHandler.read()); + assertNull("Transmission completed", responseHandler.read()); } @SuppressWarnings("unchecked") @@ -482,7 +479,7 @@ public class ProcessingHandlerTestCase { assertEquals("Data is completed, so post data is read", "Hello, world!", postReader.bodyDataFuture.get().trim()); assertEquals(200, responseHandler.getStatus()); - assertTrue("Transmission completed", null == responseHandler.read()); + assertNull("Transmission completed", responseHandler.read()); } private static class PostReader extends Processor { @@ -511,12 +508,12 @@ public class ProcessingHandlerTestCase { @Test public void testStatusAndHeadersCanBeSetAsynchronously() throws Exception { Map<String,List<String>> responseHeaders = new HashMap<>(); - responseHeaders.put("foo", Collections.singletonList("fooValue")); - responseHeaders.put("bar", Arrays.asList(new String[] { "barValue", "bazValue"})); + responseHeaders.put("foo", List.of("fooValue")); + responseHeaders.put("bar", List.of("barValue", "bazValue")); // Set up ProcessorLibrary.ListenableFutureDataSource futureDataSource = new ProcessorLibrary.ListenableFutureDataSource(true, false); - Chain<Processor> chain = new Chain<Processor>("main", new ProcessorLibrary.AsyncDataProcessingInitiator(new Chain<Processor>("async", new ProcessorLibrary.StatusSetter(500), new ResponseHeaderSetter(responseHeaders))), futureDataSource); + Chain<Processor> chain = new Chain<>("main", new ProcessorLibrary.AsyncDataProcessingInitiator(new Chain<>("async", new ProcessorLibrary.StatusSetter(500), new ResponseHeaderSetter(responseHeaders))), futureDataSource); driver = new ProcessingTestDriver(chain); ProcessingTestDriver.MockResponseHandler responseHandler = driver.sendRequest("http://localhost/?chain=main"); @@ -530,18 +527,18 @@ public class ProcessingHandlerTestCase { assertEquals(500, responseHandler.getStatus()); assertEquals("[fooValue]", responseHandler.getResponse().headers().get("foo").toString()); assertEquals("[barValue, bazValue]", responseHandler.getResponse().headers().get("bar").toString()); - assertTrue("Transmission completed", null == responseHandler.read()); + assertNull("Transmission completed", responseHandler.read()); } @SuppressWarnings("unchecked") @Test - public void testAsyncRenderingDoesNotHoldThreads() throws Exception { + public void testAsyncRenderingDoesNotHoldThreads() { // Set up ProcessorLibrary.FutureDataSource futureDataSource = new ProcessorLibrary.FutureDataSource(); // Add some sync data as well to cause rendering to start before async data is added. // This allows us to wait on return data rather than having to wait for the 100 requests // to be done, which is cumbersome. - Chain<Processor> chain = new Chain<Processor>("main", new ProcessorLibrary.Federator(new Chain<Processor>(new ProcessorLibrary.DataSource()), new Chain<Processor>(futureDataSource))); + Chain<Processor> chain = new Chain<>("main", new ProcessorLibrary.Federator(new Chain<Processor>(new ProcessorLibrary.DataSource()), new Chain<Processor>(futureDataSource))); driver = new ProcessingTestDriver(chain); int requestCount = 1000; @@ -561,7 +558,7 @@ public class ProcessingHandlerTestCase { futureDataSource.incomingData.get(i).addLast(new ProcessorLibrary.StringData(null, "d2")); assertEquals(",{\"data\":\"d2\"}]}", responseHandler[i].read()); assertEquals("]}", responseHandler[i].read()); - assertTrue("Transmission completed", null == responseHandler[i].read()); + assertNull("Transmission completed", responseHandler[i].read()); } } @@ -569,10 +566,10 @@ public class ProcessingHandlerTestCase { @Test public void testStreamedRendering() throws Exception { // Set up - Chain<Processor> streamChain = new Chain<Processor>(new StreamProcessor()); + Chain<Processor> streamChain = new Chain<>(new StreamProcessor()); ProcessorLibrary.ListenableFutureDataSource futureDataSource = new ProcessorLibrary.ListenableFutureDataSource(); - Chain<Processor> mainChain = new Chain<Processor>("main", new ProcessorLibrary.StreamProcessingInitiator(streamChain), futureDataSource); + Chain<Processor> mainChain = new Chain<>("main", new ProcessorLibrary.StreamProcessingInitiator(streamChain), futureDataSource); driver = new ProcessingTestDriver(mainChain); ProcessingTestDriver.MockResponseHandler responseHandler = driver.sendRequest("http://localhost/?chain=main"); @@ -596,14 +593,13 @@ public class ProcessingHandlerTestCase { ",{\"data\":\"map data: {streamProcessed=true}\"}]}", responseHandler.read()); - assertTrue("Transmission completed", null == responseHandler.read()); + assertNull("Transmission completed", responseHandler.read()); } - @SuppressWarnings("unchecked") @Test - public void testEagerStreamedRenderingOnFreeze() throws Exception { + public void testEagerStreamedRenderingOnFreeze() { FreezingDataSource source = new FreezingDataSource(); - Chain<Processor> mainChain = new Chain<Processor>("main", source); + Chain<Processor> mainChain = new Chain<>("main", source); driver = new ProcessingTestDriver(mainChain); ProcessingTestDriver.MockResponseHandler responseHandler = driver.sendRequest("http://localhost/?chain=main"); assertEquals("No data is available at this point", 0, responseHandler.available()); @@ -611,18 +607,17 @@ public class ProcessingHandlerTestCase { assertEquals("{\"datalist\":[{\"data\":\"d1\"}", responseHandler.read()); source.addLastData.set(true); // signal completion assertEquals(",{\"data\":\"d2\"}]}", responseHandler.read()); - assertTrue("Transmission completed", null == responseHandler.read()); + assertNull("Transmission completed", responseHandler.read()); } - @SuppressWarnings("unchecked") @Test @Ignore // TODO - public void testNestedEagerStreamedRenderingOnFreeze() throws Exception { + public void testNestedEagerStreamedRenderingOnFreeze() { try { FreezingDataSource source1 = new FreezingDataSource("s1"); FreezingDataSource source2 = new FreezingDataSource("s2"); FreezingDataSource source3 = new FreezingDataSource("s3"); - Chain<Processor> mainChain = new Chain<Processor>("main", + Chain<Processor> mainChain = new Chain<>("main", new ProcessorLibrary.StringDataAdder("main-data"), new ProcessorLibrary.EagerReturnFederator(true, new Chain<Processor>(source1), @@ -639,7 +634,7 @@ public class ProcessingHandlerTestCase { assertEquals("{\"datalist\":[{\"data\":\"s1d1\"}", responseHandler.read()); source1.addLastData.set(true); // Make source 1 and 2 available assertEquals(",{\"data\":\"d2\"}]}", responseHandler.read()); - assertTrue("Transmission completed", null == responseHandler.read()); + assertNull("Transmission completed", responseHandler.read()); } catch (Throwable t) { t.printStackTrace(); @@ -648,31 +643,31 @@ public class ProcessingHandlerTestCase { } @Test(expected = IllegalArgumentException.class) - public void testRetrievingNonExistingRendererThrows() throws Exception { - driver = new ProcessingTestDriver(Collections.<Chain<Processor>>emptyList()); + public void testRetrievingNonExistingRendererThrows() { + driver = new ProcessingTestDriver(List.of()); driver.processingHandler().getRendererCopy(ComponentSpecification.fromString("non-existent")); } @Test - public void testDefaultRendererIsAddedToRegistryWhenNoneIsGivenByUser() throws Exception { + public void testDefaultRendererIsAddedToRegistryWhenNoneIsGivenByUser() { String defaultId = AbstractProcessingHandler.DEFAULT_RENDERER_ID; - driver = new ProcessingTestDriver(Collections.<Chain<Processor>>emptyList()); + driver = new ProcessingTestDriver(List.of()); Renderer defaultRenderer = driver.processingHandler().getRenderers().getComponent(defaultId); - assertThat(defaultRenderer, notNullValue()); + assertNotNull(defaultRenderer); } @Test - public void testUserSpecifiedDefaultRendererIsNotReplacedInRegistry() throws Exception { + public void testUserSpecifiedDefaultRendererIsNotReplacedInRegistry() { String defaultId = AbstractProcessingHandler.DEFAULT_RENDERER_ID; Renderer myDefaultRenderer = new ProcessingRenderer(); ComponentRegistry<Renderer> renderers = new ComponentRegistry<>(); renderers.register(ComponentId.fromString(defaultId), myDefaultRenderer); - driver = new ProcessingTestDriver(Collections.<Chain<Processor>>emptyList(), renderers); + driver = new ProcessingTestDriver(List.of(), renderers); Renderer defaultRenderer = driver.processingHandler().getRenderers().getComponent(defaultId); - assertThat(defaultRenderer, sameInstance(myDefaultRenderer)); + assertSame(defaultRenderer, myDefaultRenderer); } @@ -727,7 +722,7 @@ public class ProcessingHandlerTestCase { } - private String defaultChainResponse = + private final String defaultChainResponse = "{\"datalist\":[" + "{\"data\":\"StringData.toString()\"}," + "{\"datalist\":[" + @@ -736,17 +731,17 @@ public class ProcessingHandlerTestCase { "}]" + "}"; - private String simpleChainResponse = + private final String simpleChainResponse = "{\"datalist\":[" + "{\"data\":\"StringData.toString()\"}]" + "}"; - private String trace1 = + private final String trace1 = "{\"trace\":[" + "\"TraceMessage\"" + "],"; - private String trace1WithFullResult = + private final String trace1WithFullResult = "{\"trace\":[" + "\"TraceMessage\"" + "]," + @@ -758,7 +753,7 @@ public class ProcessingHandlerTestCase { "]}" + "]}"; - private String trace4 = + private final String trace4 = "{\"trace\":[" + "\"Invoke '(anonymous)' of class 'com.yahoo.processing.test.ProcessorLibrary$StringDataListAdder'\"," + "\"Invoke '(anonymous)' of class 'com.yahoo.processing.test.ProcessorLibrary$Trace'\"," + @@ -769,7 +764,7 @@ public class ProcessingHandlerTestCase { "\"Return '(anonymous)' of class 'com.yahoo.processing.test.ProcessorLibrary$StringDataListAdder'\"" + "],"; - private String trace5 = + private final String trace5 = "{\"trace\":[" + "\"Invoke '(anonymous)' of class 'com.yahoo.processing.test.ProcessorLibrary$StringDataListAdder'\"," + "\"Invoke '(anonymous)' of class 'com.yahoo.processing.test.ProcessorLibrary$Trace'\"," + @@ -780,7 +775,7 @@ public class ProcessingHandlerTestCase { "\"Return '(anonymous)' of class 'com.yahoo.processing.test.ProcessorLibrary$StringDataListAdder'\"" + "],"; - private String trace6 = + private final String trace6 = "{\"trace\":[" + "{\"timestamp\":ddddddddddddd,\"message\":\"Invoke '(anonymous)' of class 'com.yahoo.processing.test.ProcessorLibrary$StringDataListAdder'\"}," + "{\"timestamp\":ddddddddddddd,\"message\":\"Invoke '(anonymous)' of class 'com.yahoo.processing.test.ProcessorLibrary$Trace'\"}," + diff --git a/container-core/src/test/java/com/yahoo/processing/rendering/AsynchronousSectionedRendererTest.java b/container-core/src/test/java/com/yahoo/processing/rendering/AsynchronousSectionedRendererTest.java index 627081e0d3b..0ea8a157410 100644 --- a/container-core/src/test/java/com/yahoo/processing/rendering/AsynchronousSectionedRendererTest.java +++ b/container-core/src/test/java/com/yahoo/processing/rendering/AsynchronousSectionedRendererTest.java @@ -23,15 +23,12 @@ import java.nio.charset.Charset; import java.util.ArrayList; import java.util.List; import java.util.concurrent.CompletableFuture; -import java.util.concurrent.ExecutionException; import java.util.concurrent.Executor; import java.util.concurrent.TimeUnit; -import java.util.concurrent.TimeoutException; -import static org.hamcrest.CoreMatchers.equalTo; import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertThat; -import static org.junit.Assert.assertTrue; +import static org.junit.Assert.assertNotSame; +import static org.junit.Assert.assertSame; /** * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a> @@ -45,7 +42,7 @@ public class AsynchronousSectionedRendererTest { TestRenderer rendererPrototype = new TestRenderer(); TestRenderer rendererCopy1 = (TestRenderer)rendererPrototype.clone(); rendererCopy1.init(); - assertTrue(rendererPrototype.getRenderingExecutor() == rendererCopy1.getRenderingExecutor()); + assertSame(rendererPrototype.getRenderingExecutor(), rendererCopy1.getRenderingExecutor()); } @Test @@ -57,7 +54,7 @@ public class AsynchronousSectionedRendererTest { TestRenderer rendererPrototype2 = new TestRenderer(); TestRenderer rendererCopy2 = (TestRenderer)rendererPrototype2.clone(); rendererCopy2.init(); - assertTrue(rendererPrototype1.getRenderingExecutor() != rendererCopy2.getRenderingExecutor()); + assertNotSame(rendererPrototype1.getRenderingExecutor(), rendererCopy2.getRenderingExecutor()); } @Test @@ -69,9 +66,9 @@ public class AsynchronousSectionedRendererTest { String str = render(renderer, dataList); - assertThat(str, - equalTo(" beginResponse beginList[f\\o\"o, [b/a\br, f\f\no\ro\tbar\u0005]] dataf\\o\"o beginList[b/a\br, " + - "f\f\no\ro\tbar\u0005] datab/a\br dataf\f\no\ro\tbar\u0005 endList[b/a\br, f\f\no\ro\tbar\u0005] endList[f\\o\"o, [b/a\br, f\f\no\ro\tbar\u0005]] endResponse")); + assertEquals(" beginResponse beginList[f\\o\"o, [b/a\br, f\f\no\ro\tbar\u0005]] dataf\\o\"o beginList[b/a\br, " + + "f\f\no\ro\tbar\u0005] datab/a\br dataf\f\no\ro\tbar\u0005 endList[b/a\br, f\f\no\ro\tbar\u0005] endList[f\\o\"o, [b/a\br, f\f\no\ro\tbar\u0005]] endResponse", + str); } @Test @@ -79,23 +76,21 @@ public class AsynchronousSectionedRendererTest { Request request = new Request(); DataList dataList = ArrayDataList.create(request); - assertThat(render(dataList), - equalTo("{\"datalist\":[" + - "]}")); + assertEquals("{\"datalist\":[]}", render(dataList)); } @Test public void testProcessingRendering() throws IOException, InterruptedException { StringDataList dataList = createDataListWithStrangeStrings(); - assertThat(render(dataList), - equalTo("{\"datalist\":[" + + assertEquals("{\"datalist\":[" + "{\"data\":\"f\\\\o\\\"o\"}," + "{\"datalist\":[" + "{\"data\":\"b/a\\br\"}," + "{\"data\":\"f\\f\\no\\ro\\tbar\\u0005\"}" + "]}" + - "]}")); + "]}", + render(dataList)); } @Test @@ -106,8 +101,7 @@ public class AsynchronousSectionedRendererTest { dataList.request().errors().add(new ErrorMessage("m1","d1")); dataList.request().errors().add(new ErrorMessage("m2","d2")); - assertThat(render(dataList), - equalTo("{\"errors\":[" + + assertEquals("{\"errors\":[" + "\"m1: d1\"," + "\"m2: d2\"" + "]," + @@ -117,12 +111,13 @@ public class AsynchronousSectionedRendererTest { "{\"data\":\"l11\"}," + "{\"data\":\"l12\"}" + "]}" + - "]}")); + "]}", + render(dataList)); } @Test public void testProcessingRenderingWithStackTraces() throws IOException, InterruptedException { - Exception exception=null; + Exception exception; // Create thrown exception try { throw new RuntimeException("Thrown"); @@ -155,8 +150,7 @@ public class AsynchronousSectionedRendererTest { dataList.add(new StringDataList(dataList.request().clone())); // Cloning a request which contains errors // ... should not cause repetition of those errors - assertThat(render(dataList), - equalTo("{\"errors\":[" + + assertEquals("{\"errors\":[" + "\"m1: d1\"," + "\"m2: d2\"" + "]," + @@ -167,7 +161,8 @@ public class AsynchronousSectionedRendererTest { "{\"data\":\"l12\"}" + "]}," + "{\"datalist\":[]}" + - "]}")); + "]}", + render(dataList)); } @Test @@ -181,8 +176,7 @@ public class AsynchronousSectionedRendererTest { // and adding new errors to it dataList.asList().get(2).request().errors().add(new ErrorMessage("m3","d3")); - assertThat(render(dataList), - equalTo("{\"errors\":[" + + assertEquals("{\"errors\":[" + "\"m1: d1\"," + "\"m2: d2\"" + "]," + @@ -196,7 +190,8 @@ public class AsynchronousSectionedRendererTest { "\"m3: d3\"" + "]," + "\"datalist\":[]}" + - "]}")); + "]}", + render(dataList)); } public StringDataList createDataList() { @@ -334,7 +329,7 @@ public class AsynchronousSectionedRendererTest { } } - private abstract class StringData extends ListenableFreezableClass implements Data { + private static abstract class StringData extends ListenableFreezableClass implements Data { private final Request request; private StringData(Request request) { @@ -410,7 +405,7 @@ public class AsynchronousSectionedRendererTest { @Override @SuppressWarnings("removal") public ListenableFuture<DataList<StringData>> complete() { - return new ListenableFuture<DataList<StringData>>() { + return new ListenableFuture<>() { @Override public void addListener(Runnable runnable, Executor executor) { } @@ -431,13 +426,12 @@ public class AsynchronousSectionedRendererTest { } @Override - public DataList<StringData> get() throws InterruptedException, ExecutionException { + public DataList<StringData> get() { return StringDataList.this; } @Override - public DataList<StringData> get(long l, TimeUnit timeUnit) - throws InterruptedException, ExecutionException, TimeoutException { + public DataList<StringData> get(long l, TimeUnit timeUnit) { return StringDataList.this; } }; diff --git a/container-dependency-versions/pom.xml b/container-dependency-versions/pom.xml index b34dcafacdf..ff6786080ef 100644 --- a/container-dependency-versions/pom.xml +++ b/container-dependency-versions/pom.xml @@ -424,7 +424,7 @@ <!-- and then verify by doing: ' ls -l vespa/vespa_jersey2/target/dependency' --> <hk2.version>2.5.0-b32</hk2.version> <hk2.osgi-resource-locator.version>1.0.1</hk2.osgi-resource-locator.version> - <jackson2.version>2.12.1</jackson2.version> + <jackson2.version>2.12.6</jackson2.version> <jackson-databind.version>${jackson2.version}</jackson-databind.version> <javassist.version>3.20.0-GA</javassist.version> <javax.annotation-api.version>1.2</javax.annotation-api.version> diff --git a/container-dev/pom.xml b/container-dev/pom.xml index a76f295a7b1..034081f4620 100644 --- a/container-dev/pom.xml +++ b/container-dev/pom.xml @@ -194,10 +194,6 @@ <version>${project.version}</version> <exclusions> <exclusion> - <groupId>com.optimaize.languagedetector</groupId> - <artifactId>language-detector</artifactId> - </exclusion> - <exclusion> <groupId>org.apache.opennlp</groupId> <artifactId>opennlp-tools</artifactId> </exclusion> diff --git a/container-disc/src/main/sh/vespa-start-container-daemon.sh b/container-disc/src/main/sh/vespa-start-container-daemon.sh index 5d47392292d..1aad9f18616 100755 --- a/container-disc/src/main/sh/vespa-start-container-daemon.sh +++ b/container-disc/src/main/sh/vespa-start-container-daemon.sh @@ -281,7 +281,6 @@ exec $numactlcmd $envcmd java \ --add-opens=java.base/java.nio=ALL-UNNAMED \ --add-opens=java.base/jdk.internal.loader=ALL-UNNAMED \ --add-opens=java.base/sun.security.ssl=ALL-UNNAMED \ - --add-opens=java.base/sun.security.util=ALL-UNNAMED \ -Djava.io.tmpdir="${VESPA_HOME}/tmp" \ -Djava.library.path="${VESPA_HOME}/lib64" \ -Djava.awt.headless=true \ diff --git a/container-disc/src/test/java/com/yahoo/container/jdisc/FilterBindingsProviderTest.java b/container-disc/src/test/java/com/yahoo/container/jdisc/FilterBindingsProviderTest.java index 40c66e115e4..70e956480ed 100644 --- a/container-disc/src/test/java/com/yahoo/container/jdisc/FilterBindingsProviderTest.java +++ b/container-disc/src/test/java/com/yahoo/container/jdisc/FilterBindingsProviderTest.java @@ -11,7 +11,12 @@ import com.yahoo.jdisc.http.filter.ResponseFilter; import com.yahoo.jdisc.http.server.jetty.FilterBindings; import org.junit.Test; -import static org.assertj.core.api.Assertions.assertThat; +import java.util.Set; +import java.util.stream.Collectors; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; import static org.mockito.Mockito.mock; @@ -39,9 +44,9 @@ public class FilterBindingsProviderTest { final FilterBindings filterBindings = provider.get(); - assertThat(filterBindings).isNotNull(); - assertThat(filterBindings.requestFilterIds()).isEmpty(); - assertThat(filterBindings.responseFilterIds()).isEmpty(); + assertNotNull(filterBindings); + assertTrue(filterBindings.requestFilterIds().isEmpty()); + assertTrue(filterBindings.responseFilterIds().isEmpty()); } @Test @@ -92,11 +97,11 @@ public class FilterBindingsProviderTest { final FilterBindings filterBindings = provider.get(); // Verify. - assertThat(filterBindings).isNotNull(); - assertThat(filterBindings.requestFilters()) - .containsExactlyInAnyOrder(requestFilter1Instance, requestFilter2Instance); - assertThat(filterBindings.responseFilters()) - .containsExactlyInAnyOrder(responseFilter1Instance, responseFilter3Instance); + assertNotNull(filterBindings); + assertEquals(filterBindings.requestFilters().stream().collect(Collectors.toSet()), + Set.of(requestFilter1Instance, requestFilter2Instance)); + assertEquals(filterBindings.responseFilters().stream().collect(Collectors.toSet()), + Set.of(responseFilter1Instance, responseFilter3Instance)); } private interface DualRoleFilter extends RequestFilter, ResponseFilter {} @@ -127,7 +132,7 @@ public class FilterBindingsProviderTest { new ComponentRegistry<>()); fail("Dual-role filter should not be accepted"); } catch (RuntimeException e) { - assertThat(e.getMessage()).contains("Invalid config"); + assertTrue(e.getMessage().contains("Invalid config")); } } @@ -152,7 +157,7 @@ public class FilterBindingsProviderTest { new ComponentRegistry<>()); fail("Config with unknown filter reference should not be accepted"); } catch (RuntimeException e) { - assertThat(e.getMessage()).contains("Invalid config"); + assertTrue(e.getMessage().contains("Invalid config")); } } diff --git a/container-messagebus/src/test/java/com/yahoo/messagebus/jdisc/ClientThreadingTestCase.java b/container-messagebus/src/test/java/com/yahoo/messagebus/jdisc/ClientThreadingTestCase.java index 4336c491071..549a1e3b887 100644 --- a/container-messagebus/src/test/java/com/yahoo/messagebus/jdisc/ClientThreadingTestCase.java +++ b/container-messagebus/src/test/java/com/yahoo/messagebus/jdisc/ClientThreadingTestCase.java @@ -31,8 +31,7 @@ import java.util.concurrent.Executors; import java.util.concurrent.Future; import java.util.concurrent.TimeUnit; -import static org.hamcrest.CoreMatchers.is; -import static org.junit.Assert.assertThat; +import static org.junit.Assert.assertTrue; /** * @author Simon Thoresen Hult @@ -56,11 +55,11 @@ public class ClientThreadingTestCase { } final ExecutorService executor = Executors.newFixedThreadPool(NUM_THREADS); for (final Future<Boolean> res : executor.invokeAll(lst, 60, TimeUnit.SECONDS)) { - assertThat(res.get(), is(true)); + assertTrue(res.get()); } - assertThat(client.close(), is(true)); - assertThat(server.close(), is(true)); + assertTrue(client.close()); + assertTrue(server.close()); } private static final class RequestTask implements Callable<Boolean> { diff --git a/container-messagebus/src/test/java/com/yahoo/messagebus/jdisc/MbusServerConformanceTest.java b/container-messagebus/src/test/java/com/yahoo/messagebus/jdisc/MbusServerConformanceTest.java index 13059f7c91f..397a3eb5242 100644 --- a/container-messagebus/src/test/java/com/yahoo/messagebus/jdisc/MbusServerConformanceTest.java +++ b/container-messagebus/src/test/java/com/yahoo/messagebus/jdisc/MbusServerConformanceTest.java @@ -18,7 +18,6 @@ import com.yahoo.messagebus.shared.ServerSession; import com.yahoo.messagebus.shared.SharedMessageBus; import com.yahoo.messagebus.test.SimpleMessage; import com.yahoo.messagebus.test.SimpleProtocol; -import org.hamcrest.Matcher; import org.junit.Ignore; import org.junit.Test; @@ -31,12 +30,12 @@ import java.util.concurrent.LinkedBlockingDeque; import java.util.concurrent.TimeUnit; import static com.yahoo.messagebus.ErrorCode.APP_FATAL_ERROR; -import static com.yahoo.messagebus.ErrorCode.SEND_QUEUE_CLOSED; import static com.yahoo.messagebus.ErrorCode.SESSION_BUSY; -import static org.hamcrest.CoreMatchers.is; -import static org.hamcrest.CoreMatchers.notNullValue; -import static org.hamcrest.CoreMatchers.nullValue; -import static org.hamcrest.MatcherAssert.assertThat; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertTrue; /** * @author Simon Thoresen Hult @@ -51,28 +50,28 @@ public class MbusServerConformanceTest extends ServerProviderConformanceTest { @Test public void testContainerNotReadyException() throws Throwable { new TestRunner().setRequestTimeout(100, TimeUnit.MILLISECONDS) - .expectError(is(SESSION_BUSY)) + .expectError(SESSION_BUSY) .executeAndClose(); } @Override @Test public void testBindingSetNotFoundException() throws Throwable { - new TestRunner().expectError(is(APP_FATAL_ERROR)) + new TestRunner().expectError(APP_FATAL_ERROR) .executeAndClose(); } @Override @Test public void testNoBindingSetSelectedException() throws Throwable { - new TestRunner().expectError(is(APP_FATAL_ERROR)) + new TestRunner().expectError(APP_FATAL_ERROR) .executeAndClose(); } @Override @Test public void testBindingNotFoundException() throws Throwable { - new TestRunner().expectError(is(APP_FATAL_ERROR)) + new TestRunner().expectError(APP_FATAL_ERROR) .executeAndClose(); } @@ -107,7 +106,7 @@ public class MbusServerConformanceTest extends ServerProviderConformanceTest { @Override @Test public void testRequestException() throws Throwable { - new TestRunner().expectError(is(APP_FATAL_ERROR)) + new TestRunner().expectError(APP_FATAL_ERROR) .executeAndClose(); } @@ -142,7 +141,7 @@ public class MbusServerConformanceTest extends ServerProviderConformanceTest { @Override @Test @Ignore // N/A: The messagebus protocol does not have content. - public void testRequestExceptionAfterResponseWriteWithSyncHandleResponse() throws Throwable { + public void testRequestExceptionAfterResponseWriteWithSyncHandleResponse() { } @Override @@ -154,7 +153,7 @@ public class MbusServerConformanceTest extends ServerProviderConformanceTest { @Override @Test public void testRequestExceptionBeforeResponseWriteWithAsyncHandleResponse() throws Throwable { - new TestRunner().expectError(is(APP_FATAL_ERROR)) + new TestRunner().expectError(APP_FATAL_ERROR) .executeAndClose(); } @@ -168,181 +167,181 @@ public class MbusServerConformanceTest extends ServerProviderConformanceTest { @Override @Test @Ignore // N/A: The messagebus protocol does not have content. - public void testRequestExceptionAfterResponseWriteWithAsyncHandleResponse() throws Throwable { + public void testRequestExceptionAfterResponseWriteWithAsyncHandleResponse() { } @Override @Test @Ignore // N/A: The messagebus protocol does not have content. - public void testRequestContentWriteWithSyncCompletion() throws Throwable { + public void testRequestContentWriteWithSyncCompletion() { } @Override @Test @Ignore // N/A: The messagebus protocol does not have content. - public void testRequestContentWriteWithAsyncCompletion() throws Throwable { + public void testRequestContentWriteWithAsyncCompletion() { } @Override @Test @Ignore // N/A: The messagebus protocol does not have content. - public void testRequestContentWriteWithNondeterministicSyncFailure() throws Throwable { + public void testRequestContentWriteWithNondeterministicSyncFailure() { } @Override @Test @Ignore // N/A: The messagebus protocol does not have content. - public void testRequestContentWriteWithSyncFailureBeforeResponseWrite() throws Throwable { + public void testRequestContentWriteWithSyncFailureBeforeResponseWrite() { } @Override @Test @Ignore // N/A: The messagebus protocol does not have content. - public void testRequestContentWriteWithSyncFailureAfterResponseWrite() throws Throwable { + public void testRequestContentWriteWithSyncFailureAfterResponseWrite() { } @Override @Test @Ignore // N/A: The messagebus protocol does not have content. - public void testRequestContentWriteWithNondeterministicAsyncFailure() throws Throwable { + public void testRequestContentWriteWithNondeterministicAsyncFailure() { } @Override @Test @Ignore // N/A: The messagebus protocol does not have content. - public void testRequestContentWriteWithAsyncFailureBeforeResponseWrite() throws Throwable { + public void testRequestContentWriteWithAsyncFailureBeforeResponseWrite() { } @Override @Test @Ignore // N/A: The messagebus protocol does not have content. - public void testRequestContentWriteWithAsyncFailureAfterResponseWrite() throws Throwable { + public void testRequestContentWriteWithAsyncFailureAfterResponseWrite() { } @Override @Test @Ignore // N/A: The messagebus protocol does not have content. - public void testRequestContentWriteWithAsyncFailureAfterResponseCloseNoContent() throws Throwable { + public void testRequestContentWriteWithAsyncFailureAfterResponseCloseNoContent() { } @Override @Test @Ignore // N/A: The messagebus protocol does not have content. - public void testRequestContentWriteNondeterministicException() throws Throwable { + public void testRequestContentWriteNondeterministicException() { } @Override @Test @Ignore // N/A: The messagebus protocol does not have content. - public void testRequestContentWriteExceptionBeforeResponseWrite() throws Throwable { + public void testRequestContentWriteExceptionBeforeResponseWrite() { } @Override @Test @Ignore // N/A: The messagebus protocol does not have content. - public void testRequestContentWriteExceptionAfterResponseWrite() throws Throwable { + public void testRequestContentWriteExceptionAfterResponseWrite() { } @Override @Test @Ignore // N/A: The messagebus protocol does not have content. - public void testRequestContentWriteExceptionAfterResponseCloseNoContent() throws Throwable { + public void testRequestContentWriteExceptionAfterResponseCloseNoContent() { } @Override @Test @Ignore // N/A: The messagebus protocol does not have content. - public void testRequestContentWriteNondeterministicExceptionWithSyncCompletion() throws Throwable { + public void testRequestContentWriteNondeterministicExceptionWithSyncCompletion() { } @Override @Test @Ignore // N/A: The messagebus protocol does not have content. - public void testRequestContentWriteExceptionBeforeResponseWriteWithSyncCompletion() throws Throwable { + public void testRequestContentWriteExceptionBeforeResponseWriteWithSyncCompletion() { } @Override @Test @Ignore // N/A: The messagebus protocol does not have content. - public void testRequestContentWriteExceptionAfterResponseWriteWithSyncCompletion() throws Throwable { + public void testRequestContentWriteExceptionAfterResponseWriteWithSyncCompletion() { } @Override @Test @Ignore // N/A: The messagebus protocol does not have content. - public void testRequestContentWriteExceptionAfterResponseCloseNoContentWithSyncCompletion() throws Throwable { + public void testRequestContentWriteExceptionAfterResponseCloseNoContentWithSyncCompletion() { } @Override @Test @Ignore // N/A: The messagebus protocol does not have content. - public void testRequestContentWriteNondeterministicExceptionWithAsyncCompletion() throws Throwable { + public void testRequestContentWriteNondeterministicExceptionWithAsyncCompletion() { } @Override @Test @Ignore // N/A: The messagebus protocol does not have content. - public void testRequestContentWriteExceptionBeforeResponseWriteWithAsyncCompletion() throws Throwable { + public void testRequestContentWriteExceptionBeforeResponseWriteWithAsyncCompletion() { } @Override @Test @Ignore // N/A: The messagebus protocol does not have content. - public void testRequestContentWriteExceptionAfterResponseWriteWithAsyncCompletion() throws Throwable { + public void testRequestContentWriteExceptionAfterResponseWriteWithAsyncCompletion() { } @Override @Test @Ignore // N/A: The messagebus protocol does not have content. - public void testRequestContentWriteExceptionAfterResponseCloseNoContentWithAsyncCompletion() throws Throwable { + public void testRequestContentWriteExceptionAfterResponseCloseNoContentWithAsyncCompletion() { } @Override @Test @Ignore // N/A: The messagebus protocol does not have content. - public void testRequestContentWriteExceptionWithNondeterministicSyncFailure() throws Throwable { + public void testRequestContentWriteExceptionWithNondeterministicSyncFailure() { } @Override @Test @Ignore // N/A: The messagebus protocol does not have content. - public void testRequestContentWriteExceptionWithSyncFailureBeforeResponseWrite() throws Throwable { + public void testRequestContentWriteExceptionWithSyncFailureBeforeResponseWrite() { } @Override @Test @Ignore // N/A: The messagebus protocol does not have content. - public void testRequestContentWriteExceptionWithSyncFailureAfterResponseWrite() throws Throwable { + public void testRequestContentWriteExceptionWithSyncFailureAfterResponseWrite() { } @Override @Test @Ignore // N/A: The messagebus protocol does not have content. - public void testRequestContentWriteExceptionWithSyncFailureAfterResponseCloseNoContent() throws Throwable { + public void testRequestContentWriteExceptionWithSyncFailureAfterResponseCloseNoContent() { } @Override @Test @Ignore // N/A: The messagebus protocol does not have content. - public void testRequestContentWriteExceptionWithNondeterministicAsyncFailure() throws Throwable { + public void testRequestContentWriteExceptionWithNondeterministicAsyncFailure() { } @Override @Test @Ignore // N/A: The messagebus protocol does not have content. - public void testRequestContentWriteExceptionWithAsyncFailureBeforeResponseWrite() throws Throwable { + public void testRequestContentWriteExceptionWithAsyncFailureBeforeResponseWrite() { } @Override @Test @Ignore // N/A: The messagebus protocol does not have content. - public void testRequestContentWriteExceptionWithAsyncFailureAfterResponseWrite() throws Throwable { + public void testRequestContentWriteExceptionWithAsyncFailureAfterResponseWrite() { } @Override @Test @Ignore // N/A: The messagebus protocol does not have content. - public void testRequestContentWriteExceptionWithAsyncFailureAfterResponseCloseNoContent() throws Throwable { + public void testRequestContentWriteExceptionWithAsyncFailureAfterResponseCloseNoContent() { } @Override @@ -376,7 +375,7 @@ public class MbusServerConformanceTest extends ServerProviderConformanceTest { @Override @Test @Ignore // N/A: The messagebus protocol does not have content. - public void testRequestContentCloseWithSyncFailureAfterResponseWrite() throws Throwable { + public void testRequestContentCloseWithSyncFailureAfterResponseWrite() { } @Override @@ -403,7 +402,7 @@ public class MbusServerConformanceTest extends ServerProviderConformanceTest { @Override @Test @Ignore // N/A: The messagebus protocol does not have content. - public void testRequestContentCloseWithAsyncFailureAfterResponseWrite() throws Throwable { + public void testRequestContentCloseWithAsyncFailureAfterResponseWrite() { } @Override @@ -430,7 +429,7 @@ public class MbusServerConformanceTest extends ServerProviderConformanceTest { @Override @Test @Ignore // N/A: The messagebus protocol does not have content. - public void testRequestContentCloseExceptionAfterResponseWrite() throws Throwable { + public void testRequestContentCloseExceptionAfterResponseWrite() { } @Override @@ -457,7 +456,7 @@ public class MbusServerConformanceTest extends ServerProviderConformanceTest { @Override @Test @Ignore // N/A: The messagebus protocol does not have content. - public void testRequestContentCloseExceptionAfterResponseWriteWithSyncCompletion() throws Throwable { + public void testRequestContentCloseExceptionAfterResponseWriteWithSyncCompletion() { } @Override @@ -484,7 +483,7 @@ public class MbusServerConformanceTest extends ServerProviderConformanceTest { @Override @Test @Ignore // N/A: The messagebus protocol does not have content. - public void testRequestContentCloseExceptionAfterResponseWriteWithAsyncCompletion() throws Throwable { + public void testRequestContentCloseExceptionAfterResponseWriteWithAsyncCompletion() { } @Override @@ -511,7 +510,7 @@ public class MbusServerConformanceTest extends ServerProviderConformanceTest { @Override @Test @Ignore // N/A: The messagebus protocol does not have content. - public void testRequestContentCloseExceptionAfterResponseWriteWithSyncFailure() throws Throwable { + public void testRequestContentCloseExceptionAfterResponseWriteWithSyncFailure() { } @Override @@ -538,7 +537,7 @@ public class MbusServerConformanceTest extends ServerProviderConformanceTest { @Override @Test @Ignore // N/A: The messagebus protocol does not have content. - public void testRequestContentCloseExceptionAfterResponseWriteWithAsyncFailure() throws Throwable { + public void testRequestContentCloseExceptionAfterResponseWriteWithAsyncFailure() { } @Override @@ -551,7 +550,7 @@ public class MbusServerConformanceTest extends ServerProviderConformanceTest { @Override @Test @Ignore // N/A: The messagebus protocol does not have content. - public void testResponseWriteCompletionException() throws Throwable { + public void testResponseWriteCompletionException() { } @Override @@ -573,7 +572,7 @@ public class MbusServerConformanceTest extends ServerProviderConformanceTest { final LocalWire wire = new LocalWire(); final SharedMessageBus mbus; final ServerSession session; - Matcher<Integer> expectedError = null; + Integer expectedError = null; boolean successExpected = false; long timeoutMillis = TimeUnit.SECONDS.toMillis(60); @@ -592,14 +591,14 @@ public class MbusServerConformanceTest extends ServerProviderConformanceTest { return this; } - TestRunner expectError(Matcher<Integer> matcher) { - assertThat(successExpected, is(false)); + TestRunner expectError(Integer matcher) { + assertFalse(successExpected); expectedError = matcher; return this; } TestRunner expectSuccess() { - assertThat(expectedError, is(nullValue())); + assertNull(expectedError); successExpected = true; return this; } @@ -621,24 +620,24 @@ public class MbusServerConformanceTest extends ServerProviderConformanceTest { } @Override - public MyClient newClient(MbusServer server) throws Throwable { + public MyClient newClient(MbusServer server) { return new MyClient(wire, server.connectionSpec()); } @Override public Reply executeRequest(MyClient client, boolean withRequestContent) throws Throwable { // This protocol doesn't have the concept of "request content", so if we are asked to send any, it's a bug. - assertThat(withRequestContent, is(false)); + assertFalse(withRequestContent); final SimpleMessage msg = new SimpleMessage("foo"); msg.getTrace().setLevel(9); msg.setRoute(client.route); msg.setTimeRemaining(timeoutMillis); - assertThat("client.session.send(msg).isAccepted()", - client.session.send(msg).isAccepted(), is(true)); + assertTrue("client.session.send(msg).isAccepted()", + client.session.send(msg).isAccepted()); final Reply reply = client.replies.poll(60, TimeUnit.SECONDS); - assertThat("reply != null", reply, notNullValue()); + assertNotNull("reply != null", reply); return reply; } @@ -648,15 +647,15 @@ public class MbusServerConformanceTest extends ServerProviderConformanceTest { } @Override - public void validateResponse(Reply reply) throws Throwable { + public void validateResponse(Reply reply) { final String trace = String.valueOf(reply.getTrace()); if (expectedError != null) { - assertThat(reply.hasErrors(), is(true)); + assertTrue(reply.hasErrors()); final int error = reply.getError(0).getCode(); - assertThat(trace, error, expectedError); + assertEquals(trace, Integer.valueOf(error), expectedError); } if (successExpected) { - assertThat(trace, reply.hasErrors(), is(false)); + assertFalse(trace, reply.hasErrors()); } } diff --git a/container-messagebus/src/test/java/com/yahoo/messagebus/jdisc/ServerThreadingTestCase.java b/container-messagebus/src/test/java/com/yahoo/messagebus/jdisc/ServerThreadingTestCase.java index 0884762195b..0888a02c74c 100644 --- a/container-messagebus/src/test/java/com/yahoo/messagebus/jdisc/ServerThreadingTestCase.java +++ b/container-messagebus/src/test/java/com/yahoo/messagebus/jdisc/ServerThreadingTestCase.java @@ -28,9 +28,8 @@ import java.util.concurrent.Executors; import java.util.concurrent.LinkedBlockingDeque; import java.util.concurrent.TimeUnit; -import static org.hamcrest.CoreMatchers.instanceOf; -import static org.hamcrest.CoreMatchers.is; -import static org.hamcrest.MatcherAssert.assertThat; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; /** * @author Simon Thoresen Hult @@ -50,16 +49,16 @@ public class ServerThreadingTestCase { final Message msg = new SimpleMessage("foo"); msg.setRoute(Route.parse(server.delegate.connectionSpec())); msg.pushHandler(client); - assertThat(client.session.send(msg).isAccepted(), is(true)); + assertTrue(client.session.send(msg).isAccepted()); } for (int i = 0; i < NUM_REQUESTS; ++i) { final Reply reply = client.replies.poll(600, TimeUnit.SECONDS); - assertThat(reply, instanceOf(EmptyReply.class)); - assertThat(reply.hasErrors(), is(false)); + assertTrue(reply instanceof EmptyReply); + assertFalse(reply.hasErrors()); } - assertThat(client.close(), is(true)); - assertThat(server.close(), is(true)); + assertTrue(client.close()); + assertTrue(server.close()); } private static class Client implements ReplyHandler { @@ -106,14 +105,10 @@ public class ServerThreadingTestCase { @Override public void handleMessage(final Message msg) { - executor.execute(new Runnable() { - - @Override - public void run() { - final Reply reply = new EmptyReply(); - reply.swapState(msg); - reply.popHandler().handleReply(reply); - } + executor.execute(() -> { + final Reply reply = new EmptyReply(); + reply.swapState(msg); + reply.popHandler().handleReply(reply); }); } diff --git a/container-search-and-docproc/pom.xml b/container-search-and-docproc/pom.xml index 90c9c056131..e98b2a08662 100644 --- a/container-search-and-docproc/pom.xml +++ b/container-search-and-docproc/pom.xml @@ -217,11 +217,6 @@ <scope>test</scope> </dependency> <dependency> - <groupId>org.hamcrest</groupId> - <artifactId>hamcrest-library</artifactId> - <scope>test</scope> - </dependency> - <dependency> <groupId>org.mockito</groupId> <artifactId>mockito-core</artifactId> <scope>test</scope> diff --git a/container-search-and-docproc/src/test/java/com/yahoo/container/handler/observability/ApplicationStatusHandlerTest.java b/container-search-and-docproc/src/test/java/com/yahoo/container/handler/observability/ApplicationStatusHandlerTest.java index 93668cf788d..317f5fc1329 100644 --- a/container-search-and-docproc/src/test/java/com/yahoo/container/handler/observability/ApplicationStatusHandlerTest.java +++ b/container-search-and-docproc/src/test/java/com/yahoo/container/handler/observability/ApplicationStatusHandlerTest.java @@ -5,7 +5,6 @@ import com.yahoo.component.ComponentId; import com.yahoo.component.chain.Chain; import com.yahoo.container.core.ApplicationMetadataConfig; import com.yahoo.container.jdisc.JdiscBindingsConfig; -import com.yahoo.jdisc.Metric; import com.yahoo.jdisc.handler.RequestHandler; import com.yahoo.jdisc.service.ClientProvider; import com.yahoo.processing.Processor; @@ -17,11 +16,9 @@ import org.junit.Test; import org.mockito.Mockito; import java.util.HashMap; -import java.util.concurrent.Executors; import static com.yahoo.container.jdisc.JdiscBindingsConfig.Handlers; -import static org.junit.Assert.assertThat; -import static org.hamcrest.CoreMatchers.containsString; +import static org.junit.Assert.assertTrue; /** * @author gjoranv @@ -30,7 +27,7 @@ import static org.hamcrest.CoreMatchers.containsString; public class ApplicationStatusHandlerTest { @Test - public void application_configs_are_rendered() throws Exception { + public void application_configs_are_rendered() { ApplicationMetadataConfig metaConfig = new ApplicationMetadataConfig( new ApplicationMetadataConfig.Builder() .checksum("abc") @@ -44,28 +41,28 @@ public class ApplicationStatusHandlerTest { .version("v1")); String json = ApplicationStatusHandler.renderApplicationConfigs(metaConfig, userConfig).toString(); - assertThat(json, containsString("version")); - assertThat(json, containsString("meta")); - assertThat(json, containsString("abc")); - assertThat(json, containsString("app")); - assertThat(json, containsString("/a/b/c")); - assertThat(json, containsString("3000")); - assertThat(json, containsString("donald")); - - assertThat(json, containsString("v1")); + assertTrue(json.contains("version")); + assertTrue(json.contains("meta")); + assertTrue(json.contains("abc")); + assertTrue(json.contains("app")); + assertTrue(json.contains("/a/b/c")); + assertTrue(json.contains("3000")); + assertTrue(json.contains("donald")); + + assertTrue(json.contains("v1")); } @Test - public void object_components_are_rendered() throws Exception { + public void object_components_are_rendered() { HashMap<ComponentId, Object> id2object = new HashMap<>(); id2object.put(new ComponentId("myComponent"), new Object()); String json = ApplicationStatusHandler.renderObjectComponents(id2object).toString(); - assertThat(json, containsString("myComponent")); + assertTrue(json.contains("myComponent")); } @Test - public void request_handlers_are_rendered() throws Exception { + public void request_handlers_are_rendered() { final String id = "myHandler"; final String serverBinding1 = "http://*/serverBinding"; final String serverBinding2 = "http://*/anotherServerBinding"; @@ -81,14 +78,14 @@ public class ApplicationStatusHandlerTest { .clientBindings(clientBinding)) ); String json = ApplicationStatusHandler.renderRequestHandlers(bindingsConfig, handlersById).toString(); - assertThat(json, containsString("\"" + id + "\"")); - assertThat(json, containsString(serverBinding1)); - assertThat(json, containsString(serverBinding2)); - assertThat(json, containsString(clientBinding)); + assertTrue(json.contains("\"" + id + "\"")); + assertTrue(json.contains(serverBinding1)); + assertTrue(json.contains(serverBinding2)); + assertTrue(json.contains(clientBinding)); } @Test - public void client_providers_are_rendered() throws Exception { + public void client_providers_are_rendered() { final String id = "myClient"; final String clientBinding = "http://*/clientBinding"; final String clientBinding2 = "http://*/anotherClientBinding"; @@ -105,21 +102,21 @@ public class ApplicationStatusHandlerTest { ); String json = ApplicationStatusHandler.renderRequestHandlers(bindingsConfig, clientsById).toString(); System.out.println(json); - assertThat(json, containsString("\"" + id + "\"")); - assertThat(json, containsString(clientBinding)); - assertThat(json, containsString(clientBinding2)); - assertThat(json, containsString(serverBinding)); + assertTrue(json.contains("\"" + id + "\"")); + assertTrue(json.contains(clientBinding)); + assertTrue(json.contains(clientBinding2)); + assertTrue(json.contains(serverBinding)); } @Test - public void chains_are_rendered() throws Exception { + public void chains_are_rendered() { ChainRegistry<Processor> chains = new ChainRegistry<>(); - Chain<Processor> chain = new Chain<Processor>("myChain", new VoidProcessor(new ComponentId("voidProcessor"))); + Chain<Processor> chain = new Chain<>("myChain", new VoidProcessor(new ComponentId("voidProcessor"))); chains.register(new ComponentId("myChain"), chain); String json = ApplicationStatusHandler.StatusResponse.renderChains(chains).toString(); - assertThat(json, containsString("myChain")); - assertThat(json, containsString("voidProcessor")); + assertTrue(json.contains("myChain")); + assertTrue(json.contains("voidProcessor")); } private static class VoidProcessor extends Processor { diff --git a/container-search/src/main/antlr4/com/yahoo/search/yql/yqlplus.g4 b/container-search/src/main/antlr4/com/yahoo/search/yql/yqlplus.g4 index 38efba3b511..8b0ebcf98fc 100644 --- a/container-search/src/main/antlr4/com/yahoo/search/yql/yqlplus.g4 +++ b/container-search/src/main/antlr4/com/yahoo/search/yql/yqlplus.g4 @@ -90,7 +90,7 @@ options { * LEXER RULES *------------------------------------------------------------------*/ -ID : ('a'..'z'|'A'..'Z'|'_') ('a'..'z'|'A'..'Z'|'0'..'9'|'_'|':'|'-')* +IDENTIFIER : ('a'..'z'|'A'..'Z'|'_') ('a'..'z'|'A'..'Z'|'0'..'9'|'_'|'-')* ; LONG_INT : '-'?'0'..'9'+ ('L'|'l') @@ -117,8 +117,8 @@ LETTER : 'a'..'z' | 'A'..'Z' ; -STRING : '"' ( ESC_SEQ | ~('\\'| '"') )* '"' - | '\'' ( ESC_SEQ | ~('\\' | '\'') )* '\'' +STRING : DQ ( ESC_SEQ | ~('\\'| '"') )* DQ + | SQ ( ESC_SEQ | ~('\\' | '\'') )* SQ ; fragment @@ -166,24 +166,15 @@ VESPA_GROUPING_ARG // --------- parser rules ------------ ident - : keyword_as_ident //{addChild(new TerminalNodeImpl(keyword_as_ident.getText()));} - | ID + : keyword_as_ident + | IDENTIFIER ; keyword_as_ident : SELECT | LIMIT | OFFSET | WHERE | 'order' | 'by' | DESC | OUTPUT | COUNT | SOURCES | MATCHES | LIKE ; -program : (statement SEMI)* EOF - ; - -moduleId - : ID - ; - -moduleName - : literalString - | namespaced_name +program : (statement SEMI?)* EOF ; statement @@ -261,7 +252,7 @@ source_spec ; alias_def - : (AS? ID) + : (AS? IDENTIFIER) ; data_source @@ -428,7 +419,7 @@ indexref[boolean in_select] : LBRACKET idx=expression[in_select] RBRACKET ; propertyref - : DOT nm=ID + : DOT nm=IDENTIFIER ; primaryExpression @@ -459,10 +450,14 @@ propertyNameAndValue ; propertyName - : ID - | literalString + : dottedIdentifiers + | STRING ; +dottedIdentifiers + : IDENTIFIER (DOT IDENTIFIER)* + ; + constantExpression : scalar_literal | mapExpression @@ -483,10 +478,6 @@ scalar_literal | FLOAT ; -literalString - : STRING - ; - array_parameter : AT i=ident {isArrayParameter($i.ctx)}? ; diff --git a/container-search/src/main/java/com/yahoo/search/yql/ExpressionOperator.java b/container-search/src/main/java/com/yahoo/search/yql/ExpressionOperator.java index 06a095df6dd..762d906585c 100644 --- a/container-search/src/main/java/com/yahoo/search/yql/ExpressionOperator.java +++ b/container-search/src/main/java/com/yahoo/search/yql/ExpressionOperator.java @@ -55,7 +55,6 @@ enum ExpressionOperator implements Operator { READ_RECORD(String.class), READ_FIELD(String.class, String.class), - READ_MODULE(TypeCheckers.LIST_OF_STRING), VESPA_GROUPING(String.class), @@ -64,7 +63,7 @@ enum ExpressionOperator implements Operator { private final ArgumentsTypeChecker checker; - private ExpressionOperator(Object... types) { + ExpressionOperator(Object... types) { checker = TypeCheckers.make(this, types); } diff --git a/container-search/src/main/java/com/yahoo/search/yql/ProgramParser.java b/container-search/src/main/java/com/yahoo/search/yql/ProgramParser.java index df582ee8141..74f2894f04c 100644 --- a/container-search/src/main/java/com/yahoo/search/yql/ProgramParser.java +++ b/container-search/src/main/java/com/yahoo/search/yql/ProgramParser.java @@ -389,7 +389,7 @@ final class ProgramParser { OperatorNode<SequenceOperator> result = convertQuery(queryStatementContext.getChild(0), scope.getRoot()); for (Pipeline_stepContext step:nodes) { if (getParseTreeIndex(step.getChild(0)) == yqlplusParser.RULE_vespa_grouping) { - result = OperatorNode.create(SequenceOperator.PIPE, result, ImmutableList.<String>of(), + result = OperatorNode.create(SequenceOperator.PIPE, result, List.of(), ImmutableList.of(convertExpr(step.getChild(0), scope))); } else { List<String> name = readName(step.namespaced_name()); @@ -421,7 +421,6 @@ final class ProgramParser { } else { throw new IllegalArgumentException("Unexpected argument type to convertQueryStatement: " + node.toStringTree()); } - } private String assignAlias(String alias, ParserRuleContext node, Scope scope) { @@ -583,7 +582,7 @@ final class ProgramParser { String aliasName = null; if (rulenode.getChildCount() > 1) { // ^(ALIAS ID) - aliasName = rulenode.alias_def().ID().getText(); + aliasName = rulenode.alias_def().IDENTIFIER().getText(); } proj.addField(aliasName, expr); // no grammar for the other rule types at this time @@ -873,17 +872,11 @@ final class ProgramParser { result = OperatorNode.create(loc, ExpressionOperator.READ_RECORD, alias); start = 1; } - } else if (scope.isBound(alias)) { - return OperatorNode.create(loc, ExpressionOperator.READ_MODULE, scope.getBinding(alias).toPathWith(path.subList(1, path.size()))); } else if (scope.getCursors().size() == 1) { alias = scope.getCursors().iterator().next(); result = OperatorNode.create(loc, ExpressionOperator.READ_FIELD, alias, path.get(0)); start = 1; } else { - // ah ha, we can't end up with a 'loose' UDF call because it - // won't be a module or known alias - // so we need not support implicit imports for constants used in - // UDFs throw new ProgramCompileException(loc, "Unknown field or alias '%s'", alias); } for (int idx = start; idx < path.size(); ++idx) { @@ -933,7 +926,7 @@ final class ProgramParser { case yqlplusParser.LONG_INT: return Long.parseLong(text.substring(0, text.length()-1)); default: - throw new ProgramCompileException("Unknow literal type " + text); + throw new ProgramCompileException("Unknown literal type " + text); } } diff --git a/container-search/src/main/java/com/yahoo/search/yql/ProjectionBuilder.java b/container-search/src/main/java/com/yahoo/search/yql/ProjectionBuilder.java index ac7ac0334cc..e7ec28a9b97 100644 --- a/container-search/src/main/java/com/yahoo/search/yql/ProjectionBuilder.java +++ b/container-search/src/main/java/com/yahoo/search/yql/ProjectionBuilder.java @@ -9,8 +9,8 @@ import java.util.Set; class ProjectionBuilder { - private Map<String, OperatorNode<ExpressionOperator>> fields = Maps.newLinkedHashMap(); - private Set<String> aliasNames = Sets.newHashSet(); + private final Map<String, OperatorNode<ExpressionOperator>> fields = Maps.newLinkedHashMap(); + private final Set<String> aliasNames = Sets.newHashSet(); public void addField(String name, OperatorNode<ExpressionOperator> expr) { String aliasName = name; diff --git a/container-search/src/main/java/com/yahoo/search/yql/VespaSerializer.java b/container-search/src/main/java/com/yahoo/search/yql/VespaSerializer.java index 897bcde4fc3..3e7084acc56 100644 --- a/container-search/src/main/java/com/yahoo/search/yql/VespaSerializer.java +++ b/container-search/src/main/java/com/yahoo/search/yql/VespaSerializer.java @@ -178,7 +178,7 @@ public class VespaSerializer { } destination.append("([{"); serializeOrigin(destination, image, offset, length); - destination.append(", \"").append(AND_SEGMENTING).append("\": true"); + destination.append(", ").append(AND_SEGMENTING).append(": true"); destination.append("}]"); destination.append(PHRASE).append('('); serializeWords(destination, item); @@ -287,7 +287,7 @@ public class VespaSerializer { static String nearAnnotations(NearItem n) { if (n.getDistance() != NearItem.defaultDistance) { - return "[{\"" + DISTANCE + "\": " + n.getDistance() + "}]"; + return "[{" + DISTANCE + ": " + n.getDistance() + "}]"; } else { return ""; } @@ -323,12 +323,12 @@ public class VespaSerializer { StringBuilder b = new StringBuilder(); b.append("[{"); if (item.hasStartAnchor() != item.isStartAnchorDefault()) { - b.append("\"" + START_ANCHOR + "\": " + item.hasStartAnchor()); + b.append(START_ANCHOR + ": " + item.hasStartAnchor()); } if (item.hasEndAnchor() != item.isEndAnchorDefault()) { if (b.length() > 2) b.append(", "); - b.append("\"" + END_ANCHOR + "\": " + item.hasEndAnchor()); + b.append(END_ANCHOR + ": " + item.hasEndAnchor()); } b.append("}]"); return b.toString(); @@ -404,11 +404,11 @@ public class VespaSerializer { int initLen; if (leftOpen && rightOpen) { - boundsAnnotation = "\"" + BOUNDS + "\": " + "\"" + BOUNDS_OPEN + "\""; + boundsAnnotation = BOUNDS + ": " + "\"" + BOUNDS_OPEN + "\""; } else if (leftOpen) { - boundsAnnotation = "\"" + BOUNDS + "\": " + "\"" + BOUNDS_LEFT_OPEN + "\""; + boundsAnnotation = BOUNDS + ": " + "\"" + BOUNDS_LEFT_OPEN + "\""; } else if (rightOpen) { - boundsAnnotation = "\"" + BOUNDS + "\": " + "\"" + BOUNDS_RIGHT_OPEN + "\""; + boundsAnnotation = BOUNDS + ": " + "\"" + BOUNDS_RIGHT_OPEN + "\""; } if (annotations.length() > 0 || boundsAnnotation.length() > 0) { destination.append("([{"); @@ -464,9 +464,7 @@ public class VespaSerializer { image.append('L'); } } catch (NumberFormatException e) { - // somebody has managed to init an IntItem containing noise, - // just give up - return; + // somebody has managed to init an IntItem containing noise, just give up } } } @@ -910,7 +908,7 @@ public class VespaSerializer { } if (!isFromQuery) { comma(destination, initLen); - destination.append('"').append(IMPLICIT_TRANSFORMS).append("\": false"); + destination.append(IMPLICIT_TRANSFORMS).append(": false"); } if (annotations.length() > 0) { comma(destination, initLen); @@ -957,15 +955,15 @@ public class VespaSerializer { double scoreThreshold = w.getScoreThreshold(); double thresholdBoostFactor = w.getThresholdBoostFactor(); if (targetNumHits != 10) { - annotations.append('"').append(TARGET_NUM_HITS).append("\": ").append(targetNumHits); + annotations.append(TARGET_NUM_HITS).append(": ").append(targetNumHits); } if (scoreThreshold != 0) { comma(annotations, 0); - annotations.append('"').append(SCORE_THRESHOLD).append("\": ").append(scoreThreshold); + annotations.append(SCORE_THRESHOLD).append(": ").append(scoreThreshold); } if (thresholdBoostFactor != 1) { comma(annotations, 0); - annotations.append('"').append(THRESHOLD_BOOST_FACTOR).append("\": ").append(thresholdBoostFactor); + annotations.append(THRESHOLD_BOOST_FACTOR).append(": ").append(thresholdBoostFactor); } return annotations.toString(); } @@ -999,11 +997,11 @@ public class VespaSerializer { } int lengthBeforeAnnotations = destination.length(); if (nonDefaultTargetNumHits(item)) { - destination.append('"').append(TARGET_NUM_HITS).append("\": ").append(item.getN()); + destination.append(TARGET_NUM_HITS).append(": ").append(item.getN()); } if (nonDefaultScoreThreshold(item)) { comma(destination, lengthBeforeAnnotations); - destination.append('"').append(SCORE_THRESHOLD).append("\": ").append(item.getScoreThreshold()); + destination.append(SCORE_THRESHOLD).append(": ").append(item.getScoreThreshold()); } if (needsAnnotationBlock(item)) { destination.append("}]"); @@ -1098,41 +1096,41 @@ public class VespaSerializer { if (!image.substring(offset, offset + length).equals(item.getIndexedString())) { VespaSerializer.serializeOrigin(annotation, image, offset, length); } - if (usePositionData != true) { + if ( ! usePositionData) { VespaSerializer.comma(annotation, initLen); - annotation.append('"').append(USE_POSITION_DATA).append("\": false"); + annotation.append(USE_POSITION_DATA).append(": false"); } - if (stemmed == true) { + if (stemmed) { VespaSerializer.comma(annotation, initLen); - annotation.append('"').append(STEM).append("\": false"); + annotation.append(STEM).append(": false"); } - if (lowercased == true) { + if (lowercased) { VespaSerializer.comma(annotation, initLen); - annotation.append('"').append(NORMALIZE_CASE).append("\": false"); + annotation.append(NORMALIZE_CASE).append(": false"); } - if (accentDrop == false) { + if ( ! accentDrop) { VespaSerializer.comma(annotation, initLen); - annotation.append('"').append(ACCENT_DROP).append("\": false"); + annotation.append(ACCENT_DROP).append(": false"); } if (andSegmenting == SegmentingRule.BOOLEAN_AND) { VespaSerializer.comma(annotation, initLen); - annotation.append('"').append(AND_SEGMENTING).append("\": true"); + annotation.append(AND_SEGMENTING).append(": true"); } if (!isFromQuery) { VespaSerializer.comma(annotation, initLen); - annotation.append('"').append(IMPLICIT_TRANSFORMS).append("\": false"); + annotation.append(IMPLICIT_TRANSFORMS).append(": false"); } if (prefix) { VespaSerializer.comma(annotation, initLen); - annotation.append('"').append(PREFIX).append("\": true"); + annotation.append(PREFIX).append(": true"); } if (suffix) { VespaSerializer.comma(annotation, initLen); - annotation.append('"').append(SUFFIX).append("\": true"); + annotation.append(SUFFIX).append(": true"); } if (substring) { VespaSerializer.comma(annotation, initLen); - annotation.append('"').append(SUBSTRING).append("\": true"); + annotation.append(SUBSTRING).append(": true"); } return annotation.toString(); } @@ -1328,7 +1326,7 @@ public class VespaSerializer { private static void serialize(GroupingRequest request, StringBuilder out) { Iterator<Continuation> it = request.continuations().iterator(); if (it.hasNext()) { - out.append("[{ 'continuations':["); + out.append("[{ continuations:["); while (it.hasNext()) { out.append('\'').append(it.next()).append('\''); if (it.hasNext()) { @@ -1413,7 +1411,7 @@ public class VespaSerializer { } private static StringBuilder annotationKey(StringBuilder annotation, String val) { - annotation.append("\"").append(val).append("\": "); + annotation.append(val).append(": "); return annotation; } @@ -1424,8 +1422,7 @@ public class VespaSerializer { } private static String leafAnnotations(TaggableItem item) { - // TODO there is no usable API for the general annotations map in the - // Item instances + // TODO: There is no usable API for the general annotations map in the Item instances StringBuilder annotation = new StringBuilder(); int initLen = annotation.length(); { @@ -1434,19 +1431,19 @@ public class VespaSerializer { TaggableItem connectedTo = (TaggableItem) item.getConnectedItem(); double significance = item.getSignificance(); if (connectedTo != null && connectedTo.getUniqueID() != 0) { - annotation.append('"').append(CONNECTIVITY).append("\": {\"") - .append(CONNECTION_ID).append("\": ") - .append(connectedTo.getUniqueID()).append(", \"") - .append(CONNECTION_WEIGHT).append("\": ") + annotation.append(CONNECTIVITY).append(": {") + .append(CONNECTION_ID).append(": ") + .append(connectedTo.getUniqueID()).append(", ") + .append(CONNECTION_WEIGHT).append(": ") .append(connectivity).append("}"); } if (item.hasExplicitSignificance()) { comma(annotation, initLen); - annotation.append('"').append(SIGNIFICANCE).append("\": ").append(significance); + annotation.append(SIGNIFICANCE).append(": ").append(significance); } if (uniqueId != 0) { comma(annotation, initLen); - annotation.append('"').append(UNIQUE_ID).append("\": ").append(uniqueId); + annotation.append(UNIQUE_ID).append(": ").append(uniqueId); } } { @@ -1456,41 +1453,41 @@ public class VespaSerializer { String label = leaf.getLabel(); int weight = leaf.getWeight(); - if (filter == true) { + if (filter) { comma(annotation, initLen); - annotation.append("\"").append(FILTER).append("\": true"); + annotation.append(FILTER).append(": true"); } - if (isRanked == false) { + if ( ! isRanked) { comma(annotation, initLen); - annotation.append("\"").append(RANKED).append("\": false"); + annotation.append(RANKED).append(": false"); } if (label != null) { comma(annotation, initLen); - annotation.append("\"").append(LABEL).append("\": \""); + annotation.append(LABEL).append(": \""); escape(label, annotation); annotation.append("\""); } if (weight != 100) { comma(annotation, initLen); - annotation.append('"').append(WEIGHT).append("\": ").append(weight); + annotation.append(WEIGHT).append(": ").append(weight); } } if (item instanceof IntItem) { int hitLimit = ((IntItem) item).getHitLimit(); if (hitLimit != 0) { comma(annotation, initLen); - annotation.append('"').append(HIT_LIMIT).append("\": ").append(hitLimit); + annotation.append(HIT_LIMIT).append(": ").append(hitLimit); } } return annotation.toString(); } private static void serializeOrigin(StringBuilder destination, String image, int offset, int length) { - destination.append('"').append(ORIGIN).append("\": {\"").append(ORIGIN_ORIGINAL).append("\": \""); + destination.append(ORIGIN).append(": {").append(ORIGIN_ORIGINAL).append(": \""); escape(image, destination); - destination.append("\", \"").append(ORIGIN_OFFSET).append("\": ") - .append(offset).append(", \"").append(ORIGIN_LENGTH) - .append("\": ").append(length).append("}"); + destination.append("\", '").append(ORIGIN_OFFSET).append("': ") + .append(offset).append(", ").append(ORIGIN_LENGTH) + .append(": ").append(length).append("}"); } private static String normalizeIndexName(String indexName) { diff --git a/container-search/src/test/java/com/yahoo/prelude/searcher/test/ValidatePredicateSearcherTestCase.java b/container-search/src/test/java/com/yahoo/prelude/searcher/test/ValidatePredicateSearcherTestCase.java index 5e3c4897d73..061c9bc5681 100644 --- a/container-search/src/test/java/com/yahoo/prelude/searcher/test/ValidatePredicateSearcherTestCase.java +++ b/container-search/src/test/java/com/yahoo/prelude/searcher/test/ValidatePredicateSearcherTestCase.java @@ -30,7 +30,7 @@ public class ValidatePredicateSearcherTestCase { @Test public void testValidQuery() { ValidatePredicateSearcher searcher = new ValidatePredicateSearcher(); - String q = "select * from sources * where predicate(predicate_field,0,{\"age\":20L});"; + String q = "select * from sources * where predicate(predicate_field,0,{\"age\":20L})"; Result r = doSearch(searcher, q, "predicate-bounds [0..99]"); assertNull(r.hits().getError()); } @@ -38,7 +38,7 @@ public class ValidatePredicateSearcherTestCase { @Test public void testQueryOutOfBounds() { ValidatePredicateSearcher searcher = new ValidatePredicateSearcher(); - String q = "select * from sources * where predicate(predicate_field,0,{\"age\":200L});"; + String q = "select * from sources * where predicate(predicate_field,0,{\"age\":200L})"; Result r = doSearch(searcher, q, "predicate-bounds [0..99]"); assertEquals(ErrorMessage.createIllegalQuery("age=200 outside configured predicate bounds."), r.hits().getError()); } @@ -46,7 +46,7 @@ public class ValidatePredicateSearcherTestCase { @Test public void queryFailsWhenPredicateFieldIsUsedInTermSearch() { ValidatePredicateSearcher searcher = new ValidatePredicateSearcher(); - String q = "select * from sources * where predicate_field CONTAINS \"true\";"; + String q = "select * from sources * where predicate_field CONTAINS \"true\""; Result r = doSearch(searcher, q, "predicate-bounds [0..99]"); assertEquals(ErrorMessage.createIllegalQuery("Index 'predicate_field' is predicate attribute and can only be used in conjunction with a predicate query operator."), r.hits().getError()); } diff --git a/container-search/src/test/java/com/yahoo/search/grouping/request/parser/GroupingParserTestCase.java b/container-search/src/test/java/com/yahoo/search/grouping/request/parser/GroupingParserTestCase.java index 08997244adf..c42aa905dd4 100644 --- a/container-search/src/test/java/com/yahoo/search/grouping/request/parser/GroupingParserTestCase.java +++ b/container-search/src/test/java/com/yahoo/search/grouping/request/parser/GroupingParserTestCase.java @@ -640,7 +640,7 @@ public class GroupingParserTestCase { private static void assertYqlParsable(String request, String... expectedOperations) { YqlParser parser = new YqlParser(new ParserEnvironment()); - parser.parse(new Parsable().setQuery("select foo from bar where baz contains 'baz' | " + request + ";")); + parser.parse(new Parsable().setQuery("select foo from bar where baz contains 'baz' | " + request)); List<VespaGroupingStep> steps = parser.getGroupingSteps(); List<String> actual = new ArrayList<>(steps.size()); for (VespaGroupingStep step : steps) { diff --git a/container-search/src/test/java/com/yahoo/search/handler/test/JSONSearchHandlerTestCase.java b/container-search/src/test/java/com/yahoo/search/handler/test/JSONSearchHandlerTestCase.java index 6079c03ec5c..48dfa98768a 100644 --- a/container-search/src/test/java/com/yahoo/search/handler/test/JSONSearchHandlerTestCase.java +++ b/container-search/src/test/java/com/yahoo/search/handler/test/JSONSearchHandlerTestCase.java @@ -155,7 +155,7 @@ public class JSONSearchHandlerTestCase { assertTrue("Do I have a new instance of the search handler?", searchHandler != newSearchHandler); try (RequestHandlerTestDriver newDriver = new RequestHandlerTestDriver(newSearchHandler)) { ObjectNode json = jsonMapper.createObjectNode(); - json.put("yql", "select * from foo where bar > 1453501295"); + json.put("yql", "selectz * from foo where bar > 1453501295"); RequestHandlerTestDriver.MockResponseHandler responseHandler = newDriver.sendRequest(uri, com.yahoo.jdisc.http.HttpRequest.Method.POST, json.toString(), JSON_CONTENT_TYPE); responseHandler.readAll(); assertThat(responseHandler.getStatus(), is(400)); diff --git a/container-search/src/test/java/com/yahoo/search/searchers/ValidateNearestNeighborTestCase.java b/container-search/src/test/java/com/yahoo/search/searchers/ValidateNearestNeighborTestCase.java index 778b1ea54f3..ef36e16a2b7 100644 --- a/container-search/src/test/java/com/yahoo/search/searchers/ValidateNearestNeighborTestCase.java +++ b/container-search/src/test/java/com/yahoo/search/searchers/ValidateNearestNeighborTestCase.java @@ -99,7 +99,7 @@ public class ValidateNearestNeighborTestCase { } private String makeQuery(String attributeTensor, String queryTensor) { - return "select * from sources * where [{\"targetHits\":1}]nearestNeighbor(" + attributeTensor + ", " + queryTensor + ");"; + return "select * from sources * where [{targetHits:1}]nearestNeighbor(" + attributeTensor + ", " + queryTensor + ")"; } @Test @@ -153,7 +153,7 @@ public class ValidateNearestNeighborTestCase { @Test public void testMissingTargetNumHits() { - String q = "select * from sources * where nearestNeighbor(dvector,qvector);"; + String q = "select * from sources * where nearestNeighbor(dvector,qvector)"; Tensor t = makeTensor(tt_dense_dvector_3); Result r = doSearch(searcher, q, t); assertErrMsg(desc("dvector", "qvector", 0, "has invalid targetHits 0: Must be >= 1"), r); diff --git a/container-search/src/test/java/com/yahoo/search/test/QueryTestCase.java b/container-search/src/test/java/com/yahoo/search/test/QueryTestCase.java index 4b94df17fe0..abea2b0f259 100644 --- a/container-search/src/test/java/com/yahoo/search/test/QueryTestCase.java +++ b/container-search/src/test/java/com/yahoo/search/test/QueryTestCase.java @@ -100,7 +100,7 @@ public class QueryTestCase { @Ignore @Test public void testSimpleProgramParameterAlias() { - Query q = new Query(httpEncode("/sdfsd.html?yql=select * from source where myfield contains(word);")); + Query q = new Query(httpEncode("/sdfsd.html?yql=select * from source where myfield contains(word)")); assertEquals("", q.getModel().getQueryTree().toString()); } @@ -901,7 +901,7 @@ public class QueryTestCase { { // Select all Persons whose hobbies contains 'sport' // YQL - Query yqlQuery = new Query(httpEncode("?query=select * from Persons where hobbies contains 'sports';&type=yql")); + Query yqlQuery = new Query(httpEncode("?query=select * from Persons where hobbies contains 'sports'&type=yql")); assertEquals("hobbies:sports", yqlQuery.getModel().getQueryTree().toString()); // JSON @@ -916,7 +916,7 @@ public class QueryTestCase { { // Select all Persons whose Phones areaCode equals 'NY' // YQL - Query yqlQuery = new Query(httpEncode("?query=select * from Persons where phones.areaCode contains 'NY';&type=yql")); + Query yqlQuery = new Query(httpEncode("?query=select * from Persons where phones.areaCode contains 'NY'&type=yql")); assertEquals("phones.areaCode:NY", yqlQuery.getModel().getQueryTree().toString()); // JSON @@ -931,7 +931,7 @@ public class QueryTestCase { { // Select all Persons whose Mother's Birthyear is greater than 1960 // YQL - Query yqlQuery = new Query(httpEncode("?query=select * from Persons where mother.Birthyear > 1960;&type=yql")); + Query yqlQuery = new Query(httpEncode("?query=select * from Persons where mother.Birthyear > 1960&type=yql")); assertEquals("mother.Birthyear:>1960", yqlQuery.getModel().getQueryTree().toString()); // JSON diff --git a/container-search/src/test/java/com/yahoo/search/yql/FieldFilterTestCase.java b/container-search/src/test/java/com/yahoo/search/yql/FieldFilterTestCase.java index b1093e54b68..8a6a48afd9f 100644 --- a/container-search/src/test/java/com/yahoo/search/yql/FieldFilterTestCase.java +++ b/container-search/src/test/java/com/yahoo/search/yql/FieldFilterTestCase.java @@ -42,7 +42,7 @@ public class FieldFilterTestCase { DocumentSourceSearcher mockBackend = new DocumentSourceSearcher(); mockBackend.addResult(query, result); - searchChain = new Chain<Searcher>(new FieldFilter(), mockBackend); + searchChain = new Chain<>(new FieldFilter(), mockBackend); context = Execution.Context.createContextStub(); execution = new Execution(searchChain, context); diff --git a/container-search/src/test/java/com/yahoo/search/yql/MinimalQueryInserterTestCase.java b/container-search/src/test/java/com/yahoo/search/yql/MinimalQueryInserterTestCase.java index 09d1dc71249..36e8830004b 100644 --- a/container-search/src/test/java/com/yahoo/search/yql/MinimalQueryInserterTestCase.java +++ b/container-search/src/test/java/com/yahoo/search/yql/MinimalQueryInserterTestCase.java @@ -68,7 +68,7 @@ public class MinimalQueryInserterTestCase { assertEquals("foo", query.getPresentation().getSummaryFields().toArray(new String[1])[0]); builder.setParameter("yql", "select foo from bar where baz contains 'cox' " + - "| all(group(a) each(output(count())));"); + "| all(group(a) each(output(count())))"); query = new Query(builder.toString()); execution.search(query); assertEquals("baz:cox", query.getModel().getQueryTree().toString()); @@ -76,7 +76,7 @@ public class MinimalQueryInserterTestCase { builder.setParameter("yql", "select foo from bar where baz contains 'cox' " + "| all(group(a) each(output(count()))) " + - "| all(group(b) each(output(count())));"); + "| all(group(b) each(output(count())))"); query = new Query(builder.toString()); execution.search(query); assertEquals("baz:cox", query.getModel().getQueryTree().toString()); @@ -89,25 +89,25 @@ public class MinimalQueryInserterTestCase { URIBuilder builder = new URIBuilder(); builder.setPath("search/"); - builder.setParameter("yql", "select foo from bar where baz contains 'cox';"); + builder.setParameter("yql", "select foo from bar where baz contains 'cox'"); Query query = new Query(builder.toString()); execution.search(query); assertEquals("baz:cox", query.getModel().getQueryTree().toString()); assertGrouping("[]", query); builder.setParameter("yql", "select foo from bar where baz contains 'cox' " + - "| [{ 'continuations':['BCBCBCBEBG', 'BCBKCBACBKCCK'] }]" + - "all(group(a) each(output(count())));"); + "| [{ continuations:['BCBCBCBEBG', 'BCBKCBACBKCCK'] }]" + + "all(group(a) each(output(count())))"); query = new Query(builder.toString()); execution.search(query); assertEquals("baz:cox", query.getModel().getQueryTree().toString()); assertGrouping("[[BCBCBCBEBG, BCBKCBACBKCCK]all(group(a) each(output(count())))]", query); builder.setParameter("yql", "select foo from bar where baz contains 'cox' " + - "| [{ 'continuations':['BCBCBCBEBG', 'BCBKCBACBKCCK'] }]" + + "| [{ continuations:['BCBCBCBEBG', 'BCBKCBACBKCCK'] }]" + "all(group(a) each(output(count()))) " + - "| [{ 'continuations':['BCBBBBBDBF', 'BCBJBPCBJCCJ'] }]" + - "all(group(b) each(output(count())));"); + "| [{ continuations:['BCBBBBBDBF', 'BCBJBPCBJCCJ'] }]" + + "all(group(b) each(output(count())))"); query = new Query(builder.toString()); execution.search(query); assertEquals("baz:cox", query.getModel().getQueryTree().toString()); @@ -119,14 +119,14 @@ public class MinimalQueryInserterTestCase { @Ignore // TODO: YQL work in progress (jon) public void testTmp() { - Query query = new Query("search/?query=easilyRecognizedString&yql=select%20ignoredfield%20from%20ignoredsource%20where%20title%20contains%20%22madonna%22%20and%20userQuery()%3B"); + Query query = new Query("search/?query=easilyRecognizedString&yql=select%20ignoredfield%20from%20ignoredsource%20where%20title%20contains%20%22madonna%22%20and%20userQuery()"); //execution.search(query); assertEquals("AND title:madonna easilyRecognizedString", query.getModel().getQueryTree().toString()); } @Test public void testSearch() { - Query query = new Query("search/?query=easilyRecognizedString&yql=select%20ignoredfield%20from%20ignoredsource%20where%20title%20contains%20%22madonna%22%20and%20userQuery()%3B"); + Query query = new Query("search/?query=easilyRecognizedString&yql=select%20ignoredfield%20from%20ignoredsource%20where%20title%20contains%20%22madonna%22%20and%20userQuery()"); execution.search(query); assertEquals("AND title:madonna easilyRecognizedString", query.getModel().getQueryTree().toString()); assertEquals(Language.ENGLISH, query.getModel().getParsingLanguage()); @@ -135,7 +135,7 @@ public class MinimalQueryInserterTestCase { @Test public void testExplicitLanguageIsHonoredWithVerbatimQuery() { String japaneseWord = "\u30ab\u30bf\u30ab\u30ca"; - Query query = new Query("search/?language=ja" + "&yql=select%20ignoredField%20from%20ignoredsource%20where%20title%20contains%20%22" + encode(japaneseWord) + "%22%3B"); + Query query = new Query("search/?language=ja" + "&yql=select%20ignoredField%20from%20ignoredsource%20where%20title%20contains%20%22" + encode(japaneseWord) + "%22"); execution.search(query); assertEquals(Language.JAPANESE, query.getModel().getParsingLanguage()); assertEquals("title:"+ japaneseWord, query.getModel().getQueryTree().toString()); @@ -144,7 +144,7 @@ public class MinimalQueryInserterTestCase { @Test public void testUserLanguageIsDetectedWithVerbatimQuery() { String japaneseWord = "\u30ab\u30bf\u30ab\u30ca"; - Query query = new Query("search/?yql=select%20ignoredField%20from%20ignoredsource%20where%20title%20contains%20%22" + encode(japaneseWord) + "%22%3B"); + Query query = new Query("search/?yql=select%20ignoredField%20from%20ignoredsource%20where%20title%20contains%20%22" + encode(japaneseWord) + "%22"); execution.search(query); assertEquals(Language.JAPANESE, query.getModel().getParsingLanguage()); assertEquals("title:"+ japaneseWord, query.getModel().getQueryTree().toString()); @@ -153,7 +153,7 @@ public class MinimalQueryInserterTestCase { @Test public void testUserLanguageIsDetectedWithUserInput() { String japaneseWord = "\u30ab\u30bf\u30ab\u30ca"; - Query query = new Query("search/?userString=" + encode(japaneseWord) + "&yql=select%20ignoredfield%20from%20ignoredsource%20where%20title%20contains%20%22madonna%22%20and%20userInput(@userString)%3B"); + Query query = new Query("search/?userString=" + encode(japaneseWord) + "&yql=select%20ignoredfield%20from%20ignoredsource%20where%20title%20contains%20%22madonna%22%20and%20userInput(@userString)"); execution.search(query); assertEquals(Language.JAPANESE, query.getModel().getParsingLanguage()); assertEquals("AND title:madonna default:" + japaneseWord, query.getModel().getQueryTree().toString()); @@ -162,7 +162,7 @@ public class MinimalQueryInserterTestCase { @Test public void testUserLanguageIsDetectedWithUserQuery() { String japaneseWord = "\u30ab\u30bf\u30ab\u30ca"; - Query query = new Query("search/?query=" + encode(japaneseWord) + "&yql=select%20ignoredfield%20from%20ignoredsource%20where%20title%20contains%20%22madonna%22%20and%20userQuery()%3B"); + Query query = new Query("search/?query=" + encode(japaneseWord) + "&yql=select%20ignoredfield%20from%20ignoredsource%20where%20title%20contains%20%22madonna%22%20and%20userQuery()"); execution.search(query); assertEquals(Language.JAPANESE, query.getModel().getParsingLanguage()); assertEquals("AND title:madonna " + japaneseWord, query.getModel().getQueryTree().toString()); @@ -170,14 +170,14 @@ public class MinimalQueryInserterTestCase { @Test public void testUserQueryFailsWithoutArgument() { - Query query = new Query("search/?query=easilyRecognizedString&yql=select%20ignoredfield%20from%20ignoredsource%20where%20title%20contains%20%22madonna%22%20and%20userQuery()%3B"); + Query query = new Query("search/?query=easilyRecognizedString&yql=select%20ignoredfield%20from%20ignoredsource%20where%20title%20contains%20%22madonna%22%20and%20userQuery()"); execution.search(query); assertEquals("AND title:madonna easilyRecognizedString", query.getModel().getQueryTree().toString()); } @Test public void testSearchFromAllSourcesWithUserSource() { - Query query = new Query("search/?query=easilyRecognizedString&sources=abc&yql=select%20ignoredfield%20from%20sources%20*%20where%20title%20contains%20%22madonna%22%20and%20userQuery()%3B"); + Query query = new Query("search/?query=easilyRecognizedString&sources=abc&yql=select%20ignoredfield%20from%20sources%20*%20where%20title%20contains%20%22madonna%22%20and%20userQuery()"); execution.search(query); assertEquals("AND title:madonna easilyRecognizedString", query.getModel().getQueryTree().toString()); assertEquals(0, query.getModel().getSources().size()); @@ -185,7 +185,7 @@ public class MinimalQueryInserterTestCase { @Test public void testSearchFromAllSourcesWithoutUserSource() { - Query query = new Query("search/?query=easilyRecognizedString&yql=select%20ignoredfield%20from%20sources%20*%20where%20title%20contains%20%22madonna%22%20and%20userQuery()%3B"); + Query query = new Query("search/?query=easilyRecognizedString&yql=select%20ignoredfield%20from%20sources%20*%20where%20title%20contains%20%22madonna%22%20and%20userQuery()"); execution.search(query); assertEquals("AND title:madonna easilyRecognizedString", query.getModel().getQueryTree().toString()); assertEquals(0, query.getModel().getSources().size()); @@ -193,7 +193,7 @@ public class MinimalQueryInserterTestCase { @Test public void testSearchFromSomeSourcesWithoutUserSource() { - Query query = new Query("search/?query=easilyRecognizedString&yql=select%20ignoredfield%20from%20sources%20sourceA,%20sourceB%20where%20title%20contains%20%22madonna%22%20and%20userQuery()%3B"); + Query query = new Query("search/?query=easilyRecognizedString&yql=select%20ignoredfield%20from%20sources%20sourceA,%20sourceB%20where%20title%20contains%20%22madonna%22%20and%20userQuery()"); execution.search(query); assertEquals("AND title:madonna easilyRecognizedString", query.getModel().getQueryTree().toString()); assertEquals(2, query.getModel().getSources().size()); @@ -203,7 +203,7 @@ public class MinimalQueryInserterTestCase { @Test public void testSearchFromSomeSourcesWithUserSource() { - Query query = new Query("search/?query=easilyRecognizedString&sources=abc&yql=select%20ignoredfield%20from%20sources%20sourceA,%20sourceB%20where%20title%20contains%20%22madonna%22%20and%20userQuery()%3B"); + Query query = new Query("search/?query=easilyRecognizedString&sources=abc&yql=select%20ignoredfield%20from%20sources%20sourceA,%20sourceB%20where%20title%20contains%20%22madonna%22%20and%20userQuery()"); execution.search(query); assertEquals("AND title:madonna easilyRecognizedString", query.getModel().getQueryTree().toString()); assertEquals(3, query.getModel().getSources().size()); @@ -214,7 +214,7 @@ public class MinimalQueryInserterTestCase { @Test public final void testSearchFromSomeSourcesWithOverlappingUserSource() { - final Query query = new Query("search/?query=easilyRecognizedString&sources=abc,sourceA&yql=select%20ignoredfield%20from%20sources%20sourceA,%20sourceB%20where%20title%20contains%20%22madonna%22%20and%20userQuery()%3B"); + final Query query = new Query("search/?query=easilyRecognizedString&sources=abc,sourceA&yql=select%20ignoredfield%20from%20sources%20sourceA,%20sourceB%20where%20title%20contains%20%22madonna%22%20and%20userQuery()"); execution.search(query); assertEquals("AND title:madonna easilyRecognizedString", query.getModel().getQueryTree().toString()); assertEquals(3, query.getModel().getSources().size()); @@ -225,7 +225,7 @@ public class MinimalQueryInserterTestCase { @Test public void testLimitAndOffset() { - Query query = new Query("search/?yql=select%20*%20from%20sources%20*%20where%20title%20contains%20%22madonna%22%20limit%2031offset%207%3B"); + Query query = new Query("search/?yql=select%20*%20from%20sources%20*%20where%20title%20contains%20%22madonna%22%20limit%2031offset%207"); execution.search(query); assertEquals(7, query.getOffset()); assertEquals(24, query.getHits()); @@ -235,7 +235,7 @@ public class MinimalQueryInserterTestCase { @Test public void testMaxOffset() { - Query query = new Query("search/?yql=select%20*%20from%20sources%20*%20where%20title%20contains%20%22madonna%22%20limit%2040031offset%2040000%3B"); + Query query = new Query("search/?yql=select%20*%20from%20sources%20*%20where%20title%20contains%20%22madonna%22%20limit%2040031offset%2040000"); Result r = execution.search(query); assertEquals(1, r.hits().getErrorHit().errors().size()); ErrorMessage e = r.hits().getErrorHit().errorIterator().next(); @@ -245,7 +245,7 @@ public class MinimalQueryInserterTestCase { @Test public void testMaxLimit() { - Query query = new Query("search/?yql=select%20*%20from%20sources%20*%20where%20title%20contains%20%22madonna%22%20limit%2040000offset%207%3B"); + Query query = new Query("search/?yql=select%20*%20from%20sources%20*%20where%20title%20contains%20%22madonna%22%20limit%2040000offset%207"); Result r = execution.search(query); assertEquals(1, r.hits().getErrorHit().errors().size()); ErrorMessage e = r.hits().getErrorHit().errorIterator().next(); @@ -255,7 +255,7 @@ public class MinimalQueryInserterTestCase { @Test public void testTimeout() { - Query query = new Query("search/?yql=select%20*%20from%20sources%20*%20where%20title%20contains%20%22madonna%22%20timeout%2051%3B"); + Query query = new Query("search/?yql=select%20*%20from%20sources%20*%20where%20title%20contains%20%22madonna%22%20timeout%2051"); execution.search(query); assertEquals(51L, query.getTimeout()); assertEquals("select * from sources * where title contains \"madonna\" timeout 51;", query.yqlRepresentation()); @@ -281,7 +281,7 @@ public class MinimalQueryInserterTestCase { @Test public void testOrdering() { { - String yql = "select%20ignoredfield%20from%20ignoredsource%20where%20title%20contains%20%22madonna%22%20order%20by%20something%2C%20shoesize%20desc%20limit%20300%20timeout%203%3B"; + String yql = "select%20ignoredfield%20from%20ignoredsource%20where%20title%20contains%20%22madonna%22%20order%20by%20something%2C%20shoesize%20desc%20limit%20300%20timeout%203"; Query query = new Query("search/?yql=" + yql); execution.search(query); assertEquals(2, query.getRanking().getSorting().fieldOrders() @@ -297,7 +297,7 @@ public class MinimalQueryInserterTestCase { assertEquals("select ignoredfield from ignoredsource where title contains \"madonna\" order by something, shoesize desc limit 300 timeout 3;", query.yqlRepresentation()); } { - String yql = "select%20ignoredfield%20from%20ignoredsource%20where%20title%20contains%20%22madonna%22%20order%20by%20other%20limit%20300%20timeout%203%3B"; + String yql = "select%20ignoredfield%20from%20ignoredsource%20where%20title%20contains%20%22madonna%22%20order%20by%20other%20limit%20300%20timeout%203"; Query query = new Query("search/?yql=" + yql); execution.search(query); assertEquals("other", query.getRanking().getSorting().fieldOrders() @@ -307,7 +307,7 @@ public class MinimalQueryInserterTestCase { assertEquals("select ignoredfield from ignoredsource where title contains \"madonna\" order by other limit 300 timeout 3;", query.yqlRepresentation()); } { - String yql = "select%20foo%20from%20bar%20where%20title%20contains%20%22madonna%22%20order%20by%20%5B%7B%22function%22%3A%20%22uca%22%2C%20%22locale%22%3A%20%22en_US%22%2C%20%22strength%22%3A%20%22IDENTICAL%22%7D%5Dother%20desc%2C%20%5B%7B%22function%22%3A%20%22lowercase%22%7D%5Dsomething%20limit%20300%20timeout%203%3B"; + String yql = "select%20foo%20from%20bar%20where%20title%20contains%20%22madonna%22%20order%20by%20%5B%7B%22function%22%3A%20%22uca%22%2C%20%22locale%22%3A%20%22en_US%22%2C%20%22strength%22%3A%20%22IDENTICAL%22%7D%5Dother%20desc%2C%20%5B%7B%22function%22%3A%20%22lowercase%22%7D%5Dsomething%20limit%20300%20timeout%203"; Query query = new Query("search/?yql=" + yql); execution.search(query); { @@ -334,7 +334,7 @@ public class MinimalQueryInserterTestCase { @Test public void testStringRepresentation() { - String yql = "select%20ignoredfield%20from%20ignoredsource%20where%20title%20contains%20%22madonna%22%20order%20by%20something%2C%20shoesize%20desc%20limit%20300%20timeout%203%3B"; + String yql = "select%20ignoredfield%20from%20ignoredsource%20where%20title%20contains%20%22madonna%22%20order%20by%20something%2C%20shoesize%20desc%20limit%20300%20timeout%203"; Query query = new Query("search/?yql=" + yql); execution.search(query); assertEquals("select ignoredfield from ignoredsource where title contains \"madonna\" order by something, shoesize desc limit 300 timeout 3;", @@ -345,7 +345,7 @@ public class MinimalQueryInserterTestCase { public void testAndSegmenting() { Query query = new Query("?yql=select%20%2A%20from%20sources%20%2A%20where%20%5B%7B%22defaultIndex%22%3A%20%22default%22%2C%22grammar%22%3A%20%22web%22%2C%22stem%22%3A%20true%2C%22allowEmpty%22%3A%20true%7D%5DuserInput%28%40animal%29%3B&animal=m%26m%27s&tracelevel=3"); execution.search(query); - assertEquals("select * from sources * where (default contains \"m\" AND default contains ([{\"origin\": {\"original\": \"m\\'s\", \"offset\": 0, \"length\": 3}, \"andSegmenting\": true}]phrase(\"m\", \"s\")));", + assertEquals("select * from sources * where (default contains \"m\" AND default contains ([{origin: {original: \"m\\'s\", 'offset': 0, length: 3}, andSegmenting: true}]phrase(\"m\", \"s\")));", query.yqlRepresentation()); } diff --git a/container-search/src/test/java/com/yahoo/search/yql/UserInputTestCase.java b/container-search/src/test/java/com/yahoo/search/yql/UserInputTestCase.java index 3eb6f5e7e38..b220be83b18 100644 --- a/container-search/src/test/java/com/yahoo/search/yql/UserInputTestCase.java +++ b/container-search/src/test/java/com/yahoo/search/yql/UserInputTestCase.java @@ -48,24 +48,21 @@ public class UserInputTestCase { public void testSimpleUserInput() { { URIBuilder builder = searchUri(); - builder.setParameter("yql", - "select * from sources * where userInput(\"nalle\");"); + builder.setParameter("yql", "select * from sources * where userInput(\"nalle\")"); Query query = searchAndAssertNoErrors(builder); assertEquals("select * from sources * where default contains \"nalle\";", query.yqlRepresentation()); } { URIBuilder builder = searchUri(); builder.setParameter("nalle", "bamse"); - builder.setParameter("yql", - "select * from sources * where userInput(@nalle);"); + builder.setParameter("yql", "select * from sources * where userInput(@nalle)"); Query query = searchAndAssertNoErrors(builder); assertEquals("select * from sources * where default contains \"bamse\";", query.yqlRepresentation()); } { URIBuilder builder = searchUri(); builder.setParameter("nalle", "bamse"); - builder.setParameter("yql", - "select * from sources * where userInput(nalle);"); + builder.setParameter("yql", "select * from sources * where userInput(nalle)"); Query query = new Query(builder.toString()); Result r = execution.search(query); assertNotNull(r.hits().getError()); @@ -75,8 +72,7 @@ public class UserInputTestCase { @Test public void testRawUserInput() { URIBuilder builder = searchUri(); - builder.setParameter("yql", - "select * from sources * where [{\"grammar\": \"raw\"}]userInput(\"nal le\");"); + builder.setParameter("yql", "select * from sources * where [{grammar: \"raw\"}]userInput(\"nal le\")"); Query query = searchAndAssertNoErrors(builder); assertEquals("select * from sources * where default contains \"nal le\";", query.yqlRepresentation()); } @@ -85,16 +81,16 @@ public class UserInputTestCase { public void testSegmentedUserInput() { URIBuilder builder = searchUri(); builder.setParameter("yql", - "select * from sources * where [{\"grammar\": \"segment\"}]userInput(\"nal le\");"); + "select * from sources * where [{grammar: \"segment\"}]userInput(\"nal le\")"); Query query = searchAndAssertNoErrors(builder); - assertEquals("select * from sources * where default contains ([{\"origin\": {\"original\": \"nal le\", \"offset\": 0, \"length\": 6}}]phrase(\"nal\", \"le\"));", query.yqlRepresentation()); + assertEquals("select * from sources * where default contains ([{origin: {original: \"nal le\", 'offset': 0, length: 6}}]phrase(\"nal\", \"le\"));", query.yqlRepresentation()); } @Test public void testSegmentedNoiseUserInput() { URIBuilder builder = searchUri(); builder.setParameter("yql", - "select * from sources * where [{\"grammar\": \"segment\"}]userInput(\"^^^^^^^^\");"); + "select * from sources * where [{grammar: \"segment\"}]userInput(\"^^^^^^^^\")"); Query query = searchAndAssertNoErrors(builder); assertEquals("select * from sources * where default contains \"^^^^^^^^\";", query.yqlRepresentation()); } @@ -103,7 +99,7 @@ public class UserInputTestCase { public void testCustomDefaultIndexUserInput() { URIBuilder builder = searchUri(); builder.setParameter("yql", - "select * from sources * where [{\"defaultIndex\": \"glompf\"}]userInput(\"nalle\");"); + "select * from sources * where [{defaultIndex: \"glompf\"}]userInput(\"nalle\")"); Query query = searchAndAssertNoErrors(builder); assertEquals("select * from sources * where glompf contains \"nalle\";", query.yqlRepresentation()); } @@ -112,10 +108,10 @@ public class UserInputTestCase { public void testAnnotatedUserInputStemming() { URIBuilder builder = searchUri(); builder.setParameter("yql", - "select * from sources * where [{\"stem\": false}]userInput(\"nalle\");"); + "select * from sources * where [{stem: false}]userInput(\"nalle\")"); Query query = searchAndAssertNoErrors(builder); assertEquals( - "select * from sources * where default contains ([{\"stem\": false}]\"nalle\");", + "select * from sources * where default contains ([{stem: false}]\"nalle\");", query.yqlRepresentation()); } @@ -124,7 +120,7 @@ public class UserInputTestCase { URIBuilder builder = searchUri(); builder.setParameter("myinput", "-5"); builder.setParameter("yql", - "select * from ecitem where rank(([{\"defaultIndex\":\"myfield\"}](userInput(@myinput))));"); + "select * from ecitem where rank(([{defaultIndex:\"myfield\"}](userInput(@myinput))))"); Query query = searchAndAssertNoErrors(builder); assertEquals("select * from ecitem where rank(myfield = (-5));", query.yqlRepresentation()); assertEquals("RANK myfield:-5", query.getModel().getQueryTree().getRoot().toString()); @@ -134,10 +130,10 @@ public class UserInputTestCase { public void testAnnotatedUserInputUnrankedTerms() { URIBuilder builder = searchUri(); builder.setParameter("yql", - "select * from sources * where [{\"ranked\": false}]userInput(\"nalle\");"); + "select * from sources * where [{ranked: false}]userInput(\"nalle\")"); Query query = searchAndAssertNoErrors(builder); assertEquals( - "select * from sources * where default contains ([{\"ranked\": false}]\"nalle\");", + "select * from sources * where default contains ([{ranked: false}]\"nalle\");", query.yqlRepresentation()); } @@ -145,10 +141,10 @@ public class UserInputTestCase { public void testAnnotatedUserInputFiltersTerms() { URIBuilder builder = searchUri(); builder.setParameter("yql", - "select * from sources * where [{\"filter\": true}]userInput(\"nalle\");"); + "select * from sources * where [{filter: true}]userInput(\"nalle\")"); Query query = searchAndAssertNoErrors(builder); assertEquals( - "select * from sources * where default contains ([{\"filter\": true}]\"nalle\");", + "select * from sources * where default contains ([{filter: true}]\"nalle\");", query.yqlRepresentation()); } @@ -157,10 +153,10 @@ public class UserInputTestCase { URIBuilder builder = searchUri(); builder.setParameter( "yql", - "select * from sources * where [{\"normalizeCase\": false}]userInput(\"nalle\");"); + "select * from sources * where [{normalizeCase: false}]userInput(\"nalle\")"); Query query = searchAndAssertNoErrors(builder); assertEquals( - "select * from sources * where default contains ([{\"normalizeCase\": false}]\"nalle\");", + "select * from sources * where default contains ([{normalizeCase: false}]\"nalle\");", query.yqlRepresentation()); } @@ -168,10 +164,10 @@ public class UserInputTestCase { public void testAnnotatedUserInputAccentRemoval() { URIBuilder builder = searchUri(); builder.setParameter("yql", - "select * from sources * where [{\"accentDrop\": false}]userInput(\"nalle\");"); + "select * from sources * where [{accentDrop: false}]userInput(\"nalle\")"); Query query = searchAndAssertNoErrors(builder); assertEquals( - "select * from sources * where default contains ([{\"accentDrop\": false}]\"nalle\");", + "select * from sources * where default contains ([{accentDrop: false}]\"nalle\");", query.yqlRepresentation()); } @@ -179,10 +175,10 @@ public class UserInputTestCase { public void testAnnotatedUserInputPositionData() { URIBuilder builder = searchUri(); builder.setParameter("yql", - "select * from sources * where [{\"usePositionData\": false}]userInput(\"nalle\");"); + "select * from sources * where [{usePositionData: false}]userInput(\"nalle\")"); Query query = searchAndAssertNoErrors(builder); assertEquals( - "select * from sources * where default contains ([{\"usePositionData\": false}]\"nalle\");", + "select * from sources * where default contains ([{usePositionData: false}]\"nalle\");", query.yqlRepresentation()); } @@ -192,7 +188,7 @@ public class UserInputTestCase { builder.setParameter("nalle", "bamse"); builder.setParameter("meta", "syntactic"); builder.setParameter("yql", - "select * from sources * where foo contains @nalle and foo contains phrase(@nalle, @meta, @nalle);"); + "select * from sources * where foo contains @nalle and foo contains phrase(@nalle, @meta, @nalle)"); Query query = searchAndAssertNoErrors(builder); assertEquals("select * from sources * where (foo contains \"bamse\" AND foo contains phrase(\"bamse\", \"syntactic\", \"bamse\"));", query.yqlRepresentation()); } @@ -201,7 +197,7 @@ public class UserInputTestCase { public void testReferenceInComparison() { URIBuilder builder = searchUri(); builder.setParameter("varref", "1980"); - builder.setParameter("yql", "select * from sources * where year > @varref;"); + builder.setParameter("yql", "select * from sources * where year > @varref"); Query query = searchAndAssertNoErrors(builder); assertEquals("select * from sources * where year > 1980;", query.yqlRepresentation()); } @@ -212,9 +208,9 @@ public class UserInputTestCase { builder.setParameter("continuation", "BCBCBCBEBG"); builder.setParameter("yql", "select * from sources * where myfield contains 'token'" + - "| [{'continuations':[@continuation, 'BCBKCBACBKCCK'] }] all(group(f) each(output(count())));"); + "| [{'continuations':[@continuation, 'BCBKCBACBKCCK'] }] all(group(f) each(output(count())))"); Query query = searchAndAssertNoErrors(builder); - assertEquals("select * from sources * where myfield contains \"token\" | [{ 'continuations':['BCBCBCBEBG', 'BCBKCBACBKCCK'] }]all(group(f) each(output(count())));", query.yqlRepresentation()); + assertEquals("select * from sources * where myfield contains \"token\" | [{ continuations:['BCBCBCBEBG', 'BCBKCBACBKCCK'] }]all(group(f) each(output(count())));", query.yqlRepresentation()); } @Test @@ -222,7 +218,7 @@ public class UserInputTestCase { URIBuilder builder = searchUri(); builder.setParameter("term", "A"); builder.setParameter("yql", - "select foo from bar where fieldName contains equiv(@term,'B');"); + "select foo from bar where fieldName contains equiv(@term,'B')"); Query query = searchAndAssertNoErrors(builder); assertEquals("select foo from bar where fieldName contains equiv(\"A\", \"B\");", query.yqlRepresentation()); } @@ -249,7 +245,7 @@ public class UserInputTestCase { @Test public void testEmptyUserInput() { URIBuilder builder = searchUri(); - builder.setParameter("yql", "select * from sources * where userInput(\"\");"); + builder.setParameter("yql", "select * from sources * where userInput(\"\")"); assertQueryFails(builder); } @@ -257,7 +253,7 @@ public class UserInputTestCase { public void testEmptyUserInputFromQueryProperty() { URIBuilder builder = searchUri(); builder.setParameter("foo", ""); - builder.setParameter("yql", "select * from sources * where userInput(@foo);"); + builder.setParameter("yql", "select * from sources * where userInput(@foo)"); assertQueryFails(builder); } @@ -265,7 +261,7 @@ public class UserInputTestCase { public void testEmptyQueryProperty() { URIBuilder builder = searchUri(); builder.setParameter("foo", ""); - builder.setParameter("yql", "select * from sources * where bar contains \"a\" and nonEmpty(foo contains @foo);"); + builder.setParameter("yql", "select * from sources * where bar contains \"a\" and nonEmpty(foo contains @foo)"); assertQueryFails(builder); } @@ -274,17 +270,17 @@ public class UserInputTestCase { URIBuilder builder = searchUri(); builder.setParameter("foo", ""); builder.setParameter("yql", - "select * from sources * where bar contains \"a\" and nonEmpty(bar contains \"bar\" and foo contains @foo);"); + "select * from sources * where bar contains \"a\" and nonEmpty(bar contains \"bar\" and foo contains @foo)"); assertQueryFails(builder); } @Test public void testCompositeWithoutArguments() { URIBuilder builder = searchUri(); - builder.setParameter("yql", "select * from sources * where bar contains \"a\" and foo contains phrase();"); + builder.setParameter("yql", "select * from sources * where bar contains \"a\" and foo contains phrase()"); searchAndAssertNoErrors(builder); builder = searchUri(); - builder.setParameter("yql", "select * from sources * where bar contains \"a\" and nonEmpty(foo contains phrase());"); + builder.setParameter("yql", "select * from sources * where bar contains \"a\" and nonEmpty(foo contains phrase())"); assertQueryFails(builder); } @@ -292,7 +288,7 @@ public class UserInputTestCase { public void testAnnoyingPlacementOfNonEmpty() { URIBuilder builder = searchUri(); builder.setParameter("yql", - "select * from sources * where bar contains \"a\" and foo contains nonEmpty(phrase(\"a\", \"b\"));"); + "select * from sources * where bar contains \"a\" and foo contains nonEmpty(phrase(\"a\", \"b\"))"); assertQueryFails(builder); } @@ -305,7 +301,7 @@ public class UserInputTestCase { public void testAllowEmptyUserInput() { URIBuilder builder = searchUri(); builder.setParameter("foo", ""); - builder.setParameter("yql", "select * from sources * where [{\"allowEmpty\": true}]userInput(@foo);"); + builder.setParameter("yql", "select * from sources * where [{allowEmpty: true}]userInput(@foo)"); searchAndAssertNoErrors(builder); } @@ -313,7 +309,7 @@ public class UserInputTestCase { public void testAllowEmptyNullFromQueryParsing() { URIBuilder builder = searchUri(); builder.setParameter("foo", ",,,,,,,,"); - builder.setParameter("yql", "select * from sources * where [{\"allowEmpty\": true}]userInput(@foo);"); + builder.setParameter("yql", "select * from sources * where [{allowEmpty: true}]userInput(@foo)"); searchAndAssertNoErrors(builder); } @@ -321,7 +317,7 @@ public class UserInputTestCase { public void testDisallowEmptyNullFromQueryParsing() { URIBuilder builder = searchUri(); builder.setParameter("foo", ",,,,,,,,"); - builder.setParameter("yql", "select * from sources * where userInput(@foo);"); + builder.setParameter("yql", "select * from sources * where userInput(@foo)"); assertQueryFails(builder); } diff --git a/container-search/src/test/java/com/yahoo/search/yql/VespaSerializerTestCase.java b/container-search/src/test/java/com/yahoo/search/yql/VespaSerializerTestCase.java index cc13658648d..d1759daae47 100644 --- a/container-search/src/test/java/com/yahoo/search/yql/VespaSerializerTestCase.java +++ b/container-search/src/test/java/com/yahoo/search/yql/VespaSerializerTestCase.java @@ -48,18 +48,18 @@ public class VespaSerializerTestCase { public void requireThatGroupingRequestsAreSerialized() { Query query = new Query(); query.getModel().getQueryTree().setRoot(new WordItem("foo")); - assertEquals("default contains ([{\"implicitTransforms\": false}]\"foo\")", + assertEquals("default contains ([{implicitTransforms: false}]\"foo\")", VespaSerializer.serialize(query)); newGroupingRequest(query, new AllOperation().setGroupBy(new AttributeFunction("a")) .addChild(new EachOperation().addOutput(new CountAggregator()))); - assertEquals("default contains ([{\"implicitTransforms\": false}]\"foo\") " + + assertEquals("default contains ([{implicitTransforms: false}]\"foo\") " + "| all(group(attribute(a)) each(output(count())))", VespaSerializer.serialize(query)); newGroupingRequest(query, new AllOperation().setGroupBy(new AttributeFunction("b")) .addChild(new EachOperation().addOutput(new CountAggregator()))); - assertEquals("default contains ([{\"implicitTransforms\": false}]\"foo\") " + + assertEquals("default contains ([{implicitTransforms: false}]\"foo\") " + "| all(group(attribute(a)) each(output(count()))) " + "| all(group(attribute(b)) each(output(count())))", VespaSerializer.serialize(query)); @@ -69,15 +69,15 @@ public class VespaSerializerTestCase { public void requireThatGroupingContinuationsAreSerialized() { Query query = new Query(); query.getModel().getQueryTree().setRoot(new WordItem("foo")); - assertEquals("default contains ([{\"implicitTransforms\": false}]\"foo\")", + assertEquals("default contains ([{implicitTransforms: false}]\"foo\")", VespaSerializer.serialize(query)); newGroupingRequest(query, new AllOperation().setGroupBy(new AttributeFunction("a")) .addChild(new EachOperation().addOutput(new CountAggregator())), Continuation.fromString("BCBCBCBEBG"), Continuation.fromString("BCBKCBACBKCCK")); - assertEquals("default contains ([{\"implicitTransforms\": false}]\"foo\") " + - "| [{ 'continuations':['BCBCBCBEBG', 'BCBKCBACBKCCK'] }]" + + assertEquals("default contains ([{implicitTransforms: false}]\"foo\") " + + "| [{ continuations:['BCBCBCBEBG', 'BCBKCBACBKCCK'] }]" + "all(group(attribute(a)) each(output(count())))", VespaSerializer.serialize(query)); @@ -85,10 +85,10 @@ public class VespaSerializerTestCase { .addChild(new EachOperation().addOutput(new CountAggregator())), Continuation.fromString("BCBBBBBDBF"), Continuation.fromString("BCBJBPCBJCCJ")); - assertEquals("default contains ([{\"implicitTransforms\": false}]\"foo\") " + - "| [{ 'continuations':['BCBCBCBEBG', 'BCBKCBACBKCCK'] }]" + + assertEquals("default contains ([{implicitTransforms: false}]\"foo\") " + + "| [{ continuations:['BCBCBCBEBG', 'BCBKCBACBKCCK'] }]" + "all(group(attribute(a)) each(output(count()))) " + - "| [{ 'continuations':['BCBBBBBDBF', 'BCBJBPCBJCCJ'] }]" + + "| [{ continuations:['BCBBBBBDBF', 'BCBJBPCBJCCJ'] }]" + "all(group(attribute(b)) each(output(count())))", VespaSerializer.serialize(query)); } @@ -129,17 +129,17 @@ public class VespaSerializerTestCase { @Test public void testNear() { parseAndConfirm("title contains near(\"a\", \"b\")"); - parseAndConfirm("title contains ([{\"distance\": 50}]near(\"a\", \"b\"))"); + parseAndConfirm("title contains ([{distance: 50}]near(\"a\", \"b\"))"); } @Test public void testNearestNeighbor() { - parseAndConfirm("[{\"label\": \"foo\", \"targetNumHits\": 1000}]nearestNeighbor(semantic_embedding, my_property)"); - parseAndConfirm("[{\"targetNumHits\": 42}]nearestNeighbor(semantic_embedding, my_property)"); - parseAndConfirm("[{\"targetNumHits\": 1, \"hnsw.exploreAdditionalHits\": 76}]nearestNeighbor(semantic_embedding, my_property)"); - parseAndConfirm("[{\"targetNumHits\": 2, \"approximate\": false}]nearestNeighbor(semantic_embedding, my_property)"); - parseAndConfirm("[{\"targetNumHits\": 3, \"hnsw.exploreAdditionalHits\": 67, \"approximate\": false}]nearestNeighbor(semantic_embedding, my_property)"); - parseAndConfirm("[{\"targetNumHits\": 4, \"distanceThreshold\": 100100.25}]nearestNeighbor(semantic_embedding, my_property)"); + parseAndConfirm("[{label: \"foo\", targetNumHits: 1000}]nearestNeighbor(semantic_embedding, my_property)"); + parseAndConfirm("[{targetNumHits: 42}]nearestNeighbor(semantic_embedding, my_property)"); + parseAndConfirm("[{targetNumHits: 1, hnsw.exploreAdditionalHits: 76}]nearestNeighbor(semantic_embedding, my_property)"); + parseAndConfirm("[{targetNumHits: 2, approximate: false}]nearestNeighbor(semantic_embedding, my_property)"); + parseAndConfirm("[{targetNumHits: 3, hnsw.exploreAdditionalHits: 67, approximate: false}]nearestNeighbor(semantic_embedding, my_property)"); + parseAndConfirm("[{targetNumHits: 4, distanceThreshold: 100100.25}]nearestNeighbor(semantic_embedding, my_property)"); } @Test @@ -163,11 +163,11 @@ public class VespaSerializerTestCase { @Test public void testAnnotatedNumbers() { - parseAndConfirm("title = ([{\"filter\": true}]500)"); - parseAndConfirm("title > ([{\"filter\": true}]500)"); - parseAndConfirm("title < ([{\"filter\": true}](-500))"); - parseAndConfirm("title <= ([{\"filter\": true}](-500))", "([{\"filter\": true}](-500)) >= title"); - parseAndConfirm("title <= ([{\"filter\": true}](-500))"); + parseAndConfirm("title = ([{filter: true}]500)"); + parseAndConfirm("title > ([{filter: true}]500)"); + parseAndConfirm("title < ([{filter: true}](-500))"); + parseAndConfirm("title <= ([{filter: true}](-500))", "([{filter: true}](-500)) >= title"); + parseAndConfirm("title <= ([{filter: true}](-500))"); } @Test @@ -177,7 +177,7 @@ public class VespaSerializerTestCase { @Test public void testAnnotatedRange() { - parseAndConfirm("[{\"filter\": true}]range(title, 1, 500)"); + parseAndConfirm("[{filter: true}]range(title, 1, 500)"); } @Test @@ -215,22 +215,22 @@ public class VespaSerializerTestCase { @Test public void testAnnotatedPhrase() { - parseAndConfirm("description contains ([{\"id\": 1}]phrase(\"a\", \"b\"))"); + parseAndConfirm("description contains ([{id: 1}]phrase(\"a\", \"b\"))"); } @Test public void testAnnotatedNear() { - parseAndConfirm("description contains ([{\"distance\": 37}]near(\"a\", \"b\"))"); + parseAndConfirm("description contains ([{distance: 37}]near(\"a\", \"b\"))"); } @Test public void testAnnotatedOnear() { - parseAndConfirm("description contains ([{\"distance\": 37}]onear(\"a\", \"b\"))"); + parseAndConfirm("description contains ([{distance: 37}]onear(\"a\", \"b\"))"); } @Test public void testAnnotatedEquiv() { - parseAndConfirm("description contains ([{\"id\": 1}]equiv(\"a\", \"b\"))"); + parseAndConfirm("description contains ([{id: 1}]equiv(\"a\", \"b\"))"); } @Test @@ -242,7 +242,7 @@ public class VespaSerializerTestCase { phraseSegment.setLabel("labeled"); phraseSegment.lock(); String q = VespaSerializer.serialize(phraseSegment); - assertEquals("someIndexName contains ([{\"origin\": {\"original\": \"abc\", \"offset\": 0, \"length\": 3}, \"label\": \"labeled\"}]phrase(\"a\", \"b\"))", q); + assertEquals("someIndexName contains ([{origin: {original: \"abc\", 'offset': 0, length: 3}, label: \"labeled\"}]phrase(\"a\", \"b\"))", q); } @Test @@ -251,7 +251,7 @@ public class VespaSerializerTestCase { sameElement.addItem(new WordItem("a", "f1")); sameElement.addItem(new WordItem("b", "f2")); assertEquals("ss:{f1:a f2:b}", sameElement.toString()); - assertEquals("ss contains sameElement(f1 contains ([{\"implicitTransforms\": false}]\"a\"), f2 contains ([{\"implicitTransforms\": false}]\"b\"))", VespaSerializer.serialize(sameElement)); + assertEquals("ss contains sameElement(f1 contains ([{implicitTransforms: false}]\"a\"), f2 contains ([{implicitTransforms: false}]\"b\"))", VespaSerializer.serialize(sameElement)); } @Test @@ -261,22 +261,22 @@ public class VespaSerializerTestCase { andSegment.addItem(new WordItem("b", "indexNamePlaceholder")); andSegment.setLabel("labeled"); String q = VespaSerializer.serialize(andSegment); - assertEquals("indexNamePlaceholder contains ([{\"origin\": {\"original\": \"abc\", \"offset\": 0, \"length\": 3}, \"andSegmenting\": true}]phrase(\"a\", \"b\"))", q); + assertEquals("indexNamePlaceholder contains ([{origin: {original: \"abc\", 'offset': 0, length: 3}, andSegmenting: true}]phrase(\"a\", \"b\"))", q); andSegment.setIndexName("someIndexName"); andSegment.lock(); q = VespaSerializer.serialize(andSegment); - assertEquals("someIndexName contains ([{\"origin\": {\"original\": \"abc\", \"offset\": 0, \"length\": 3}, \"andSegmenting\": true}]phrase(\"a\", \"b\"))", q); + assertEquals("someIndexName contains ([{origin: {original: \"abc\", 'offset': 0, length: 3}, andSegmenting: true}]phrase(\"a\", \"b\"))", q); } @Test public void testPhraseWithAnnotations() { - parseAndConfirm("description contains phrase(([{\"id\": 15}]\"a\"), \"b\")"); + parseAndConfirm("description contains phrase(([{id: 15}]\"a\"), \"b\")"); } @Test public void testPhraseSegmentInPhrase() { - parseAndConfirm("description contains phrase(\"a\", \"b\", ([{\"origin\": {\"original\": \"c d\", \"offset\": 0, \"length\": 3}}]phrase(\"c\", \"d\")))"); + parseAndConfirm("description contains phrase(\"a\", \"b\", ([{origin: {original: \"c d\", 'offset': 0, length: 3}}]phrase(\"c\", \"d\")))"); } @Test @@ -296,10 +296,10 @@ public class VespaSerializerTestCase { @Test public void testAnnotatedWeakAnd() { - parseAndConfirm("([{\"" + YqlParser.TARGET_NUM_HITS + "\": 10}]weakAnd(a contains \"A\", b contains \"B\"))"); - parseAndConfirm("([{\"" + YqlParser.SCORE_THRESHOLD + "\": 10}]weakAnd(a contains \"A\", b contains \"B\"))"); - parseAndConfirm("([{\"" + YqlParser.TARGET_NUM_HITS + "\": 10, \"" + YqlParser.SCORE_THRESHOLD - + "\": 20}]weakAnd(a contains \"A\", b contains \"B\"))"); + parseAndConfirm("([{" + YqlParser.TARGET_NUM_HITS + ": 10}]weakAnd(a contains \"A\", b contains \"B\"))"); + parseAndConfirm("([{" + YqlParser.SCORE_THRESHOLD + ": 10}]weakAnd(a contains \"A\", b contains \"B\"))"); + parseAndConfirm("([{" + YqlParser.TARGET_NUM_HITS + ": 10, " + YqlParser.SCORE_THRESHOLD + + ": 20}]weakAnd(a contains \"A\", b contains \"B\"))"); } @Test @@ -309,36 +309,36 @@ public class VespaSerializerTestCase { @Test public void testAnnotatedWord() { - parseAndConfirm("description contains ([{\"andSegmenting\": true}]\"a\")"); - parseAndConfirm("description contains ([{\"weight\": 37}]\"a\")"); - parseAndConfirm("description contains ([{\"id\": 37}]\"a\")"); - parseAndConfirm("description contains ([{\"filter\": true}]\"a\")"); - parseAndConfirm("description contains ([{\"ranked\": false}]\"a\")"); - parseAndConfirm("description contains ([{\"significance\": 37.0}]\"a\")"); - parseAndConfirm("description contains ([{\"implicitTransforms\": false}]\"a\")"); - parseAndConfirm("(description contains ([{\"connectivity\": {\"id\": 2, \"weight\": 0.42}, \"id\": 1}]\"a\") AND description contains ([{\"id\": 2}]\"b\"))"); + parseAndConfirm("description contains ([{andSegmenting: true}]\"a\")"); + parseAndConfirm("description contains ([{weight: 37}]\"a\")"); + parseAndConfirm("description contains ([{id: 37}]\"a\")"); + parseAndConfirm("description contains ([{filter: true}]\"a\")"); + parseAndConfirm("description contains ([{ranked: false}]\"a\")"); + parseAndConfirm("description contains ([{significance: 37.0}]\"a\")"); + parseAndConfirm("description contains ([{implicitTransforms: false}]\"a\")"); + parseAndConfirm("(description contains ([{connectivity: {id: 2, weight: 0.42}, id: 1}]\"a\") AND description contains ([{id: 2}]\"b\"))"); } @Test public void testPrefix() { - parseAndConfirm("description contains ([{\"prefix\": true}]\"a\")"); + parseAndConfirm("description contains ([{prefix: true}]\"a\")"); } @Test public void testSuffix() { - parseAndConfirm("description contains ([{\"suffix\": true}]\"a\")"); + parseAndConfirm("description contains ([{suffix: true}]\"a\")"); } @Test public void testSubstring() { - parseAndConfirm("description contains ([{\"substring\": true}]\"a\")"); + parseAndConfirm("description contains ([{substring: true}]\"a\")"); } @Test public void testExoticItemTypes() { Item item = MarkerWordItem.createEndOfHost(); String q = VespaSerializer.serialize(item); - assertEquals("default contains ([{\"implicitTransforms\": false}]\"$\")", q); + assertEquals("default contains ([{implicitTransforms: false}]\"$\")", q); } @Test @@ -357,7 +357,7 @@ public class VespaSerializerTestCase { item.addItem(new WordItem("c")); item.addItem(new WordItem("d")); String q = VespaSerializer.serialize(item); - assertEquals("(default contains ([{\"implicitTransforms\": false}]\"a\")) AND !(default contains ([{\"implicitTransforms\": false}]\"b\") OR default contains ([{\"implicitTransforms\": false}]\"c\") OR default contains ([{\"implicitTransforms\": false}]\"d\"))", q); + assertEquals("(default contains ([{implicitTransforms: false}]\"a\")) AND !(default contains ([{implicitTransforms: false}]\"b\") OR default contains ([{implicitTransforms: false}]\"c\") OR default contains ([{implicitTransforms: false}]\"d\"))", q); } @Test @@ -406,23 +406,23 @@ public class VespaSerializerTestCase { @Test public void testAnnotatedLong() { - parseAndConfirm("title >= ([{\"id\": 2014}](-549755813888L))"); + parseAndConfirm("title >= ([{id: 2014}](-549755813888L))"); } @Test public void testHitLimit() { - parseAndConfirm("title <= ([{\"hitLimit\": 89}](-500))"); - parseAndConfirm("title <= ([{\"hitLimit\": 89}](-500))"); - parseAndConfirm("[{\"hitLimit\": 89}]range(title, 1, 500)"); + parseAndConfirm("title <= ([{hitLimit: 89}](-500))"); + parseAndConfirm("title <= ([{hitLimit: 89}](-500))"); + parseAndConfirm("[{hitLimit: 89}]range(title, 1, 500)"); } @Test public void testOpenIntervals() { parseAndConfirm("range(title, 0.0, 500.0)"); - parseAndConfirm("([{\"bounds\": \"open\"}]range(title, 0.0, 500.0))"); - parseAndConfirm("([{\"bounds\": \"leftOpen\"}]range(title, 0.0, 500.0))"); - parseAndConfirm("([{\"bounds\": \"rightOpen\"}]range(title, 0.0, 500.0))"); - parseAndConfirm("([{\"id\": 500, \"bounds\": \"rightOpen\"}]range(title, 0.0, 500.0))"); + parseAndConfirm("([{bounds: \"open\"}]range(title, 0.0, 500.0))"); + parseAndConfirm("([{bounds: \"leftOpen\"}]range(title, 0.0, 500.0))"); + parseAndConfirm("([{bounds: \"rightOpen\"}]range(title, 0.0, 500.0))"); + parseAndConfirm("([{id: 500, bounds: \"rightOpen\"}]range(title, 0.0, 500.0))"); } @Test @@ -432,14 +432,14 @@ public class VespaSerializerTestCase { @Test public void testWordAlternatives() { - parseAndConfirm("foo contains" + " ([{\"origin\": {\"original\": \" trees \", \"offset\": 1, \"length\": 5}}]" + parseAndConfirm("foo contains" + " ([{origin: {original: \" trees \", 'offset': 1, length: 5}}]" + "alternatives({\"trees\": 1.0, \"tree\": 0.7}))"); } @Test public void testWordAlternativesInPhrase() { parseAndConfirm("foo contains phrase(\"forest\"," - + " ([{\"origin\": {\"original\": \" trees \", \"offset\": 1, \"length\": 5}}]" + + " ([{origin: {original: \" trees \", 'offset': 1, length: 5}}]" + "alternatives({\"trees\": 1.0, \"tree\": 0.7}))" + ")"); } diff --git a/container-search/src/test/java/com/yahoo/search/yql/YqlFieldAndSourceTestCase.java b/container-search/src/test/java/com/yahoo/search/yql/YqlFieldAndSourceTestCase.java index 0cf5ea75526..27959948536 100644 --- a/container-search/src/test/java/com/yahoo/search/yql/YqlFieldAndSourceTestCase.java +++ b/container-search/src/test/java/com/yahoo/search/yql/YqlFieldAndSourceTestCase.java @@ -27,8 +27,7 @@ import com.yahoo.search.searchchain.testutil.DocumentSourceSearcher; import static com.yahoo.search.searchchain.testutil.DocumentSourceSearcher.DEFAULT_SUMMARY_CLASS;; /** - * Test translation of fields and sources in YQL+ to the associated concepts in - * Vespa. + * Test translation of fields and sources in YQL+ to the associated concepts in Vespa. */ public class YqlFieldAndSourceTestCase { diff --git a/container-search/src/test/java/com/yahoo/search/yql/YqlParserTestCase.java b/container-search/src/test/java/com/yahoo/search/yql/YqlParserTestCase.java index 2a21214c702..ec326f4bdad 100644 --- a/container-search/src/test/java/com/yahoo/search/yql/YqlParserTestCase.java +++ b/container-search/src/test/java/com/yahoo/search/yql/YqlParserTestCase.java @@ -71,7 +71,7 @@ public class YqlParserTestCase { @Test(timeout = 120_000) public void failsGracefullyOnMissingQuoteEscapingAndSubsequentUnicodeCharacter() { - assertParseFail("select * from bar where rank(ids contains 'http://en.wikipedia.org/wiki/Hors_d'œuvre') limit 10;", + assertParseFail("select * from bar where rank(ids contains 'http://en.wikipedia.org/wiki/Hors_d'œuvre') limit 10", new IllegalInputException("com.yahoo.search.yql.ProgramCompileException: query:L1:79 token recognition error at: 'œ'")); } @@ -84,26 +84,26 @@ public class YqlParserTestCase { @Test public void testLanguageDetection() { // SimpleDetector used here can detect japanese and will set that as language at the root of the user input - QueryTree tree = parse("select * from sources * where userInput(\"\u30ab\u30bf\u30ab\u30ca\");"); + QueryTree tree = parse("select * from sources * where userInput(\"\u30ab\u30bf\u30ab\u30ca\")"); assertEquals(Language.JAPANESE, tree.getRoot().getLanguage()); } @Test public void testGroupingStep() { - assertParse("select foo from bar where baz contains 'cox';", + assertParse("select foo from bar where baz contains 'cox'", "baz:cox"); assertEquals("[]", toString(parser.getGroupingSteps())); assertParse("select foo from bar where baz contains 'cox' " + - "| all(group(a) each(output(count())));", + "| all(group(a) each(output(count())))", "baz:cox"); assertEquals("[[]all(group(a) each(output(count())))]", toString(parser.getGroupingSteps())); assertParse("select foo from bar where baz contains 'cox' " + "| all(group(a) each(output(count()))) " + - "| all(group(b) each(output(count())));", + "| all(group(b) each(output(count())))", "baz:cox"); assertEquals("[[]all(group(a) each(output(count())))," + " []all(group(b) each(output(count())))]", @@ -113,14 +113,14 @@ public class YqlParserTestCase { @Test public void testGroupingContinuation() { assertParse("select foo from bar where baz contains 'cox' " + - "| [{ 'continuations': ['BCBCBCBEBG', 'BCBKCBACBKCCK'] }]all(group(a) each(output(count())));", + "| [{ 'continuations': ['BCBCBCBEBG', 'BCBKCBACBKCCK'] }]all(group(a) each(output(count())))", "baz:cox"); assertEquals("[[BCBCBCBEBG, BCBKCBACBKCCK]all(group(a) each(output(count())))]", toString(parser.getGroupingSteps())); assertParse("select foo from bar where baz contains 'cox' " + "| [{ 'continuations': ['BCBCBCBEBG', 'BCBKCBACBKCCK'] }]all(group(a) each(output(count()))) " + - "| [{ 'continuations': ['BCBBBBBDBF', 'BCBJBPCBJCCJ'] }]all(group(b) each(output(count())));", + "| [{ 'continuations': ['BCBBBBBDBF', 'BCBJBPCBJCCJ'] }]all(group(b) each(output(count())))", "baz:cox"); assertEquals("[[BCBCBCBEBG, BCBKCBACBKCCK]all(group(a) each(output(count())))," + " [BCBBBBBDBF, BCBJBPCBJCCJ]all(group(b) each(output(count())))]", @@ -129,57 +129,57 @@ public class YqlParserTestCase { @Test public void testHitLimit() { - assertParse("select artist_name, track_name, track_uri from sources * where (myField contains ([{\"prefix\":true}]\"m\") and ([{\"hitLimit\": 5000, \"descending\": true}]range(static_score,0,Infinity))) limit 30 offset 0;", + assertParse("select artist_name, track_name, track_uri from sources * where (myField contains ([{prefix:true}]\"m\") and ([{hitLimit: 5000, descending: true}]range(static_score,0,Infinity))) limit 30 offset 0", "AND myField:m* static_score:[0;;-5000]"); } @Test public void test() { - assertParse("select foo from bar where title contains \"madonna\";", + assertParse("select foo from bar where title contains \"madonna\"", "title:madonna"); } @Test public void testKeywordAsFieldName() { - assertParse("select * from sources * where cast contains sameElement(id contains '16');", + assertParse("select * from sources * where cast contains sameElement(id contains '16')", "cast:{id:16}"); } @Test public void testComplexExpression() { - String queryTreeYql = "rank((((filter contains ([{\"origin\": {\"original\": \"filter:VideoAdsCappingTestCPM\", \"offset\": 7, \"length\": 22}, \"normalizeCase\": false, \"id\": 1}]\"videoadscappingtestcpm\") AND hasRankRestriction contains ([{\"normalizeCase\": false, \"implicitTransforms\": false, \"id\": 2}]\"0\") AND ((objective contains ([{\"normalizeCase\": false, \"implicitTransforms\": false, \"id\": 3}]\"install_app\") AND availableExtendedFields contains ([{\"normalizeCase\": false, \"implicitTransforms\": false, \"id\": 4}]\"cpiparams\")) OR (availableExtendedFields contains ([{\"normalizeCase\": false, \"implicitTransforms\": false, \"id\": 5}]\"appinstallinfo\") AND availableExtendedFields contains ([{\"normalizeCase\": false, \"implicitTransforms\": false, \"id\": 6}]\"appmetroplexinfo\")) OR (dummyField contains ([{\"normalizeCase\": false, \"implicitTransforms\": false, \"id\": 7}]\"default\")) AND !(objective contains ([{\"normalizeCase\": false, \"implicitTransforms\": false, \"id\": 8}]\"install_app\"))) AND advt_age = ([{\"id\": 9}]2147483647) AND advt_gender contains ([{\"normalizeCase\": false, \"implicitTransforms\": false, \"id\": 10}]\"all\") AND advt_all_segments = ([{\"id\": 11}]2147483647) AND advt_keywords contains ([{\"normalizeCase\": false, \"implicitTransforms\": false, \"id\": 12}]\"all\") AND advMobilePlatform = ([{\"id\": 13}]2147483647) AND advMobileDeviceType = ([{\"id\": 14}]2147483647) AND advMobileCon = ([{\"id\": 15}]2147483647) AND advMobileOSVersions = ([{\"id\": 16}]2147483647) AND advCarrier = ([{\"id\": 17}]2147483647) AND ([{\"id\": 18}]weightedSet(advt_supply, {\"all\": 1, \"pub223\": 1, \"sec223\": 1, \"site223\": 1})) AND (advt_day_parting contains ([{\"normalizeCase\": false, \"implicitTransforms\": false, \"id\": 19, \"weight\": 1}]\"adv_tuesday\") OR advt_day_parting contains ([{\"normalizeCase\": false, \"implicitTransforms\": false, \"id\": 20, \"weight\": 1}]\"adv_tuesday_17\") OR advt_day_parting contains ([{\"normalizeCase\": false, \"implicitTransforms\": false, \"id\": 21, \"weight\": 1}]\"adv_tuesday_17_forty_five\") OR advt_day_parting contains ([{\"normalizeCase\": false, \"implicitTransforms\": false, \"id\": 22}]\"all\")) AND isAppReengagementAd = ([{\"id\": 23}]0) AND dummyField contains ([{\"normalizeCase\": false, \"implicitTransforms\": false, \"id\": 24}]\"default\") AND serveWithPromotionOnly = ([{\"id\": 26}]0) AND budgetAdvertiserThrottleRateFilter = ([{\"id\": 27}]0) AND budgetResellerThrottleRateFilter = ([{\"id\": 28}]0) AND (isMystiqueRequired = ([{\"id\": 29}]0) OR (isMystiqueRequired = ([{\"id\": 30}]1) AND useBcFactorFilter = ([{\"id\": 31}]1))) AND (((budgetCampaignThrottleRateBits = ([{\"id\": 32}]55) AND dummyField contains ([{\"normalizeCase\": false, \"implicitTransforms\": false, \"id\": 33}]\"default\"))) AND !(useBcFactorFilter = ([{\"id\": 34}]1)) OR ((useBcFactorFilter = ([{\"id\": 35}]1) AND dummyField contains ([{\"normalizeCase\": false, \"implicitTransforms\": false, \"id\": 36}]\"default\") AND (bcFactorTiers = ([{\"id\": 38}]127) OR bcFactorTiers = ([{\"id\": 39}]0)) AND ((firstPriceEnforced = ([{\"id\": 40}]0) AND (secondPriceEnforced = ([{\"id\": 41}]1) OR isPrivateDeal = ([{\"id\": 42}]0) OR (dummyField contains ([{\"normalizeCase\": false, \"implicitTransforms\": false, \"id\": 43}]\"default\")) AND !(bcActiveTier = ([{\"id\": 44}]0)))) OR mystiqueCampaignThrottleRateBits = ([{\"id\": 45}]18)))) AND !(isOutOfDailyBudget = ([{\"id\": 37}]1))) AND testCreative = ([{\"id\": 46}]0) AND advt_geo = ([{\"id\": 47}]2147483647) AND ((adType contains ([{\"normalizeCase\": false, \"implicitTransforms\": false, \"id\": 48}]\"strm_video\") AND isPortraitVideo = ([{\"id\": 49}]0)) OR adType contains ([{\"normalizeCase\": false, \"implicitTransforms\": false, \"id\": 50}]\"stream_ad\")) AND ((isCPM = ([{\"id\": 51}]0) AND isOCPC = ([{\"id\": 52}]0) AND isECPC = ([{\"id\": 53}]0) AND ((priceType contains ([{\"normalizeCase\": false, \"implicitTransforms\": false, \"id\": 54}]\"cpcv\") AND bid >= ([{\"id\": 55}]0.005)) OR (priceType contains ([{\"normalizeCase\": false, \"implicitTransforms\": false, \"id\": 56}]\"cpv\") AND bid >= ([{\"id\": 57}]0.01)) OR (priceType contains ([{\"normalizeCase\": false, \"implicitTransforms\": false, \"id\": 58}]\"cpc\") AND bid >= ([{\"id\": 59}]0.05)) OR (objective contains ([{\"normalizeCase\": false, \"implicitTransforms\": false, \"id\": 60}]\"promote_content\") AND bid >= ([{\"id\": 61}]0.01)) OR hasFloorPriceUsd = ([{\"id\": 62}]1))) OR isECPC = ([{\"id\": 63}]1) OR (isCPM = ([{\"id\": 64}]1) AND isOCPM = ([{\"id\": 65}]0) AND (([{\"id\": 66}]range(bid, 0.25, Infinity)) OR hasFloorPriceUsd = ([{\"id\": 67}]1)))) AND start_date <= ([{\"id\": 68}]1572976776299L) AND end_date >= ([{\"id\": 69}]1572976776299L))) AND !(isHoldoutAd = ([{\"id\": 25}]1))) AND !((disclaimerExtensionsTypes contains ([{\"normalizeCase\": false, \"implicitTransforms\": false, \"id\": 70}]\"pharma\") OR ([{\"id\": 71}]weightedSet(exclusion_advt_supply, {\"extsite223\": 1, \"pub223\": 1, \"sec223\": 1, \"site223\": 1})) OR isPersonalized = ([{\"id\": 72}]1) OR blocked_section_ids contains ([{\"normalizeCase\": false, \"implicitTransforms\": false, \"id\": 73}]\"223\") OR blocked_publisher_ids contains ([{\"normalizeCase\": false, \"implicitTransforms\": false, \"id\": 74}]\"223\") OR blocked_site_ids contains ([{\"normalizeCase\": false, \"implicitTransforms\": false, \"id\": 75}]\"223\"))), [{\"id\": 76, \"label\": \"ad_ocpc_max_cpc\"}]dotProduct(ocpc_max_cpc, {\"0\": 1}), [{\"id\": 77, \"label\": \"ad_ocpc_min_cpc\"}]dotProduct(ocpc_min_cpc, {\"0\": 1}), [{\"id\": 78, \"label\": \"ad_ocpc_max_alpha\"}]dotProduct(ocpc_max_alpha, {\"0\": 1}), [{\"id\": 79, \"label\": \"ad_ocpc_min_alpha\"}]dotProduct(ocpc_min_alpha, {\"0\": 1}), [{\"id\": 80, \"label\": \"ad_ocpc_alpha_0\"}]dotProduct(ocpc_alpha_0, {\"0\": 1}), [{\"id\": 81, \"label\": \"ad_ocpc_alpha_1\"}]dotProduct(ocpc_alpha_1, {\"0\": 1}), (bidAdjustmentDayParting contains ([{\"normalizeCase\": false, \"implicitTransforms\": false, \"id\": 82, \"weight\": 1}]\"adv_tuesday\") OR bidAdjustmentDayParting contains ([{\"normalizeCase\": false, \"implicitTransforms\": false, \"id\": 83, \"weight\": 1}]\"adv_tuesday_17\") OR bidAdjustmentDayParting contains ([{\"normalizeCase\": false, \"implicitTransforms\": false, \"id\": 84, \"weight\": 1}]\"adv_tuesday_17_forty_five\") OR bidAdjustmentDayPartingForCostCap contains ([{\"normalizeCase\": false, \"implicitTransforms\": false, \"id\": 85, \"weight\": 1}]\"adv_tuesday\") OR bidAdjustmentDayPartingForCostCap contains ([{\"normalizeCase\": false, \"implicitTransforms\": false, \"id\": 86, \"weight\": 1}]\"adv_tuesday_17\") OR bidAdjustmentDayPartingForCostCap contains ([{\"normalizeCase\": false, \"implicitTransforms\": false, \"id\": 87, \"weight\": 1}]\"adv_tuesday_17_forty_five\")), bidAdjustmentForCpi = ([{\"id\": 88, \"weight\": 1}]223), [{\"id\": 89, \"label\": \"boostingForBackfill\"}]dotProduct(boostingForBackfill, {\"priority\": 1000})) limit 0 timeout 3980 | all(group(adTypeForGrouping) each(group(advertiser_id) max(11) output(count() as(groupingCounter)) each(max(1) each(output(summary())))))"; + String queryTreeYql = "rank((((filter contains ([{origin: {original: \"filter:VideoAdsCappingTestCPM\", \"offset\": 7, length: 22}, normalizeCase: false, id: 1}]\"videoadscappingtestcpm\") AND hasRankRestriction contains ([{\"normalizeCase\": false, \"implicitTransforms\": false, \"id\": 2}]\"0\") AND ((objective contains ([{\"normalizeCase\": false, \"implicitTransforms\": false, \"id\": 3}]\"install_app\") AND availableExtendedFields contains ([{\"normalizeCase\": false, \"implicitTransforms\": false, \"id\": 4}]\"cpiparams\")) OR (availableExtendedFields contains ([{\"normalizeCase\": false, \"implicitTransforms\": false, \"id\": 5}]\"appinstallinfo\") AND availableExtendedFields contains ([{\"normalizeCase\": false, \"implicitTransforms\": false, \"id\": 6}]\"appmetroplexinfo\")) OR (dummyField contains ([{\"normalizeCase\": false, \"implicitTransforms\": false, \"id\": 7}]\"default\")) AND !(objective contains ([{\"normalizeCase\": false, \"implicitTransforms\": false, \"id\": 8}]\"install_app\"))) AND advt_age = ([{\"id\": 9}]2147483647) AND advt_gender contains ([{\"normalizeCase\": false, \"implicitTransforms\": false, \"id\": 10}]\"all\") AND advt_all_segments = ([{\"id\": 11}]2147483647) AND advt_keywords contains ([{\"normalizeCase\": false, \"implicitTransforms\": false, \"id\": 12}]\"all\") AND advMobilePlatform = ([{\"id\": 13}]2147483647) AND advMobileDeviceType = ([{\"id\": 14}]2147483647) AND advMobileCon = ([{\"id\": 15}]2147483647) AND advMobileOSVersions = ([{\"id\": 16}]2147483647) AND advCarrier = ([{\"id\": 17}]2147483647) AND ([{\"id\": 18}]weightedSet(advt_supply, {\"all\": 1, \"pub223\": 1, \"sec223\": 1, \"site223\": 1})) AND (advt_day_parting contains ([{\"normalizeCase\": false, \"implicitTransforms\": false, \"id\": 19, \"weight\": 1}]\"adv_tuesday\") OR advt_day_parting contains ([{\"normalizeCase\": false, \"implicitTransforms\": false, \"id\": 20, \"weight\": 1}]\"adv_tuesday_17\") OR advt_day_parting contains ([{\"normalizeCase\": false, \"implicitTransforms\": false, \"id\": 21, \"weight\": 1}]\"adv_tuesday_17_forty_five\") OR advt_day_parting contains ([{\"normalizeCase\": false, \"implicitTransforms\": false, \"id\": 22}]\"all\")) AND isAppReengagementAd = ([{\"id\": 23}]0) AND dummyField contains ([{\"normalizeCase\": false, \"implicitTransforms\": false, \"id\": 24}]\"default\") AND serveWithPromotionOnly = ([{\"id\": 26}]0) AND budgetAdvertiserThrottleRateFilter = ([{\"id\": 27}]0) AND budgetResellerThrottleRateFilter = ([{\"id\": 28}]0) AND (isMystiqueRequired = ([{\"id\": 29}]0) OR (isMystiqueRequired = ([{\"id\": 30}]1) AND useBcFactorFilter = ([{\"id\": 31}]1))) AND (((budgetCampaignThrottleRateBits = ([{\"id\": 32}]55) AND dummyField contains ([{\"normalizeCase\": false, \"implicitTransforms\": false, \"id\": 33}]\"default\"))) AND !(useBcFactorFilter = ([{\"id\": 34}]1)) OR ((useBcFactorFilter = ([{\"id\": 35}]1) AND dummyField contains ([{\"normalizeCase\": false, \"implicitTransforms\": false, \"id\": 36}]\"default\") AND (bcFactorTiers = ([{\"id\": 38}]127) OR bcFactorTiers = ([{\"id\": 39}]0)) AND ((firstPriceEnforced = ([{\"id\": 40}]0) AND (secondPriceEnforced = ([{\"id\": 41}]1) OR isPrivateDeal = ([{\"id\": 42}]0) OR (dummyField contains ([{\"normalizeCase\": false, \"implicitTransforms\": false, \"id\": 43}]\"default\")) AND !(bcActiveTier = ([{\"id\": 44}]0)))) OR mystiqueCampaignThrottleRateBits = ([{\"id\": 45}]18)))) AND !(isOutOfDailyBudget = ([{\"id\": 37}]1))) AND testCreative = ([{\"id\": 46}]0) AND advt_geo = ([{\"id\": 47}]2147483647) AND ((adType contains ([{\"normalizeCase\": false, \"implicitTransforms\": false, \"id\": 48}]\"strm_video\") AND isPortraitVideo = ([{\"id\": 49}]0)) OR adType contains ([{\"normalizeCase\": false, \"implicitTransforms\": false, \"id\": 50}]\"stream_ad\")) AND ((isCPM = ([{\"id\": 51}]0) AND isOCPC = ([{\"id\": 52}]0) AND isECPC = ([{\"id\": 53}]0) AND ((priceType contains ([{\"normalizeCase\": false, \"implicitTransforms\": false, \"id\": 54}]\"cpcv\") AND bid >= ([{\"id\": 55}]0.005)) OR (priceType contains ([{\"normalizeCase\": false, \"implicitTransforms\": false, \"id\": 56}]\"cpv\") AND bid >= ([{\"id\": 57}]0.01)) OR (priceType contains ([{\"normalizeCase\": false, \"implicitTransforms\": false, \"id\": 58}]\"cpc\") AND bid >= ([{\"id\": 59}]0.05)) OR (objective contains ([{\"normalizeCase\": false, \"implicitTransforms\": false, \"id\": 60}]\"promote_content\") AND bid >= ([{\"id\": 61}]0.01)) OR hasFloorPriceUsd = ([{\"id\": 62}]1))) OR isECPC = ([{\"id\": 63}]1) OR (isCPM = ([{\"id\": 64}]1) AND isOCPM = ([{\"id\": 65}]0) AND (([{\"id\": 66}]range(bid, 0.25, Infinity)) OR hasFloorPriceUsd = ([{\"id\": 67}]1)))) AND start_date <= ([{\"id\": 68}]1572976776299L) AND end_date >= ([{\"id\": 69}]1572976776299L))) AND !(isHoldoutAd = ([{\"id\": 25}]1))) AND !((disclaimerExtensionsTypes contains ([{\"normalizeCase\": false, \"implicitTransforms\": false, \"id\": 70}]\"pharma\") OR ([{\"id\": 71}]weightedSet(exclusion_advt_supply, {\"extsite223\": 1, \"pub223\": 1, \"sec223\": 1, \"site223\": 1})) OR isPersonalized = ([{\"id\": 72}]1) OR blocked_section_ids contains ([{\"normalizeCase\": false, \"implicitTransforms\": false, \"id\": 73}]\"223\") OR blocked_publisher_ids contains ([{\"normalizeCase\": false, \"implicitTransforms\": false, \"id\": 74}]\"223\") OR blocked_site_ids contains ([{\"normalizeCase\": false, \"implicitTransforms\": false, \"id\": 75}]\"223\"))), [{\"id\": 76, \"label\": \"ad_ocpc_max_cpc\"}]dotProduct(ocpc_max_cpc, {\"0\": 1}), [{\"id\": 77, \"label\": \"ad_ocpc_min_cpc\"}]dotProduct(ocpc_min_cpc, {\"0\": 1}), [{\"id\": 78, \"label\": \"ad_ocpc_max_alpha\"}]dotProduct(ocpc_max_alpha, {\"0\": 1}), [{\"id\": 79, \"label\": \"ad_ocpc_min_alpha\"}]dotProduct(ocpc_min_alpha, {\"0\": 1}), [{\"id\": 80, \"label\": \"ad_ocpc_alpha_0\"}]dotProduct(ocpc_alpha_0, {\"0\": 1}), [{\"id\": 81, \"label\": \"ad_ocpc_alpha_1\"}]dotProduct(ocpc_alpha_1, {\"0\": 1}), (bidAdjustmentDayParting contains ([{\"normalizeCase\": false, \"implicitTransforms\": false, \"id\": 82, \"weight\": 1}]\"adv_tuesday\") OR bidAdjustmentDayParting contains ([{\"normalizeCase\": false, \"implicitTransforms\": false, \"id\": 83, \"weight\": 1}]\"adv_tuesday_17\") OR bidAdjustmentDayParting contains ([{\"normalizeCase\": false, \"implicitTransforms\": false, \"id\": 84, \"weight\": 1}]\"adv_tuesday_17_forty_five\") OR bidAdjustmentDayPartingForCostCap contains ([{\"normalizeCase\": false, \"implicitTransforms\": false, \"id\": 85, \"weight\": 1}]\"adv_tuesday\") OR bidAdjustmentDayPartingForCostCap contains ([{\"normalizeCase\": false, \"implicitTransforms\": false, \"id\": 86, \"weight\": 1}]\"adv_tuesday_17\") OR bidAdjustmentDayPartingForCostCap contains ([{\"normalizeCase\": false, \"implicitTransforms\": false, \"id\": 87, \"weight\": 1}]\"adv_tuesday_17_forty_five\")), bidAdjustmentForCpi = ([{\"id\": 88, \"weight\": 1}]223), [{\"id\": 89, \"label\": \"boostingForBackfill\"}]dotProduct(boostingForBackfill, {\"priority\": 1000})) limit 0 timeout 3980 | all(group(adTypeForGrouping) each(group(advertiser_id) max(11) output(count() as(groupingCounter)) each(max(1) each(output(summary())))))"; QueryTree parsed = assertParse("select * from sources * where " + queryTreeYql + ";", "RANK (+(+(AND filter:VideoAdsCappingTestCPM hasRankRestriction:0 (OR (AND objective:install_app availableExtendedFields:cpiparams) (AND availableExtendedFields:appinstallinfo availableExtendedFields:appmetroplexinfo) (+dummyField:default -objective:install_app)) advt_age:2147483647 advt_gender:all advt_all_segments:2147483647 advt_keywords:all advMobilePlatform:2147483647 advMobileDeviceType:2147483647 advMobileCon:2147483647 advMobileOSVersions:2147483647 advCarrier:2147483647 WEIGHTEDSET advt_supply{[1]:\"site223\",[1]:\"pub223\",[1]:\"all\",[1]:\"sec223\"} (OR advt_day_parting:adv_tuesday!1 advt_day_parting:adv_tuesday_17!1 advt_day_parting:adv_tuesday_17_forty_five!1 advt_day_parting:all) isAppReengagementAd:0 dummyField:default serveWithPromotionOnly:0 budgetAdvertiserThrottleRateFilter:0 budgetResellerThrottleRateFilter:0 (OR isMystiqueRequired:0 (AND isMystiqueRequired:1 useBcFactorFilter:1)) (OR (+(AND budgetCampaignThrottleRateBits:55 dummyField:default) -useBcFactorFilter:1) (+(AND useBcFactorFilter:1 dummyField:default (OR bcFactorTiers:127 bcFactorTiers:0) (OR (AND firstPriceEnforced:0 (OR secondPriceEnforced:1 isPrivateDeal:0 (+dummyField:default -bcActiveTier:0))) mystiqueCampaignThrottleRateBits:18)) -isOutOfDailyBudget:1)) testCreative:0 advt_geo:2147483647 (OR (AND adType:strm_video isPortraitVideo:0) adType:stream_ad) (OR (AND isCPM:0 isOCPC:0 isECPC:0 (OR (AND priceType:cpcv bid:[0.005;]) (AND priceType:cpv bid:[0.01;]) (AND priceType:cpc bid:[0.05;]) (AND objective:promote_content bid:[0.01;]) hasFloorPriceUsd:1)) isECPC:1 (AND isCPM:1 isOCPM:0 (OR bid:[0.25;] hasFloorPriceUsd:1))) start_date:[;1572976776299] end_date:[1572976776299;]) -isHoldoutAd:1) -(OR disclaimerExtensionsTypes:pharma WEIGHTEDSET exclusion_advt_supply{[1]:\"extsite223\",[1]:\"site223\",[1]:\"pub223\",[1]:\"sec223\"} isPersonalized:1 blocked_section_ids:223 blocked_publisher_ids:223 blocked_site_ids:223)) DOTPRODUCT ocpc_max_cpc{[1]:\"0\"} DOTPRODUCT ocpc_min_cpc{[1]:\"0\"} DOTPRODUCT ocpc_max_alpha{[1]:\"0\"} DOTPRODUCT ocpc_min_alpha{[1]:\"0\"} DOTPRODUCT ocpc_alpha_0{[1]:\"0\"} DOTPRODUCT ocpc_alpha_1{[1]:\"0\"} (OR bidAdjustmentDayParting:adv_tuesday!1 bidAdjustmentDayParting:adv_tuesday_17!1 bidAdjustmentDayParting:adv_tuesday_17_forty_five!1 bidAdjustmentDayPartingForCostCap:adv_tuesday!1 bidAdjustmentDayPartingForCostCap:adv_tuesday_17!1 bidAdjustmentDayPartingForCostCap:adv_tuesday_17_forty_five!1) bidAdjustmentForCpi:223!1 DOTPRODUCT boostingForBackfill{[1000]:\"priority\"}"); String serializedQueryTreeYql = VespaSerializer.serialize(parsed); // Note: All the details here are not verified - assertEquals("rank((((filter contains ([{\"normalizeCase\": false, \"id\": 1}]\"VideoAdsCappingTestCPM\") AND hasRankRestriction contains ([{\"normalizeCase\": false, \"implicitTransforms\": false, \"id\": 2}]\"0\") AND ((objective contains ([{\"normalizeCase\": false, \"implicitTransforms\": false, \"id\": 3}]\"install_app\") AND availableExtendedFields contains ([{\"normalizeCase\": false, \"implicitTransforms\": false, \"id\": 4}]\"cpiparams\")) OR (availableExtendedFields contains ([{\"normalizeCase\": false, \"implicitTransforms\": false, \"id\": 5}]\"appinstallinfo\") AND availableExtendedFields contains ([{\"normalizeCase\": false, \"implicitTransforms\": false, \"id\": 6}]\"appmetroplexinfo\")) OR (dummyField contains ([{\"normalizeCase\": false, \"implicitTransforms\": false, \"id\": 7}]\"default\")) AND !(objective contains ([{\"normalizeCase\": false, \"implicitTransforms\": false, \"id\": 8}]\"install_app\"))) AND advt_age = ([{\"id\": 9}]2147483647) AND advt_gender contains ([{\"normalizeCase\": false, \"implicitTransforms\": false, \"id\": 10}]\"all\") AND advt_all_segments = ([{\"id\": 11}]2147483647) AND advt_keywords contains ([{\"normalizeCase\": false, \"implicitTransforms\": false, \"id\": 12}]\"all\") AND advMobilePlatform = ([{\"id\": 13}]2147483647) AND advMobileDeviceType = ([{\"id\": 14}]2147483647) AND advMobileCon = ([{\"id\": 15}]2147483647) AND advMobileOSVersions = ([{\"id\": 16}]2147483647) AND advCarrier = ([{\"id\": 17}]2147483647) AND ([{\"id\": 18}]weightedSet(advt_supply, {\"all\": 1, \"pub223\": 1, \"sec223\": 1, \"site223\": 1})) AND (advt_day_parting contains ([{\"normalizeCase\": false, \"implicitTransforms\": false, \"id\": 19, \"weight\": 1}]\"adv_tuesday\") OR advt_day_parting contains ([{\"normalizeCase\": false, \"implicitTransforms\": false, \"id\": 20, \"weight\": 1}]\"adv_tuesday_17\") OR advt_day_parting contains ([{\"normalizeCase\": false, \"implicitTransforms\": false, \"id\": 21, \"weight\": 1}]\"adv_tuesday_17_forty_five\") OR advt_day_parting contains ([{\"normalizeCase\": false, \"implicitTransforms\": false, \"id\": 22}]\"all\")) AND isAppReengagementAd = ([{\"id\": 23}]0) AND dummyField contains ([{\"normalizeCase\": false, \"implicitTransforms\": false, \"id\": 24}]\"default\") AND serveWithPromotionOnly = ([{\"id\": 26}]0) AND budgetAdvertiserThrottleRateFilter = ([{\"id\": 27}]0) AND budgetResellerThrottleRateFilter = ([{\"id\": 28}]0) AND (isMystiqueRequired = ([{\"id\": 29}]0) OR (isMystiqueRequired = ([{\"id\": 30}]1) AND useBcFactorFilter = ([{\"id\": 31}]1))) AND (((budgetCampaignThrottleRateBits = ([{\"id\": 32}]55) AND dummyField contains ([{\"normalizeCase\": false, \"implicitTransforms\": false, \"id\": 33}]\"default\"))) AND !(useBcFactorFilter = ([{\"id\": 34}]1)) OR ((useBcFactorFilter = ([{\"id\": 35}]1) AND dummyField contains ([{\"normalizeCase\": false, \"implicitTransforms\": false, \"id\": 36}]\"default\") AND (bcFactorTiers = ([{\"id\": 38}]127) OR bcFactorTiers = ([{\"id\": 39}]0)) AND ((firstPriceEnforced = ([{\"id\": 40}]0) AND (secondPriceEnforced = ([{\"id\": 41}]1) OR isPrivateDeal = ([{\"id\": 42}]0) OR (dummyField contains ([{\"normalizeCase\": false, \"implicitTransforms\": false, \"id\": 43}]\"default\")) AND !(bcActiveTier = ([{\"id\": 44}]0)))) OR mystiqueCampaignThrottleRateBits = ([{\"id\": 45}]18)))) AND !(isOutOfDailyBudget = ([{\"id\": 37}]1))) AND testCreative = ([{\"id\": 46}]0) AND advt_geo = ([{\"id\": 47}]2147483647) AND ((adType contains ([{\"normalizeCase\": false, \"implicitTransforms\": false, \"id\": 48}]\"strm_video\") AND isPortraitVideo = ([{\"id\": 49}]0)) OR adType contains ([{\"normalizeCase\": false, \"implicitTransforms\": false, \"id\": 50}]\"stream_ad\")) AND ((isCPM = ([{\"id\": 51}]0) AND isOCPC = ([{\"id\": 52}]0) AND isECPC = ([{\"id\": 53}]0) AND ((priceType contains ([{\"normalizeCase\": false, \"implicitTransforms\": false, \"id\": 54}]\"cpcv\") AND bid >= ([{\"id\": 55}]0.005)) OR (priceType contains ([{\"normalizeCase\": false, \"implicitTransforms\": false, \"id\": 56}]\"cpv\") AND bid >= ([{\"id\": 57}]0.01)) OR (priceType contains ([{\"normalizeCase\": false, \"implicitTransforms\": false, \"id\": 58}]\"cpc\") AND bid >= ([{\"id\": 59}]0.05)) OR (objective contains ([{\"normalizeCase\": false, \"implicitTransforms\": false, \"id\": 60}]\"promote_content\") AND bid >= ([{\"id\": 61}]0.01)) OR hasFloorPriceUsd = ([{\"id\": 62}]1))) OR isECPC = ([{\"id\": 63}]1) OR (isCPM = ([{\"id\": 64}]1) AND isOCPM = ([{\"id\": 65}]0) AND ([{\"id\": 66}]range(bid, 0.25, Infinity) OR hasFloorPriceUsd = ([{\"id\": 67}]1)))) AND start_date <= ([{\"id\": 68}]1572976776299L) AND end_date >= ([{\"id\": 69}]1572976776299L))) AND !(isHoldoutAd = ([{\"id\": 25}]1))) AND !((disclaimerExtensionsTypes contains ([{\"normalizeCase\": false, \"implicitTransforms\": false, \"id\": 70}]\"pharma\") OR ([{\"id\": 71}]weightedSet(exclusion_advt_supply, {\"extsite223\": 1, \"pub223\": 1, \"sec223\": 1, \"site223\": 1})) OR isPersonalized = ([{\"id\": 72}]1) OR blocked_section_ids contains ([{\"normalizeCase\": false, \"implicitTransforms\": false, \"id\": 73}]\"223\") OR blocked_publisher_ids contains ([{\"normalizeCase\": false, \"implicitTransforms\": false, \"id\": 74}]\"223\") OR blocked_site_ids contains ([{\"normalizeCase\": false, \"implicitTransforms\": false, \"id\": 75}]\"223\"))), ([{\"id\": 76, \"label\": \"ad_ocpc_max_cpc\"}]dotProduct(ocpc_max_cpc, {\"0\": 1})), ([{\"id\": 77, \"label\": \"ad_ocpc_min_cpc\"}]dotProduct(ocpc_min_cpc, {\"0\": 1})), ([{\"id\": 78, \"label\": \"ad_ocpc_max_alpha\"}]dotProduct(ocpc_max_alpha, {\"0\": 1})), ([{\"id\": 79, \"label\": \"ad_ocpc_min_alpha\"}]dotProduct(ocpc_min_alpha, {\"0\": 1})), ([{\"id\": 80, \"label\": \"ad_ocpc_alpha_0\"}]dotProduct(ocpc_alpha_0, {\"0\": 1})), ([{\"id\": 81, \"label\": \"ad_ocpc_alpha_1\"}]dotProduct(ocpc_alpha_1, {\"0\": 1})), (bidAdjustmentDayParting contains ([{\"normalizeCase\": false, \"implicitTransforms\": false, \"id\": 82, \"weight\": 1}]\"adv_tuesday\") OR bidAdjustmentDayParting contains ([{\"normalizeCase\": false, \"implicitTransforms\": false, \"id\": 83, \"weight\": 1}]\"adv_tuesday_17\") OR bidAdjustmentDayParting contains ([{\"normalizeCase\": false, \"implicitTransforms\": false, \"id\": 84, \"weight\": 1}]\"adv_tuesday_17_forty_five\") OR bidAdjustmentDayPartingForCostCap contains ([{\"normalizeCase\": false, \"implicitTransforms\": false, \"id\": 85, \"weight\": 1}]\"adv_tuesday\") OR bidAdjustmentDayPartingForCostCap contains ([{\"normalizeCase\": false, \"implicitTransforms\": false, \"id\": 86, \"weight\": 1}]\"adv_tuesday_17\") OR bidAdjustmentDayPartingForCostCap contains ([{\"normalizeCase\": false, \"implicitTransforms\": false, \"id\": 87, \"weight\": 1}]\"adv_tuesday_17_forty_five\")), bidAdjustmentForCpi = ([{\"id\": 88, \"weight\": 1}]223), ([{\"id\": 89, \"label\": \"boostingForBackfill\"}]dotProduct(boostingForBackfill, {\"priority\": 1000})))", + assertEquals("rank((((filter contains ([{normalizeCase: false, id: 1}]\"VideoAdsCappingTestCPM\") AND hasRankRestriction contains ([{normalizeCase: false, implicitTransforms: false, id: 2}]\"0\") AND ((objective contains ([{normalizeCase: false, implicitTransforms: false, id: 3}]\"install_app\") AND availableExtendedFields contains ([{normalizeCase: false, implicitTransforms: false, id: 4}]\"cpiparams\")) OR (availableExtendedFields contains ([{normalizeCase: false, implicitTransforms: false, id: 5}]\"appinstallinfo\") AND availableExtendedFields contains ([{normalizeCase: false, implicitTransforms: false, id: 6}]\"appmetroplexinfo\")) OR (dummyField contains ([{normalizeCase: false, implicitTransforms: false, id: 7}]\"default\")) AND !(objective contains ([{normalizeCase: false, implicitTransforms: false, id: 8}]\"install_app\"))) AND advt_age = ([{id: 9}]2147483647) AND advt_gender contains ([{normalizeCase: false, implicitTransforms: false, id: 10}]\"all\") AND advt_all_segments = ([{id: 11}]2147483647) AND advt_keywords contains ([{normalizeCase: false, implicitTransforms: false, id: 12}]\"all\") AND advMobilePlatform = ([{id: 13}]2147483647) AND advMobileDeviceType = ([{id: 14}]2147483647) AND advMobileCon = ([{id: 15}]2147483647) AND advMobileOSVersions = ([{id: 16}]2147483647) AND advCarrier = ([{id: 17}]2147483647) AND ([{id: 18}]weightedSet(advt_supply, {\"all\": 1, \"pub223\": 1, \"sec223\": 1, \"site223\": 1})) AND (advt_day_parting contains ([{normalizeCase: false, implicitTransforms: false, id: 19, weight: 1}]\"adv_tuesday\") OR advt_day_parting contains ([{normalizeCase: false, implicitTransforms: false, id: 20, weight: 1}]\"adv_tuesday_17\") OR advt_day_parting contains ([{normalizeCase: false, implicitTransforms: false, id: 21, weight: 1}]\"adv_tuesday_17_forty_five\") OR advt_day_parting contains ([{normalizeCase: false, implicitTransforms: false, id: 22}]\"all\")) AND isAppReengagementAd = ([{id: 23}]0) AND dummyField contains ([{normalizeCase: false, implicitTransforms: false, id: 24}]\"default\") AND serveWithPromotionOnly = ([{id: 26}]0) AND budgetAdvertiserThrottleRateFilter = ([{id: 27}]0) AND budgetResellerThrottleRateFilter = ([{id: 28}]0) AND (isMystiqueRequired = ([{id: 29}]0) OR (isMystiqueRequired = ([{id: 30}]1) AND useBcFactorFilter = ([{id: 31}]1))) AND (((budgetCampaignThrottleRateBits = ([{id: 32}]55) AND dummyField contains ([{normalizeCase: false, implicitTransforms: false, id: 33}]\"default\"))) AND !(useBcFactorFilter = ([{id: 34}]1)) OR ((useBcFactorFilter = ([{id: 35}]1) AND dummyField contains ([{normalizeCase: false, implicitTransforms: false, id: 36}]\"default\") AND (bcFactorTiers = ([{id: 38}]127) OR bcFactorTiers = ([{id: 39}]0)) AND ((firstPriceEnforced = ([{id: 40}]0) AND (secondPriceEnforced = ([{id: 41}]1) OR isPrivateDeal = ([{id: 42}]0) OR (dummyField contains ([{normalizeCase: false, implicitTransforms: false, id: 43}]\"default\")) AND !(bcActiveTier = ([{id: 44}]0)))) OR mystiqueCampaignThrottleRateBits = ([{id: 45}]18)))) AND !(isOutOfDailyBudget = ([{id: 37}]1))) AND testCreative = ([{id: 46}]0) AND advt_geo = ([{id: 47}]2147483647) AND ((adType contains ([{normalizeCase: false, implicitTransforms: false, id: 48}]\"strm_video\") AND isPortraitVideo = ([{id: 49}]0)) OR adType contains ([{normalizeCase: false, implicitTransforms: false, id: 50}]\"stream_ad\")) AND ((isCPM = ([{id: 51}]0) AND isOCPC = ([{id: 52}]0) AND isECPC = ([{id: 53}]0) AND ((priceType contains ([{normalizeCase: false, implicitTransforms: false, id: 54}]\"cpcv\") AND bid >= ([{id: 55}]0.005)) OR (priceType contains ([{normalizeCase: false, implicitTransforms: false, id: 56}]\"cpv\") AND bid >= ([{id: 57}]0.01)) OR (priceType contains ([{normalizeCase: false, implicitTransforms: false, id: 58}]\"cpc\") AND bid >= ([{id: 59}]0.05)) OR (objective contains ([{normalizeCase: false, implicitTransforms: false, id: 60}]\"promote_content\") AND bid >= ([{id: 61}]0.01)) OR hasFloorPriceUsd = ([{id: 62}]1))) OR isECPC = ([{id: 63}]1) OR (isCPM = ([{id: 64}]1) AND isOCPM = ([{id: 65}]0) AND ([{id: 66}]range(bid, 0.25, Infinity) OR hasFloorPriceUsd = ([{id: 67}]1)))) AND start_date <= ([{id: 68}]1572976776299L) AND end_date >= ([{id: 69}]1572976776299L))) AND !(isHoldoutAd = ([{id: 25}]1))) AND !((disclaimerExtensionsTypes contains ([{normalizeCase: false, implicitTransforms: false, id: 70}]\"pharma\") OR ([{id: 71}]weightedSet(exclusion_advt_supply, {\"extsite223\": 1, \"pub223\": 1, \"sec223\": 1, \"site223\": 1})) OR isPersonalized = ([{id: 72}]1) OR blocked_section_ids contains ([{normalizeCase: false, implicitTransforms: false, id: 73}]\"223\") OR blocked_publisher_ids contains ([{normalizeCase: false, implicitTransforms: false, id: 74}]\"223\") OR blocked_site_ids contains ([{normalizeCase: false, implicitTransforms: false, id: 75}]\"223\"))), ([{id: 76, label: \"ad_ocpc_max_cpc\"}]dotProduct(ocpc_max_cpc, {\"0\": 1})), ([{id: 77, label: \"ad_ocpc_min_cpc\"}]dotProduct(ocpc_min_cpc, {\"0\": 1})), ([{id: 78, label: \"ad_ocpc_max_alpha\"}]dotProduct(ocpc_max_alpha, {\"0\": 1})), ([{id: 79, label: \"ad_ocpc_min_alpha\"}]dotProduct(ocpc_min_alpha, {\"0\": 1})), ([{id: 80, label: \"ad_ocpc_alpha_0\"}]dotProduct(ocpc_alpha_0, {\"0\": 1})), ([{id: 81, label: \"ad_ocpc_alpha_1\"}]dotProduct(ocpc_alpha_1, {\"0\": 1})), (bidAdjustmentDayParting contains ([{normalizeCase: false, implicitTransforms: false, id: 82, weight: 1}]\"adv_tuesday\") OR bidAdjustmentDayParting contains ([{normalizeCase: false, implicitTransforms: false, id: 83, weight: 1}]\"adv_tuesday_17\") OR bidAdjustmentDayParting contains ([{normalizeCase: false, implicitTransforms: false, id: 84, weight: 1}]\"adv_tuesday_17_forty_five\") OR bidAdjustmentDayPartingForCostCap contains ([{normalizeCase: false, implicitTransforms: false, id: 85, weight: 1}]\"adv_tuesday\") OR bidAdjustmentDayPartingForCostCap contains ([{normalizeCase: false, implicitTransforms: false, id: 86, weight: 1}]\"adv_tuesday_17\") OR bidAdjustmentDayPartingForCostCap contains ([{normalizeCase: false, implicitTransforms: false, id: 87, weight: 1}]\"adv_tuesday_17_forty_five\")), bidAdjustmentForCpi = ([{id: 88, weight: 1}]223), ([{id: 89, label: \"boostingForBackfill\"}]dotProduct(boostingForBackfill, {\"priority\": 1000})))", serializedQueryTreeYql); } @Test public void testDottedFieldNames() { - assertParse("select foo from bar where my.nested.title contains \"madonna\";", + assertParse("select foo from bar where my.nested.title contains \"madonna\"", "my.nested.title:madonna"); } @Test public void testDottedNestedFieldNames() { - assertParse("select foo from bar where my.title contains \"madonna\";", + assertParse("select foo from bar where my.title contains \"madonna\"", "my.title:madonna"); } @Test public void testOr() { - assertParse("select foo from bar where title contains \"madonna\" or title contains \"saint\";", + assertParse("select foo from bar where title contains \"madonna\" or title contains \"saint\"", "OR title:madonna title:saint"); assertParse("select foo from bar where title contains \"madonna\" or title contains \"saint\" or title " + - "contains \"angel\";", + "contains \"angel\"", "OR title:madonna title:saint title:angel"); } @Test public void testAnd() { - assertParse("select foo from bar where title contains \"madonna\" and title contains \"saint\";", + assertParse("select foo from bar where title contains \"madonna\" and title contains \"saint\"", "AND title:madonna title:saint"); assertParse("select foo from bar where title contains \"madonna\" and title contains \"saint\" and title " + "contains \"angel\";", @@ -188,98 +188,98 @@ public class YqlParserTestCase { @Test public void testAndNot() { - assertParse("select foo from bar where title contains \"madonna\" and !(title contains \"saint\");", + assertParse("select foo from bar where title contains \"madonna\" and !(title contains \"saint\")", "+title:madonna -title:saint"); } @Test public void testLessThan() { - assertParse("select foo from bar where price < 500;", "price:<500"); - assertParse("select foo from bar where 500 < price;", "price:>500"); + assertParse("select foo from bar where price < 500", "price:<500"); + assertParse("select foo from bar where 500 < price", "price:>500"); } @Test public void testGreaterThan() { - assertParse("select foo from bar where price > 500;", "price:>500"); - assertParse("select foo from bar where 500 > price;", "price:<500"); + assertParse("select foo from bar where price > 500", "price:>500"); + assertParse("select foo from bar where 500 > price", "price:<500"); } @Test public void testLessThanOrEqual() { - assertParse("select foo from bar where price <= 500;", "price:[;500]"); - assertParse("select foo from bar where 500 <= price;", "price:[500;]"); + assertParse("select foo from bar where price <= 500", "price:[;500]"); + assertParse("select foo from bar where 500 <= price", "price:[500;]"); } @Test public void testGreaterThanOrEqual() { - assertParse("select foo from bar where price >= 500;", "price:[500;]"); - assertParse("select foo from bar where 500 >= price;", "price:[;500]"); + assertParse("select foo from bar where price >= 500", "price:[500;]"); + assertParse("select foo from bar where 500 >= price", "price:[;500]"); } @Test public void testEquality() { - assertParse("select foo from bar where price = 500;", "price:500"); - assertParse("select foo from bar where 500 = price;", "price:500"); + assertParse("select foo from bar where price = 500", "price:500"); + assertParse("select foo from bar where 500 = price", "price:500"); } @Test public void testNegativeLessThan() { - assertParse("select foo from bar where price < -500;", "price:<-500"); - assertParse("select foo from bar where -500 < price;", "price:>-500"); + assertParse("select foo from bar where price < -500", "price:<-500"); + assertParse("select foo from bar where -500 < price", "price:>-500"); } @Test public void testNegativeGreaterThan() { - assertParse("select foo from bar where price > -500;", "price:>-500"); - assertParse("select foo from bar where -500 > price;", "price:<-500"); + assertParse("select foo from bar where price > -500", "price:>-500"); + assertParse("select foo from bar where -500 > price", "price:<-500"); } @Test public void testNegativeLessThanOrEqual() { - assertParse("select foo from bar where price <= -500;", "price:[;-500]"); - assertParse("select foo from bar where -500 <= price;", "price:[-500;]"); + assertParse("select foo from bar where price <= -500", "price:[;-500]"); + assertParse("select foo from bar where -500 <= price", "price:[-500;]"); } @Test public void testNegativeGreaterThanOrEqual() { - assertParse("select foo from bar where price >= -500;", "price:[-500;]"); - assertParse("select foo from bar where -500 >= price;", "price:[;-500]"); + assertParse("select foo from bar where price >= -500", "price:[-500;]"); + assertParse("select foo from bar where -500 >= price", "price:[;-500]"); } @Test public void testNegativeEquality() { - assertParse("select foo from bar where price = -500;", "price:-500"); - assertParse("select foo from bar where -500 = price;", "price:-500"); + assertParse("select foo from bar where price = -500", "price:-500"); + assertParse("select foo from bar where -500 = price", "price:-500"); } @Test public void testAnnotatedLessThan() { - assertParse("select foo from bar where price < ([{\"filter\": true}](-500));", "|price:<-500"); - assertParse("select foo from bar where ([{\"filter\": true}]500) < price;", "|price:>500"); + assertParse("select foo from bar where price < ([{filter: true}](-500))", "|price:<-500"); + assertParse("select foo from bar where ([{filter: true}]500) < price", "|price:>500"); } @Test public void testAnnotatedGreaterThan() { - assertParse("select foo from bar where price > ([{\"filter\": true}]500);", "|price:>500"); - assertParse("select foo from bar where ([{\"filter\": true}](-500)) > price;", "|price:<-500"); + assertParse("select foo from bar where price > ([{filter: true}]500)", "|price:>500"); + assertParse("select foo from bar where ([{filter: true}](-500)) > price", "|price:<-500"); } @Test public void testAnnotatedLessThanOrEqual() { - assertParse("select foo from bar where price <= ([{\"filter\": true}](-500));", "|price:[;-500]"); - assertParse("select foo from bar where ([{\"filter\": true}]500) <= price;", "|price:[500;]"); + assertParse("select foo from bar where price <= ([{filter: true}](-500))", "|price:[;-500]"); + assertParse("select foo from bar where ([{filter: true}]500) <= price", "|price:[500;]"); } @Test public void testAnnotatedGreaterThanOrEqual() { - assertParse("select foo from bar where price >= ([{\"filter\": true}]500);", "|price:[500;]"); - assertParse("select foo from bar where ([{\"filter\": true}](-500)) >= price;", "|price:[;-500]"); + assertParse("select foo from bar where price >= ([{filter: true}]500)", "|price:[500;]"); + assertParse("select foo from bar where ([{filter: true}](-500)) >= price", "|price:[;-500]"); } @Test public void testAnnotatedEquality() { - assertParse("select foo from bar where price = ([{\"filter\": true}](-500));", "|price:-500"); - assertParse("select foo from bar where ([{\"filter\": true}]500) = price;", "|price:500"); + assertParse("select foo from bar where price = ([{filter: true}](-500))", "|price:-500"); + assertParse("select foo from bar where ([{filter: true}]500) = price", "|price:500"); } @Test @@ -296,26 +296,26 @@ public class YqlParserTestCase { public void testTermAnnotations() { assertEquals("merkelapp", getRootWord("select foo from bar where baz contains " + - "([ {\"label\": \"merkelapp\"} ]\"colors\");").getLabel()); + "([ {label: \"merkelapp\"} ]\"colors\");").getLabel()); assertEquals("another", getRootWord("select foo from bar where baz contains " + - "([ {\"annotations\": {\"cox\": \"another\"}} ]\"colors\");").getAnnotation("cox")); + "([ {annotations: {cox: \"another\"}} ]\"colors\")").getAnnotation("cox")); assertEquals(23.0, getRootWord("select foo from bar where baz contains " + - "([ {\"significance\": 23.0} ]\"colors\");").getSignificance(), 1E-6); + "([ {significance: 23.0} ]\"colors\")").getSignificance(), 1E-6); assertEquals(23, getRootWord("select foo from bar where baz contains " + - "([ {\"id\": 23} ]\"colors\");").getUniqueID()); + "([ {id: 23} ]\"colors\")").getUniqueID()); assertEquals(150, getRootWord("select foo from bar where baz contains " + - "([ {\"weight\": 150} ]\"colors\");").getWeight()); + "([ {weight: 150} ]\"colors\")").getWeight()); assertFalse(getRootWord("select foo from bar where baz contains " + - "([ {\"usePositionData\": false} ]\"colors\");").usePositionData()); + "([ {usePositionData: false} ]\"colors\")").usePositionData()); assertTrue(getRootWord("select foo from bar where baz contains " + - "([ {\"filter\": true} ]\"colors\");").isFilter()); + "([ {filter: true} ]\"colors\")").isFilter()); assertFalse(getRootWord("select foo from bar where baz contains " + - "([ {\"ranked\": false} ]\"colors\");").isRanked()); + "([ {ranked: false} ]\"colors\")").isRanked()); Substring origin = getRootWord("select foo from bar where baz contains " + - "([ {\"origin\": {\"original\": \"abc\", \"offset\": 1, \"length\": 2}} ]" + - "\"colors\");").getOrigin(); + "([ {origin: {original: \"abc\", 'offset': 1, length: 2}} ]" + + "\"colors\")").getOrigin(); assertEquals("abc", origin.string); assertEquals(1, origin.start); assertEquals(3, origin.end); @@ -323,34 +323,34 @@ public class YqlParserTestCase { @Test public void testSameElement() { - assertParse("select foo from bar where baz contains sameElement(f1 contains \"a\", f2 contains \"b\");", + assertParse("select foo from bar where baz contains sameElement(f1 contains \"a\", f2 contains \"b\")", "baz:{f1:a f2:b}"); - assertParse("select foo from bar where baz contains sameElement(f1 contains \"a\", f2 = 10);", + assertParse("select foo from bar where baz contains sameElement(f1 contains \"a\", f2 = 10)", "baz:{f1:a f2:10}"); - assertParse("select foo from bar where baz contains sameElement(key contains \"a\", value.f2 = 10);", + assertParse("select foo from bar where baz contains sameElement(key contains \"a\", value.f2 = 10)", "baz:{key:a value.f2:10}"); - assertCanonicalParse("select foo from bar where baz contains sameElement(key contains \"a\", value.f2 = 10);", + assertCanonicalParse("select foo from bar where baz contains sameElement(key contains \"a\", value.f2 = 10)", "baz:{key:a value.f2:10}"); - assertCanonicalParse("select foo from bar where baz contains sameElement(key contains \"a\");", + assertCanonicalParse("select foo from bar where baz contains sameElement(key contains \"a\")", "baz.key:a"); } @Test public void testPhrase() { - assertParse("select foo from bar where baz contains phrase(\"a\", \"b\");", + assertParse("select foo from bar where baz contains phrase(\"a\", \"b\")", "baz:\"a b\""); } @Test public void testNestedPhrase() { - assertParse("select foo from bar where baz contains phrase(\"a\", \"b\", phrase(\"c\", \"d\"));", + assertParse("select foo from bar where baz contains phrase(\"a\", \"b\", phrase(\"c\", \"d\"))", "baz:\"a b c d\""); } @Test public void testNestedPhraseSegment() { assertParse("select foo from bar where baz contains " + - "phrase(\"a\", \"b\", [ {\"origin\": {\"original\": \"c d\", \"offset\": 0, \"length\": 3}} ]" + + "phrase(\"a\", \"b\", [ {origin: {original: \"c d\", 'offset': 0, length: 3}} ]" + "phrase(\"c\", \"d\"));", "baz:\"a b 'c d'\""); } @@ -358,9 +358,9 @@ public class YqlParserTestCase { @Test public void testStemming() { assertTrue(getRootWord("select foo from bar where baz contains " + - "([ {\"stem\": false} ]\"colors\");").isStemmed()); + "([ {stem: false} ]\"colors\")").isStemmed()); assertFalse(getRootWord("select foo from bar where baz contains " + - "([ {\"stem\": true} ]\"colors\");").isStemmed()); + "([ {stem: true} ]\"colors\")").isStemmed()); assertFalse(getRootWord("select foo from bar where baz contains " + "\"colors\";").isStemmed()); } @@ -368,23 +368,23 @@ public class YqlParserTestCase { @Test public void testRaw() { // Default: Not raw, for comparison - Item root = parse("select foo from bar where baz contains (\"yoni jo dima\");").getRoot(); + Item root = parse("select foo from bar where baz contains (\"yoni jo dima\")").getRoot(); assertEquals("baz:'yoni jo dima'", root.toString()); assertFalse(root instanceof WordItem); assertTrue(root instanceof PhraseSegmentItem); - root = parse("select foo from bar where baz contains ([{\"grammar\":\"raw\"}]\"yoni jo dima\");").getRoot(); + root = parse("select foo from bar where baz contains ([{grammar:\"raw\"}]\"yoni jo dima\")").getRoot(); assertEquals("baz:yoni jo dima", root.toString()); assertTrue(root instanceof WordItem); assertFalse(root instanceof ExactStringItem); assertEquals("yoni jo dima", ((WordItem)root).getWord()); - root = parse("select foo from bar where userInput(\"yoni jo dima\");").getRoot(); + root = parse("select foo from bar where userInput(\"yoni jo dima\")").getRoot(); assertTrue(root instanceof AndItem); AndItem andItem = (AndItem) root; assertEquals(3, andItem.getItemCount()); - root = parse("select foo from bar where [{\"grammar\":\"raw\"}]userInput(\"yoni jo dima\");").getRoot(); + root = parse("select foo from bar where [{grammar:\"raw\"}]userInput(\"yoni jo dima\")").getRoot(); assertTrue(root instanceof WordItem); assertTrue(root instanceof ExactStringItem); assertEquals("yoni jo dima", ((WordItem)root).getWord()); @@ -393,65 +393,65 @@ public class YqlParserTestCase { @Test public void testAccentDropping() { assertFalse(getRootWord("select foo from bar where baz contains " + - "([ {\"accentDrop\": false} ]\"colors\");").isNormalizable()); + "([ {accentDrop: false} ]\"colors\")").isNormalizable()); assertTrue(getRootWord("select foo from bar where baz contains " + - "([ {\"accentDrop\": true} ]\"colors\");").isNormalizable()); + "([ {accentDrop: true} ]\"colors\")").isNormalizable()); assertTrue(getRootWord("select foo from bar where baz contains " + - "\"colors\";").isNormalizable()); + "\"colors\"").isNormalizable()); } @Test public void testCaseNormalization() { assertTrue(getRootWord("select foo from bar where baz contains " + - "([ {\"normalizeCase\": false} ]\"colors\");").isLowercased()); + "([ {normalizeCase: false} ]\"colors\")").isLowercased()); assertFalse(getRootWord("select foo from bar where baz contains " + - "([ {\"normalizeCase\": true} ]\"colors\");").isLowercased()); + "([ {normalizeCase: true} ]\"colors\")").isLowercased()); assertFalse(getRootWord("select foo from bar where baz contains " + - "\"colors\";").isLowercased()); + "\"colors\"").isLowercased()); } @Test public void testSegmentingRule() { assertEquals(SegmentingRule.PHRASE, getRootWord("select foo from bar where baz contains " + - "([ {\"andSegmenting\": false} ]\"colors\");").getSegmentingRule()); + "([ {andSegmenting: false} ]\"colors\")").getSegmentingRule()); assertEquals(SegmentingRule.BOOLEAN_AND, getRootWord("select foo from bar where baz contains " + - "([ {\"andSegmenting\": true} ]\"colors\");").getSegmentingRule()); + "([ {andSegmenting: true} ]\"colors\")").getSegmentingRule()); assertEquals(SegmentingRule.LANGUAGE_DEFAULT, getRootWord("select foo from bar where baz contains " + - "\"colors\";").getSegmentingRule()); + "\"colors\"").getSegmentingRule()); } @Test public void testNfkc() { assertEquals("a\u030a", getRootWord("select foo from bar where baz contains " + - "([ {\"nfkc\": false} ]\"a\\u030a\");").getWord()); + "([ {nfkc: false} ]\"a\\u030a\")").getWord()); assertEquals("\u00e5", getRootWord("select foo from bar where baz contains " + - "([ {\"nfkc\": true} ]\"a\\u030a\");").getWord()); + "([ {nfkc: true} ]\"a\\u030a\")").getWord()); assertEquals("No NKFC by default", "a\u030a", getRootWord("select foo from bar where baz contains " + - "(\"a\\u030a\");").getWord()); + "(\"a\\u030a\")").getWord()); } @Test public void testImplicitTransforms() { - assertFalse(getRootWord("select foo from bar where baz contains ([ {\"implicitTransforms\": " + - "false} ]\"cox\");").isFromQuery()); - assertTrue(getRootWord("select foo from bar where baz contains ([ {\"implicitTransforms\": " + - "true} ]\"cox\");").isFromQuery()); - assertTrue(getRootWord("select foo from bar where baz contains \"cox\";").isFromQuery()); + assertFalse(getRootWord("select foo from bar where baz contains ([ {implicitTransforms: " + + "false} ]\"cox\")").isFromQuery()); + assertTrue(getRootWord("select foo from bar where baz contains ([ {implicitTransforms: " + + "true} ]\"cox\")").isFromQuery()); + assertTrue(getRootWord("select foo from bar where baz contains \"cox\"").isFromQuery()); } @Test public void testConnectivity() { QueryTree parsed = parse("select foo from bar where " + - "title contains ([{\"id\": 1, \"connectivity\": {\"id\": 3, \"weight\": 7.0}}]\"madonna\") " + - "and title contains ([{\"id\": 2}]\"saint\") " + - "and title contains ([{\"id\": 3}]\"angel\");"); + "title contains ([{id: 1, connectivity: {\"id\": 3, weight: 7.0}}]\"madonna\") " + + "and title contains ([{id: 2}]\"saint\") " + + "and title contains ([{id: 3}]\"angel\")"); assertEquals("AND title:madonna title:saint title:angel", parsed.toString()); AndItem root = (AndItem)parsed.getRoot(); @@ -463,9 +463,9 @@ public class YqlParserTestCase { assertNull(second.getConnectedItem()); assertParseFail("select foo from bar where " + - "title contains ([{\"id\": 1, \"connectivity\": {\"id\": 4, \"weight\": 7.0}}]\"madonna\") " + - "and title contains ([{\"id\": 2}]\"saint\") " + - "and title contains ([{\"id\": 3}]\"angel\");", + "title contains ([{id: 1, connectivity: {id: 4, weight: 7.0}}]\"madonna\") " + + "and title contains ([{id: 2}]\"saint\") " + + "and title contains ([{id: 3}]\"angel\")", new IllegalArgumentException("Item 'title:madonna' was specified to connect to item with ID 4, " + "which does not exist in the query.")); } @@ -473,7 +473,7 @@ public class YqlParserTestCase { @Test public void testAnnotatedPhrase() { QueryTree parsed = - parse("select foo from bar where baz contains ([{\"label\": \"hello world\"}]phrase(\"a\", \"b\"));"); + parse("select foo from bar where baz contains ([{label: \"hello world\"}]phrase(\"a\", \"b\"))"); assertEquals("baz:\"a b\"", parsed.toString()); PhraseItem phrase = (PhraseItem)parsed.getRoot(); assertEquals("hello world", phrase.getLabel()); @@ -481,39 +481,39 @@ public class YqlParserTestCase { @Test public void testRange() { - QueryTree parsed = parse("select foo from bar where range(baz,1,8);"); + QueryTree parsed = parse("select foo from bar where range(baz,1,8)"); assertEquals("baz:[1;8]", parsed.toString()); } @Test public void testRangeWithEndInfinity() { - QueryTree parsed = parse("select foo from bar where range(baz,1,Infinity);"); + QueryTree parsed = parse("select foo from bar where range(baz,1,Infinity)"); assertEquals("baz:[1;]", parsed.toString()); } @Test public void testRangeWithStartInfinity() { - QueryTree parsed = parse("select foo from bar where range(baz,-Infinity,8);"); + QueryTree parsed = parse("select foo from bar where range(baz,-Infinity,8)"); assertEquals("baz:[;8]", parsed.toString()); } @Test public void testNegativeRange() { - QueryTree parsed = parse("select foo from bar where range(baz,-8,-1);"); + QueryTree parsed = parse("select foo from bar where range(baz,-8,-1)"); assertEquals("baz:[-8;-1]", parsed.toString()); } @Test public void testRangeIllegalArguments() { - assertParseFail("select foo from bar where range(baz,cox,8);", + assertParseFail("select foo from bar where range(baz,cox,8)", new IllegalArgumentException("Expected a numerical argument (or 'Infinity') to range but got 'cox'")); } @Test public void testNear() { - assertParse("select foo from bar where description contains near(\"a\", \"b\");", + assertParse("select foo from bar where description contains near(\"a\", \"b\")", "NEAR(2) description:a description:b"); - assertParse("select foo from bar where description contains ([ {\"distance\": 100} ]near(\"a\", \"b\"));", + assertParse("select foo from bar where description contains ([ {distance: 100} ]near(\"a\", \"b\"))", "NEAR(100) description:a description:b"); } @@ -521,25 +521,31 @@ public class YqlParserTestCase { public void testOrderedNear() { assertParse("select foo from bar where description contains onear(\"a\", \"b\");", "ONEAR(2) description:a description:b"); - assertParse("select foo from bar where description contains ([ {\"distance\": 100} ]onear(\"a\", \"b\"));", + assertParse("select foo from bar where description contains ([ {distance: 100} ]onear(\"a\", \"b\"))", "ONEAR(100) description:a description:b"); } - //This test is order dependent. Fix this!! @Test public void testWand() { assertParse("select foo from bar where wand(description, {\"a\":1, \"b\":2});", "WAND(10,0.0,1.0) description{[1]:\"a\",[2]:\"b\"}"); - assertParse("select foo from bar where [ {\"scoreThreshold\": 13.3, \"targetHits\": 7, " + - "\"thresholdBoostFactor\": 2.3} ]wand(description, {\"a\":1, \"b\":2});", + assertParse("select foo from bar where [ {scoreThreshold : 13.3, targetHits: 7, " + + "thresholdBoostFactor: 2.3} ]wand(description, {\"a\":1, \"b\":2})", "WAND(7,13.3,2.3) description{[1]:\"a\",[2]:\"b\"}"); } @Test + public void testQuotedAnnotations() { + assertParse("select foo from bar where [ {\"scoreThreshold\": 13.3, \"targetHits\": 7, " + + "'thresholdBoostFactor': 2.3} ]wand(description, {\"a\":1})", + "WAND(7,13.3,2.3) description{[1]:\"a\"}"); + } + + @Test public void testNumericWand() { String numWand = "WAND(10,0.0,1.0) description{[1]:\"11\",[2]:\"37\"}"; - assertParse("select foo from bar where wand(description, [[11,1], [37,2]]);", numWand); - assertParse("select foo from bar where wand(description, [[11L,1], [37L,2]]);", numWand); + assertParse("select foo from bar where wand(description, [[11,1], [37,2]])", numWand); + assertParse("select foo from bar where wand(description, [[11L,1], [37L,2]])", numWand); assertParseFail("select foo from bar where wand(description, 12);", new IllegalArgumentException("Expected ARRAY or MAP, got LITERAL.")); } @@ -547,9 +553,9 @@ public class YqlParserTestCase { @Test //This test is order dependent. Fix it! public void testWeightedSet() { - assertParse("select foo from bar where weightedSet(description, {\"a\":1, \"b\":2});", + assertParse("select foo from bar where weightedSet(description, {\"a\":1, \"b\":2})", "WEIGHTEDSET description{[1]:\"a\",[2]:\"b\"}"); - assertParseFail("select foo from bar where weightedSet(description, {\"a\":g, \"b\":2});", + assertParseFail("select foo from bar where weightedSet(description, {\"a\":g, \"b\":2})", new IllegalInputException("com.yahoo.search.yql.ProgramCompileException: " + "query:L1:56 no viable alternative at input 'weightedSet(description, {\"a\":g'")); assertParseFail("select foo from bar where weightedSet(description);", @@ -561,25 +567,25 @@ public class YqlParserTestCase { public void testDotProduct() { assertParse("select foo from bar where dotProduct(description, {\"a\":1, \"b\":2});", "DOTPRODUCT description{[1]:\"a\",[2]:\"b\"}"); - assertParse("select foo from bar where dotProduct(description, {\"a\":2});", + assertParse("select foo from bar where dotProduct(description, {\"a\":2})", "DOTPRODUCT description{[2]:\"a\"}"); } @Test public void testGeoLocation() { - assertParse("select foo from bar where geoLocation(workplace, 63.418417, 10.433033, \"0.5 deg\");", + assertParse("select foo from bar where geoLocation(workplace, 63.418417, 10.433033, \"0.5 deg\")", "GEO_LOCATION workplace:(2,10433033,63418417,500000,0,1,0,1921876103)"); - assertParse("select foo from bar where geoLocation(headquarters, \"37.416383\", \"-122.024683\", \"100 miles\");", + assertParse("select foo from bar where geoLocation(headquarters, \"37.416383\", \"-122.024683\", \"100 miles\")", "GEO_LOCATION headquarters:(2,-122024683,37416383,1450561,0,1,0,3411238761)"); - assertParse("select foo from bar where geoLocation(home, \"E10.433033\", \"N63.418417\", \"5km\");", + assertParse("select foo from bar where geoLocation(home, \"E10.433033\", \"N63.418417\", \"5km\")", "GEO_LOCATION home:(2,10433033,63418417,45066,0,1,0,1921876103)"); - assertParseFail("select foo from bar where geoLocation(qux, 1, 2);", + assertParseFail("select foo from bar where geoLocation(qux, 1, 2)", new IllegalArgumentException("Expected 4 arguments, got 3.")); assertParseFail("select foo from bar where geoLocation(qux, 2.0, \"N5.0\", \"0.5 deg\");", new IllegalArgumentException( "Invalid geoLocation coordinates 'Latitude: 2.0 degrees' and 'Latitude: 5.0 degrees'")); - assertParse("select foo from bar where geoLocation(workplace, -12, -34, \"-77 d\");", + assertParse("select foo from bar where geoLocation(workplace, -12, -34, \"-77 d\")", "GEO_LOCATION workplace:(2,-34000000,-12000000,-1,0,1,0,4201111954)"); } @@ -587,71 +593,71 @@ public class YqlParserTestCase { public void testNearestNeighbor() { assertParse("select foo from bar where nearestNeighbor(semantic_embedding, my_vector);", "NEAREST_NEIGHBOR {field=semantic_embedding,queryTensorName=my_vector,hnsw.exploreAdditionalHits=0,distanceThreshold=Infinity,approximate=true,targetHits=0}"); - assertParse("select foo from bar where [{\"targetHits\": 37}]nearestNeighbor(semantic_embedding, my_vector);", + assertParse("select foo from bar where [{targetHits: 37}]nearestNeighbor(semantic_embedding, my_vector)", "NEAREST_NEIGHBOR {field=semantic_embedding,queryTensorName=my_vector,hnsw.exploreAdditionalHits=0,distanceThreshold=Infinity,approximate=true,targetHits=37}"); - assertParse("select foo from bar where [{\"approximate\": false, \"hnsw.exploreAdditionalHits\": 8, \"targetHits\": 3}]nearestNeighbor(semantic_embedding, my_vector);", + assertParse("select foo from bar where [{\"approximate\": false, \"hnsw.exploreAdditionalHits\": 8, targetHits: 3}]nearestNeighbor(semantic_embedding, my_vector)", "NEAREST_NEIGHBOR {field=semantic_embedding,queryTensorName=my_vector,hnsw.exploreAdditionalHits=8,distanceThreshold=Infinity,approximate=false,targetHits=3}"); - assertParse("select foo from bar where [{\"targetHits\": 7, \"distanceThreshold\": 100100.25}]nearestNeighbor(semantic_embedding, my_vector);", + assertParse("select foo from bar where [{targetHits: 7, \"distanceThreshold\": 100100.25}]nearestNeighbor(semantic_embedding, my_vector)", "NEAREST_NEIGHBOR {field=semantic_embedding,queryTensorName=my_vector,hnsw.exploreAdditionalHits=0,distanceThreshold=100100.25,approximate=true,targetHits=7}"); } @Test public void testTrueAndFalse() { - assertParse("select foo from bar where true;", "TRUE"); - assertParse("select foo from bar where false;", "FALSE"); - assertParse("select foo from bar where ((title contains \"foo\") AND true) AND !((title contains \"bar\") or false);", + assertParse("select foo from bar where true", "TRUE"); + assertParse("select foo from bar where false", "FALSE"); + assertParse("select foo from bar where ((title contains \"foo\") AND true) AND !((title contains \"bar\") or false)", "+(AND title:foo TRUE) -(OR title:bar FALSE)"); } @Test public void testPredicate() { assertParse("select foo from bar where predicate(predicate_field, " + - "{\"gender\":\"male\", \"hobby\":[\"music\", \"hiking\"]}, {\"age\":23L});", + "{\"gender\":\"male\", \"hobby\":[\"music\", \"hiking\"]}, {\"age\":23L})", "PREDICATE_QUERY_ITEM gender=male, hobby=music, hobby=hiking, age:23"); assertParse("select foo from bar where predicate(predicate_field, " + - "{\"gender\":\"male\", \"hobby\":[\"music\", \"hiking\"]}, {\"age\":23});", + "{\"gender\":\"male\", \"hobby\":[\"music\", \"hiking\"]}, {\"age\":23})", "PREDICATE_QUERY_ITEM gender=male, hobby=music, hobby=hiking, age:23"); - assertParse("select foo from bar where predicate(predicate_field, 0, void);", + assertParse("select foo from bar where predicate(predicate_field, 0, void)", "PREDICATE_QUERY_ITEM "); } @Test public void testPredicateWithSubQueries() { assertParse("select foo from bar where predicate(predicate_field, " + - "{\"0x03\":{\"gender\":\"male\"},\"0x01\":{\"hobby\":[\"music\", \"hiking\"]}}, {\"0x80ffffffffffffff\":{\"age\":23L}});", + "{\"0x03\":{\"gender\":\"male\"},\"0x01\":{\"hobby\":[\"music\", \"hiking\"]}}, {\"0x80ffffffffffffff\":{\"age\":23L}})", "PREDICATE_QUERY_ITEM gender=male[0x3], hobby=music[0x1], hobby=hiking[0x1], age:23[0x80ffffffffffffff]"); - assertParseFail("select foo from bar where predicate(foo, null, {\"0x80000000000000000\":{\"age\":23}});", + assertParseFail("select foo from bar where predicate(foo, null, {\"0x80000000000000000\":{\"age\":23}})", new NumberFormatException("Too long subquery string: 0x80000000000000000")); assertParse("select foo from bar where predicate(predicate_field, " + - "{\"[0,1]\":{\"gender\":\"male\"},\"[0]\":{\"hobby\":[\"music\", \"hiking\"]}}, {\"[62, 63]\":{\"age\":23L}});", + "{\"[0,1]\":{\"gender\":\"male\"},\"[0]\":{\"hobby\":[\"music\", \"hiking\"]}}, {\"[62, 63]\":{\"age\":23L}})", "PREDICATE_QUERY_ITEM gender=male[0x3], hobby=music[0x1], hobby=hiking[0x1], age:23[0xc000000000000000]"); } @Test public void testRank() { - assertParse("select foo from bar where rank(a contains \"A\", b contains \"B\");", + assertParse("select foo from bar where rank(a contains \"A\", b contains \"B\")", "RANK a:A b:B"); assertParse("select foo from bar where rank(a contains \"A\", b contains \"B\", c " + - "contains \"C\");", + "contains \"C\")", "RANK a:A b:B c:C"); assertParse("select foo from bar where rank(a contains \"A\", b contains \"B\" or c " + - "contains \"C\");", + "contains \"C\")", "RANK a:A (OR b:B c:C)"); } @Test @SuppressWarnings("deprecation") public void testWeakAnd() { - assertParse("select foo from bar where weakAnd(a contains \"A\", b contains \"B\");", + assertParse("select foo from bar where weakAnd(a contains \"A\", b contains \"B\")", "WEAKAND(100) a:A b:B"); assertParse("select foo from bar where [{\"targetHits\": 37}]weakAnd(a contains \"A\", " + - "b contains \"B\");", + "b contains \"B\")", "WEAKAND(37) a:A b:B"); QueryTree tree = parse("select foo from bar where [{\"scoreThreshold\": 41}]weakAnd(a " + - "contains \"A\", b contains \"B\");"); + "contains \"A\", b contains \"B\")"); assertEquals("WEAKAND(100) a:A b:B", tree.toString()); assertEquals(WeakAndItem.class, tree.getRoot().getClass()); assertEquals(41, ((WeakAndItem)tree.getRoot()).getScoreThreshold()); @@ -659,111 +665,103 @@ public class YqlParserTestCase { @Test public void testEquiv() { - assertParse("select foo from bar where fieldName contains equiv(\"A\",\"B\");", + assertParse("select foo from bar where fieldName contains equiv(\"A\",\"B\")", "EQUIV fieldName:A fieldName:B"); assertParse("select foo from bar where fieldName contains " + "equiv(\"ny\",phrase(\"new\",\"york\"));", "EQUIV fieldName:ny fieldName:\"new york\""); - assertParseFail("select foo from bar where fieldName contains equiv(\"ny\");", + assertParseFail("select foo from bar where fieldName contains equiv(\"ny\")", new IllegalArgumentException("Expected 2 or more arguments, got 1.")); - assertParseFail("select foo from bar where fieldName contains equiv(\"ny\", nalle(void));", + assertParseFail("select foo from bar where fieldName contains equiv(\"ny\", nalle(void))", new IllegalArgumentException("Expected function 'phrase', got 'nalle'.")); - assertParseFail("select foo from bar where fieldName contains equiv(\"ny\", 42);", + assertParseFail("select foo from bar where fieldName contains equiv(\"ny\", 42)", new ClassCastException("Cannot cast java.lang.Integer to java.lang.String")); } @Test public void testAffixItems() { - assertRootClass("select foo from bar where baz contains ([ {\"suffix\": true} ]\"colors\");", + assertRootClass("select foo from bar where baz contains ([ {suffix: true} ]\"colors\")", SuffixItem.class); - assertRootClass("select foo from bar where baz contains ([ {\"prefix\": true} ]\"colors\");", + assertRootClass("select foo from bar where baz contains ([ {\"prefix\": true} ]\"colors\")", PrefixItem.class); - assertRootClass("select foo from bar where baz contains ([ {\"substring\": true} ]\"colors\");", + assertRootClass("select foo from bar where baz contains ([ {substring: true} ]\"colors\")", SubstringItem.class); - assertParseFail("select foo from bar where description contains ([ {\"suffix\": true, " + - "\"prefix\": true} ]\"colors\");", + assertParseFail("select foo from bar where description contains ([ {suffix: true, " + + "prefix: true} ]\"colors\")", new IllegalArgumentException("Only one of prefix, substring and suffix can be set.")); - assertParseFail("select foo from bar where description contains ([ {\"suffix\": true, " + - "\"substring\": true} ]\"colors\");", + assertParseFail("select foo from bar where description contains ([ {suffix: true, " + + "substring: true} ]\"colors\")", new IllegalArgumentException("Only one of prefix, substring and suffix can be set.")); } @Test public void testLongNumberInSimpleExpression() { - assertParse("select foo from bar where price = 8589934592L;", - "price:8589934592"); + assertParse("select foo from bar where price = 8589934592L", "price:8589934592"); } @Test public void testNegativeLongNumberInSimpleExpression() { - assertParse("select foo from bar where price = -8589934592L;", - "price:-8589934592"); + assertParse("select foo from bar where price = -8589934592L", "price:-8589934592"); } @Test public void testSources() { - assertSources("select foo from sourceA where price <= 500;", - Arrays.asList("sourceA")); + assertSources("select foo from sourceA where price <= 500", List.of("sourceA")); + } + + @Test + public void testQueryWithSemicolon() { + assertParse("select foo from bar where price = 1", "price:1"); } @Test public void testSourcesWithDash() { - assertSources("select foo from source-a where price <= 500;", - Arrays.asList("source-a")); + assertSources("select foo from source-a where price <= 500", List.of("source-a")); } @Test public void testWildCardSources() { - assertSources("select foo from sources * where price <= 500;", - Collections.<String>emptyList()); + assertSources("select foo from sources * where price <= 500", List.of()); } @Test public void testMultiSources() { - assertSources("select foo from sources sourceA, sourceB where price <= 500;", - Arrays.asList("sourceA", "sourceB")); + assertSources("select foo from sources sourceA, sourceB where price <= 500", List.of("sourceA", "sourceB")); } @Test public void testFields() { - assertSummaryFields("select fieldA from bar where price <= 500;", - Arrays.asList("fieldA")); - assertSummaryFields("select fieldA, fieldB from bar where price <= 500;", - Arrays.asList("fieldA", "fieldB")); - assertSummaryFields("select fieldA, fieldB, fieldC from bar where price <= 500;", - Arrays.asList("fieldA", "fieldB", "fieldC")); - assertSummaryFields("select * from bar where price <= 500;", - Collections.<String>emptyList()); + assertSummaryFields("select fieldA from bar where price <= 500", List.of("fieldA")); + assertSummaryFields("select fieldA, fieldB from bar where price <= 500", List.of("fieldA", "fieldB")); + assertSummaryFields("select fieldA, fieldB, fieldC from bar where price <= 500", List.of("fieldA", "fieldB", "fieldC")); + assertSummaryFields("select * from bar where price <= 500", List.of()); } @Test public void testFieldsRoot() { - assertParse("select * from bar where price <= 500;", - "price:[;500]"); + assertParse("select * from bar where price <= 500", "price:[;500]"); } @Test public void testOffset() { - assertParse("select foo from bar where title contains \"madonna\" offset 37;", - "title:madonna"); + assertParse("select foo from bar where title contains \"madonna\" offset 37", "title:madonna"); assertEquals(Integer.valueOf(37), parser.getOffset()); } @Test public void testLimit() { - assertParse("select foo from bar where title contains \"madonna\" limit 29;", - "title:madonna"); + assertParse("select foo from bar where title contains \"madonna\" limit 29", "title:madonna"); assertEquals(Integer.valueOf(29), parser.getHits()); } @Test public void testOffsetAndLimit() { - assertParse("select foo from bar where title contains \"madonna\" limit 31 offset 29;", + assertParse("select foo from bar where title contains \"madonna\" limit 31 offset 29", "title:madonna"); assertEquals(Integer.valueOf(29), parser.getOffset()); assertEquals(Integer.valueOf(2), parser.getHits()); - assertParse("select * from bar where title contains \"madonna\" limit 41 offset 37;", + assertParse("select * from bar where title contains \"madonna\" limit 41 offset 37", "title:madonna"); assertEquals(Integer.valueOf(37), parser.getOffset()); assertEquals(Integer.valueOf(4), parser.getHits()); @@ -771,19 +769,17 @@ public class YqlParserTestCase { @Test public void testTimeout() { - assertParse("select * from bar where title contains \"madonna\" timeout 7;", - "title:madonna"); + assertParse("select * from bar where title contains \"madonna\" timeout 7", "title:madonna"); assertEquals(Integer.valueOf(7), parser.getTimeout()); - assertParse("select foo from bar where title contains \"madonna\" limit 600 timeout 3;", - "title:madonna"); + assertParse("select foo from bar where title contains \"madonna\" limit 600 timeout 3", "title:madonna"); assertEquals(Integer.valueOf(3), parser.getTimeout()); } @Test public void testOrdering() { assertParse("select foo from bar where title contains \"madonna\" order by something asc, " + - "shoesize desc limit 600 timeout 3;", + "shoesize desc limit 600 timeout 3", "title:madonna"); assertEquals(2, parser.getSorting().fieldOrders().size()); assertEquals("something", parser.getSorting().fieldOrders().get(0).getFieldName()); @@ -792,7 +788,7 @@ public class YqlParserTestCase { assertEquals(Order.DESCENDING, parser.getSorting().fieldOrders().get(1).getSortOrder()); assertParse("select foo from bar where title contains \"madonna\" order by other limit 600 " + - "timeout 3;", + "timeout 3", "title:madonna"); assertEquals("other", parser.getSorting().fieldOrders().get(0).getFieldName()); assertEquals(Order.ASCENDING, parser.getSorting().fieldOrders().get(0).getSortOrder()); @@ -802,8 +798,8 @@ public class YqlParserTestCase { public void testAnnotatedOrdering() { assertParse( "select foo from bar where title contains \"madonna\"" - + " order by [{\"function\": \"uca\", \"locale\": \"en_US\", \"strength\": \"IDENTICAL\"}]other desc" - + " limit 600" + " timeout 3;", "title:madonna"); + + " order by [{function: \"uca\", locale: \"en_US\", strength: \"IDENTICAL\"}]other desc" + + " limit 600" + " timeout 3", "title:madonna"); FieldOrder fieldOrder = parser.getSorting().fieldOrders().get(0); assertEquals("other", fieldOrder.getFieldName()); assertEquals(Order.DESCENDING, fieldOrder.getSortOrder()); @@ -820,7 +816,7 @@ public class YqlParserTestCase { "select foo from bar where title contains \"madonna\"" + " order by [{\"function\": \"uca\", \"locale\": \"en_US\", \"strength\": \"IDENTICAL\"}]other desc," + " [{\"function\": \"lowercase\"}]something asc" - + " limit 600" + " timeout 3;", "title:madonna"); + + " limit 600" + " timeout 3", "title:madonna"); { FieldOrder fieldOrder = parser.getSorting().fieldOrders().get(0); assertEquals("other", fieldOrder.getFieldName()); @@ -842,29 +838,29 @@ public class YqlParserTestCase { @Test public void testSegmenting() { - assertParse("select * from bar where title contains 'foo.bar';", - "title:'foo bar'"); - - assertParse("select * from bar where title contains 'foo&123';", - "title:'foo 123'"); + assertParse("select * from bar where title contains 'foo.bar'", "title:'foo bar'"); + assertParse("select * from bar where title contains 'foo&123'", "title:'foo 123'"); } @Test public void testNegativeHitLimit() { - assertParse("select * from sources * where [{\"hitLimit\": -38}]range(foo, 0, 1);", - "foo:[0;1;-38]"); + assertParse("select * from sources * where [{hitLimit: -38}]range(foo, 0, 1)", "foo:[0;1;-38]"); } @Test public void testRangeSearchHitPopulationOrdering() { - assertParse("select * from sources * where [{\"hitLimit\": 38, \"ascending\": true}]range(foo, 0, 1);", "foo:[0;1;38]"); - assertParse("select * from sources * where [{\"hitLimit\": 38, \"ascending\": false}]range(foo, 0, 1);", "foo:[0;1;-38]"); - assertParse("select * from sources * where [{\"hitLimit\": 38, \"descending\": true}]range(foo, 0, 1);", "foo:[0;1;-38]"); - assertParse("select * from sources * where [{\"hitLimit\": 38, \"descending\": false}]range(foo, 0, 1);", "foo:[0;1;38]"); + assertParse("select * from sources * where [{hitLimit: 38, \"ascending\": true}]range(foo, 0, 1)", + "foo:[0;1;38]"); + assertParse("select * from sources * where [{hitLimit: 38, \"ascending\": false}]range(foo, 0, 1)", + "foo:[0;1;-38]"); + assertParse("select * from sources * where [{hitLimit: 38, \"descending\": true}]range(foo, 0, 1)", + "foo:[0;1;-38]"); + assertParse("select * from sources * where [{hitLimit: 38, \"descending\": false}]range(foo, 0, 1)", + "foo:[0;1;38]"); boolean gotExceptionFromParse = false; try { - parse("select * from sources * where [{\"hitLimit\": 38, \"ascending\": true, \"descending\": false}]range(foo, 0, 1);"); + parse("select * from sources * where [{hitLimit: 38, ascending: true, descending: false}]range(foo, 0, 1)"); } catch (IllegalArgumentException e) { assertTrue("Expected information about abuse of settings.", e.getMessage().contains("both ascending and descending ordering set")); @@ -875,23 +871,23 @@ public class YqlParserTestCase { @Test public void testOpenIntervals() { - assertParse("select * from sources * where range(title, 0.0, 500.0);", + assertParse("select * from sources * where range(title, 0.0, 500.0)", "title:[0.0;500.0]"); assertParse( - "select * from sources * where [{\"bounds\": \"open\"}]range(title, 0.0, 500.0);", + "select * from sources * where [{bounds: \"open\"}]range(title, 0.0, 500.0)", "title:<0.0;500.0>"); assertParse( - "select * from sources * where [{\"bounds\": \"leftOpen\"}]range(title, 0.0, 500.0);", + "select * from sources * where [{bounds: \"leftOpen\"}]range(title, 0.0, 500.0)", "title:<0.0;500.0]"); assertParse( - "select * from sources * where [{\"bounds\": \"rightOpen\"}]range(title, 0.0, 500.0);", + "select * from sources * where [{bounds: \"rightOpen\"}]range(title, 0.0, 500.0)", "title:[0.0;500.0>"); } @Test public void testInheritedAnnotations() { { - QueryTree x = parse("select * from sources * where ([{\"ranked\": false}](foo contains \"a\" and bar contains \"b\")) or foor contains ([{\"ranked\": false}]\"c\");"); + QueryTree x = parse("select * from sources * where ([{ranked: false}](foo contains \"a\" and bar contains \"b\")) or foor contains ([{ranked: false}]\"c\")"); List<IndexedItem> terms = QueryTree.getPositiveTerms(x); assertEquals(3, terms.size()); for (IndexedItem term : terms) { @@ -899,7 +895,7 @@ public class YqlParserTestCase { } } { - QueryTree x = parse("select * from sources * where [{\"ranked\": false}](foo contains \"a\" and bar contains \"b\");"); + QueryTree x = parse("select * from sources * where [{ranked: false}](foo contains \"a\" and bar contains \"b\")"); List<IndexedItem> terms = QueryTree.getPositiveTerms(x); assertEquals(2, terms.size()); for (IndexedItem term : terms) { @@ -911,10 +907,10 @@ public class YqlParserTestCase { @Test public void testMoreInheritedAnnotations() { String yqlQuery = "select * from sources * where " + - "([{\"ranked\": false}](foo contains \"a\" " + - "and ([{\"ranked\": true}](bar contains \"b\" " + - "or ([{\"ranked\": false}](foo contains \"c\" " + - "and foo contains ([{\"ranked\": true}]\"d\")))))));"; + "([{ranked: false}](foo contains \"a\" " + + "and ([{ranked: true}](bar contains \"b\" " + + "or ([{ranked: false}](foo contains \"c\" " + + "and foo contains ([{ranked: true}]\"d\")))))))"; QueryTree x = parse(yqlQuery); List<IndexedItem> terms = QueryTree.getPositiveTerms(x); assertEquals(4, terms.size()); @@ -945,7 +941,7 @@ public class YqlParserTestCase { ParserEnvironment parserEnvironment = new ParserEnvironment().setIndexFacts(indexFacts); YqlParser configuredParser = new YqlParser(parserEnvironment); QueryTree x = configuredParser.parse(new Parsable() - .setQuery("select * from sources * where title contains \"a\" and song contains \"b\";")); + .setQuery("select * from sources * where title contains \"a\" and song contains \"b\"")); List<IndexedItem> terms = QueryTree.getPositiveTerms(x); assertEquals(2, terms.size()); for (IndexedItem term : terms) { @@ -955,7 +951,7 @@ public class YqlParserTestCase { @Test public void testRegexp() { - QueryTree x = parse("select * from sources * where foo matches \"a b\";"); + QueryTree x = parse("select * from sources * where foo matches \"a b\""); Item root = x.getRoot(); assertSame(RegExpItem.class, root.getClass()); assertEquals("a b", ((RegExpItem) root).stringValue()); @@ -963,7 +959,7 @@ public class YqlParserTestCase { @Test public void testWordAlternatives() { - QueryTree x = parse("select * from sources * where foo contains alternatives({\"trees\": 1.0, \"tree\": 0.7});"); + QueryTree x = parse("select * from sources * where foo contains alternatives({trees: 1.0, \"tree\": 0.7})"); Item root = x.getRoot(); assertSame(WordAlternativesItem.class, root.getClass()); WordAlternativesItem alternatives = (WordAlternativesItem) root; @@ -973,8 +969,8 @@ public class YqlParserTestCase { @Test public void testWordAlternativesWithOrigin() { QueryTree q = parse("select * from sources * where foo contains" + - " ([{\"origin\": {\"original\": \" trees \", \"offset\": 1, \"length\": 5}}]" + - "alternatives({\"trees\": 1.0, \"tree\": 0.7}));"); + " ([{origin: {original: \" trees \", 'offset': 1, length: 5}}]" + + "alternatives({trees: 1.0, tree: 0.7}))"); Item root = q.getRoot(); assertSame(WordAlternativesItem.class, root.getClass()); WordAlternativesItem alternatives = (WordAlternativesItem) root; @@ -989,7 +985,7 @@ public class YqlParserTestCase { @Test public void testWordAlternativesInPhrase() { QueryTree q = parse("select * from sources * where" + - " foo contains phrase(\"forest\", alternatives({\"trees\": 1.0, \"tree\": 0.7}));"); + " foo contains phrase(\"forest\", alternatives({trees: 1.0, tree: 0.7}))"); Item root = q.getRoot(); assertSame(PhraseItem.class, root.getClass()); PhraseItem phrase = (PhraseItem) root; @@ -1003,7 +999,7 @@ public class YqlParserTestCase { @Test public void testBackslash() { { - String queryString = "select * from testtype where title contains \"\\\\\";"; // Java escaping * YQL escaping + String queryString = "select * from testtype where title contains \"\\\\\""; // Java escaping * YQL escaping QueryTree query = parse(queryString); @@ -1011,7 +1007,7 @@ public class YqlParserTestCase { } { - Query query = new Query("search?yql=select%20*%20from%20testtype%20where%20title%20contains%20%22%5C%5C%22;"); + Query query = new Query("search?yql=select%20*%20from%20testtype%20where%20title%20contains%20%22%5C%5C%22"); // Cause parsing :-\ Chain<Searcher> searchChain = new Chain<>(new MinimalQueryInserter()); @@ -1030,7 +1026,7 @@ public class YqlParserTestCase { // YQL query Query yql = new Query(); - yql.properties().set("yql", "select * from sources * where urlfield.hostname contains uri(\"google.com\");"); + yql.properties().set("yql", "select * from sources * where urlfield.hostname contains uri(\"google.com\")"); assertUrlQuery("urlfield.hostname", yql, false, true, true); } @@ -1041,7 +1037,7 @@ public class YqlParserTestCase { // YQL query Query yql = new Query(); - yql.properties().set("yql", "select * from sources * where urlfield.hostname contains ([{\"endAnchor\": false }]uri(\"google.com\"));"); + yql.properties().set("yql", "select * from sources * where urlfield.hostname contains ([{endAnchor: false }]uri(\"google.com\"))"); assertUrlQuery("urlfield.hostname", yql, false, false, true); } @@ -1052,7 +1048,7 @@ public class YqlParserTestCase { // YQL query Query yql = new Query(); - yql.properties().set("yql", "select * from sources * where urlfield.hostname contains ([{\"startAnchor\": true }] uri(\"google.com\"));"); + yql.properties().set("yql", "select * from sources * where urlfield.hostname contains ([{startAnchor: true }] uri(\"google.com\"))"); assertUrlQuery("urlfield.hostname", yql, true, true, true); } @@ -1063,19 +1059,19 @@ public class YqlParserTestCase { // YQL query Query yql = new Query(); - yql.properties().set("yql", "select * from sources * where urlfield contains uri(\"google.com\");"); + yql.properties().set("yql", "select * from sources * where urlfield contains uri(\"google.com\")"); assertUrlQuery("urlfield", yql, false, false, false); } @Test public void testReservedWordInSource() { - parse("select * from sources like where text contains \"test\";"); + parse("select * from sources like where text contains \"test\""); // success: parsed without exception } @Test public void testAndSegmenting() { - parse("select * from sources * where (default contains ([{\"stem\": false}]\"m\") AND default contains ([{\"origin\": {\"original\": \"m\'s\", \"offset\": 0, \"length\": 3}, \"andSegmenting\": true}]phrase(\"m\", \"s\"))) timeout 472;"); + parse("select * from sources * where (default contains ([{stem: false}]\"m\") AND default contains ([{origin: {original: \"m\'s\", 'offset': 0, length: 3}, andSegmenting: true}]phrase(\"m\", \"s\"))) timeout 472"); } private void assertUrlQuery(String field, Query query, boolean startAnchor, boolean endAnchor, boolean endAnchorIsDefault) { @@ -1112,11 +1108,11 @@ public class YqlParserTestCase { if (hasAnnotations) expectedYql.append("([{"); if (startAnchor != startAnchorIsDefault) - expectedYql.append("\"startAnchor\": " + startAnchor); + expectedYql.append("startAnchor: " + startAnchor); if (endAnchor != endAnchorIsDefault) { if (startAnchor != startAnchorIsDefault) expectedYql.append(", "); - expectedYql.append("\"endAnchor\": " + endAnchor); + expectedYql.append("endAnchor: " + endAnchor); } if (hasAnnotations) expectedYql.append("}]"); diff --git a/container-search/src/test/java/com/yahoo/select/SelectTestCase.java b/container-search/src/test/java/com/yahoo/select/SelectTestCase.java index 2e7e2a3a711..0dcfb8392ef 100644 --- a/container-search/src/test/java/com/yahoo/select/SelectTestCase.java +++ b/container-search/src/test/java/com/yahoo/select/SelectTestCase.java @@ -638,7 +638,7 @@ public class SelectTestCase { @Test public void testOpenIntervals() { assertParse("{ \"range\" : { \"children\":[ \"title\", { \">=\" : 0.0, \"<=\" : 500.0 }] } }" + - "select * from sources * where range(title, 0.0, 500.0);", + "select * from sources * where range(title, 0.0, 500.0)", "title:[0.0;500.0]"); assertParse( "{ \"range\" : { \"children\":[ \"title\", { \">\" : 0.0, \"<\" : 500.0 }] } }", diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/stubs/LoggingDeploymentIssues.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/stubs/LoggingDeploymentIssues.java index f5341ac6ddc..bfe833740d1 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/stubs/LoggingDeploymentIssues.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/stubs/LoggingDeploymentIssues.java @@ -9,7 +9,6 @@ import com.yahoo.vespa.hosted.controller.api.integration.organization.Contact; import com.yahoo.vespa.hosted.controller.api.integration.organization.DeploymentIssues; import com.yahoo.vespa.hosted.controller.api.integration.organization.IssueId; import com.yahoo.vespa.hosted.controller.api.integration.organization.User; -import org.jetbrains.annotations.TestOnly; import java.time.Clock; import java.time.Duration; @@ -48,7 +47,6 @@ public class LoggingDeploymentIssues implements DeploymentIssues { this(Clock.systemUTC()); } - @TestOnly protected LoggingDeploymentIssues(Clock clock) { this.clock = clock; } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java index e7e7101aff7..39223d6c031 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java @@ -87,6 +87,7 @@ import java.util.Optional; import java.util.Set; import java.util.TreeMap; import java.util.function.Consumer; +import java.util.function.Predicate; import java.util.logging.Level; import java.util.logging.Logger; import java.util.stream.Collectors; @@ -436,10 +437,12 @@ public class ApplicationController { // Validate new deployment spec thoroughly before storing it. controller.jobController().deploymentStatus(application.get()); - // Clear notifications for instances that are no longer declared - for (var name : existingInstances) - if ( ! declaredInstances.contains(name)) - controller.notificationsDb().removeNotifications(NotificationSource.from(application.get().id().instance(name))); + for (Notification notification : controller.notificationsDb().listNotifications(NotificationSource.from(application.get().id()), true)) { + if ( ! notification.source().instance().map(declaredInstances::contains).orElse(false)) + controller.notificationsDb().removeNotifications(notification.source()); + if ( ! notification.source().zoneId().map(application.get().require(notification.source().instance().get()).deployments()::containsKey).orElse(false)) + controller.notificationsDb().removeNotifications(notification.source()); + } store(application); return application; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/EndpointId.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/EndpointId.java index 70b011feb88..de1429d88af 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/EndpointId.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/EndpointId.java @@ -1,8 +1,6 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.application; -import org.jetbrains.annotations.NotNull; - import java.util.Objects; /** @@ -54,7 +52,7 @@ public class EndpointId implements Comparable<EndpointId> { public static EndpointId of(String id) { return new EndpointId(id); } @Override - public int compareTo(@NotNull EndpointId o) { + public int compareTo(EndpointId o) { return id.compareTo(o.id); } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/auditlog/AuditLog.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/auditlog/AuditLog.java index ef7b72632d8..19fb1b7e1bb 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/auditlog/AuditLog.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/auditlog/AuditLog.java @@ -3,7 +3,6 @@ package com.yahoo.vespa.hosted.controller.auditlog; import com.google.common.base.CharMatcher; import com.google.common.collect.Ordering; -import org.jetbrains.annotations.NotNull; import java.time.Instant; import java.util.ArrayList; @@ -101,7 +100,7 @@ public class AuditLog { } @Override - public int compareTo(@NotNull Entry that) { + public int compareTo(Entry that) { return comparator.compare(this, that); } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java index e28273870d7..0d56bc286eb 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java @@ -737,6 +737,9 @@ public class InternalStepRunner implements StepRunner { /** Sends a mail with a notification of a failed run, if one should be sent. */ private void sendEmailNotification(Run run, DualLogger logger) { + if ( ! isNewFailure(run)) + return; + Application application = controller.applications().requireApplication(TenantAndApplicationId.from(run.id().application())); Notifications notifications = application.deploymentSpec().requireInstance(run.id().application().instance()).notifications(); boolean newCommit = application.require(run.id().application().instance()).change().application() @@ -760,6 +763,12 @@ public class InternalStepRunner implements StepRunner { } } + private boolean isNewFailure(Run run) { + return controller.jobController().lastCompleted(run.id().job()) + .map(previous -> ! previous.hasFailed() || ! previous.versions().targetsMatch(run.versions())) + .orElse(true); + } + private void updateConsoleNotification(Run run) { NotificationSource source = NotificationSource.from(run.id()); Consumer<String> updater = msg -> controller.notificationsDb().setNotification(source, Notification.Type.deployment, Notification.Level.error, msg); diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentIssueReporter.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentIssueReporter.java index bb05840d34f..3958b3cb282 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentIssueReporter.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentIssueReporter.java @@ -67,6 +67,7 @@ public class DeploymentIssueReporter extends ControllerMaintainer { */ private double maintainDeploymentIssues(List<Application> applications) { List<TenantAndApplicationId> failingApplications = controller().jobController().deploymentStatuses(ApplicationList.from(applications)) + .matching(status -> ! status.jobSteps().isEmpty()) .failingApplicationChangeSince(controller().clock().instant().minus(maxFailureAge)) .mapToList(status -> status.application().id()); diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/JobRunner.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/JobRunner.java index ba891cdddd8..85d750d267c 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/JobRunner.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/JobRunner.java @@ -10,7 +10,6 @@ import com.yahoo.vespa.hosted.controller.deployment.Run; import com.yahoo.vespa.hosted.controller.deployment.Step; import com.yahoo.vespa.hosted.controller.deployment.StepInfo; import com.yahoo.vespa.hosted.controller.deployment.StepRunner; -import org.jetbrains.annotations.TestOnly; import java.time.Duration; import java.util.concurrent.ExecutorService; @@ -39,7 +38,6 @@ public class JobRunner extends ControllerMaintainer { this(controller, duration, Executors.newFixedThreadPool(32, new DaemonThreadFactory("job-runner-")), new InternalStepRunner(controller)); } - @TestOnly public JobRunner(Controller controller, Duration duration, ExecutorService executors, StepRunner runner) { super(controller, duration); this.jobs = controller.jobController(); diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/MetricsReporter.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/MetricsReporter.java index a1c25c1fb53..021b27456a4 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/MetricsReporter.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/MetricsReporter.java @@ -195,7 +195,7 @@ public class MetricsReporter extends ControllerMaintainer { Optional<Instant> lastOpened = Optional.empty(); // When the upgrade window most recently opened Instant oneWeekAgo = upgradingAt.minus(Duration.ofDays(7)); Duration step = Duration.ofHours(1); - for (Instant instant = upgradingAt; !instanceSpec.canUpgradeAt(instant); instant = instant.minus(step).truncatedTo(ChronoUnit.HOURS)) { + for (Instant instant = upgradingAt.truncatedTo(ChronoUnit.HOURS); !instanceSpec.canUpgradeAt(instant); instant = instant.minus(step)) { if (!instant.isAfter(oneWeekAgo)) { // Wrapped around, the entire week is being blocked lastOpened = Optional.empty(); break; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/ControllerVersion.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/ControllerVersion.java index 1d1b4ce7ba0..5b293081dc2 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/ControllerVersion.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/ControllerVersion.java @@ -3,7 +3,6 @@ package com.yahoo.vespa.hosted.controller.versions; import com.yahoo.component.Version; import com.yahoo.component.Vtag; -import org.jetbrains.annotations.NotNull; import java.time.Instant; import java.util.Objects; @@ -50,7 +49,7 @@ public class ControllerVersion implements Comparable<ControllerVersion> { } @Override - public int compareTo(@NotNull ControllerVersion o) { + public int compareTo(ControllerVersion o) { return version.compareTo(o.version); } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/OsVersion.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/OsVersion.java index dadca43c5a0..c240a7ddf35 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/OsVersion.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/OsVersion.java @@ -3,7 +3,6 @@ package com.yahoo.vespa.hosted.controller.versions; import com.yahoo.component.Version; import com.yahoo.config.provision.CloudName; -import org.jetbrains.annotations.NotNull; import java.util.Comparator; import java.util.Objects; @@ -56,7 +55,7 @@ public class OsVersion implements Comparable<OsVersion> { } @Override - public int compareTo(@NotNull OsVersion that) { + public int compareTo(OsVersion that) { return comparator.compare(this, that); } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/OsVersionTarget.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/OsVersionTarget.java index badb3014ddf..a8fcb6c78fc 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/OsVersionTarget.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/OsVersionTarget.java @@ -1,8 +1,6 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.versions; -import org.jetbrains.annotations.NotNull; - import java.time.Duration; import java.time.Instant; import java.util.Objects; @@ -61,7 +59,7 @@ public class OsVersionTarget implements Comparable<OsVersionTarget> { } @Override - public int compareTo(@NotNull OsVersionTarget o) { + public int compareTo(OsVersionTarget o) { return osVersion.compareTo(o.osVersion); } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunnerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunnerTest.java index ae92fd46f26..061cc69fc26 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunnerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunnerTest.java @@ -368,9 +368,7 @@ public class InternalStepRunnerTest { @Test public void notificationIsSent() { - app.startSystemTestTests(); - tester.cloud().set(TesterCloud.Status.NOT_STARTED); - tester.runner().run(); + app.submit().failDeployment(JobType.systemTest); MockMailer mailer = tester.controllerTester().serviceRegistry().mailer(); assertEquals(1, mailer.inbox("a@b").size()); assertEquals("Vespa application tenant.application: System test failing due to system error", @@ -378,6 +376,16 @@ public class InternalStepRunnerTest { assertEquals(1, mailer.inbox("b@a").size()); assertEquals("Vespa application tenant.application: System test failing due to system error", mailer.inbox("b@a").get(0).subject()); + + // Re-run failing causes no additional email to be sent. + app.failDeployment(JobType.systemTest); + assertEquals(1, mailer.inbox("a@b").size()); + assertEquals(1, mailer.inbox("b@a").size()); + + // Failure with new package causes new email to be sent. + app.submit().failDeployment(JobType.systemTest); + assertEquals(2, mailer.inbox("a@b").size()); + assertEquals(2, mailer.inbox("b@a").size()); } @Test diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/MetricsReporterTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/MetricsReporterTest.java index 3c91fb66894..71a3ce262ad 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/MetricsReporterTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/MetricsReporterTest.java @@ -510,7 +510,7 @@ public class MetricsReporterTest { .blockChange(true, false, "mon-sun", "0-7", "CET") .build(); - Instant mondayNight = Instant.parse("2021-12-13T23:00:00.00Z"); + Instant mondayNight = Instant.parse("2021-12-13T23:30:00.00Z"); DeploymentTester tester = new DeploymentTester().at(mondayNight); MetricsReporter reporter = createReporter(tester.controller()); DeploymentContext context = tester.newDeploymentContext(); @@ -535,17 +535,23 @@ public class MetricsReporterTest { assertEquals("Upgrade is not overdue yet", Duration.ZERO, metric.get()); // Upgrade continues into block window - tester.clock().advance(Duration.ofHours(3)); // Tuesday at 02:00 (03:00 CET) - assertEquals("Upgrade is overdue measured relative to window 2", Duration.ofHours(2), metric.get()); + tester.clock().advance(Duration.ofHours(1)); // Tuesday at 00:30 (01:30 CET) + assertEquals("Upgrade is overdue measured relative to window 2", Duration.ofHours(0).plusMinutes(30), metric.get()); + + tester.clock().advance(Duration.ofHours(1)); // Tuesday at 01:30 (02:30 CET) + assertEquals("Upgrade is overdue measured relative to window 2", Duration.ofHours(1).plusMinutes(30), metric.get()); + + tester.clock().advance(Duration.ofHours(1)); // Tuesday at 02:30 (03:30 CET) + assertEquals("Upgrade is overdue measured relative to window 2", Duration.ofHours(2).plusMinutes(30), metric.get()); - tester.clock().advance(Duration.ofHours(6)); // Tuesday at 08:00 (09:00 CET) - assertEquals("Upgrade is overdue measured relative to window 1", Duration.ofHours(8), metric.get()); + tester.clock().advance(Duration.ofHours(6)); // Tuesday at 08:30 (09:30 CET) + assertEquals("Upgrade is overdue measured relative to window 1", Duration.ofHours(8).plusMinutes(30), metric.get()); - tester.clock().advance(Duration.ofHours(1)); // Tuesday at 09:00 (10:00 CET) + tester.clock().advance(Duration.ofHours(1)); // Tuesday at 09:30 (10:30 CET) assertEquals("Upgrade is no longer overdue", Duration.ZERO, metric.get()); - tester.clock().advance(Duration.ofDays(2)); // Thursday at 10:00 (11:00 CET) - assertEquals("Upgrade is overdue measure relative to window 3", Duration.ofHours(34), metric.get()); + tester.clock().advance(Duration.ofDays(2)); // Thursday at 10:30 (11:30 CET) + assertEquals("Upgrade is overdue measure relative to window 3", Duration.ofHours(34).plusMinutes(30), metric.get()); } @Test diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/TenantRoleMaintainerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/TenantRoleMaintainerTest.java index c9cd7f41d2b..81daf0cbcfe 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/TenantRoleMaintainerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/TenantRoleMaintainerTest.java @@ -9,14 +9,13 @@ import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType; import com.yahoo.vespa.hosted.controller.application.pkg.ApplicationPackage; import com.yahoo.vespa.hosted.controller.deployment.ApplicationPackageBuilder; import com.yahoo.vespa.hosted.controller.deployment.DeploymentTester; -import org.hamcrest.Matchers; import org.junit.Test; import java.time.Duration; import java.util.List; import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertThat; +import static org.junit.Assert.assertTrue; /** * @author mortent @@ -54,7 +53,7 @@ public class TenantRoleMaintainerTest { var roleService = tester.controller().serviceRegistry().roleService(); List<TenantName> tenantNames = ((MockRoleService) roleService).maintainedTenants(); - assertThat(tenantNames, Matchers.containsInAnyOrder(prodAppTenant2.application().id().tenant(), perfAppTenant1.application().id().tenant())); + assertTrue(tenantNames.containsAll(List.of(prodAppTenant2.application().id().tenant(), perfAppTenant1.application().id().tenant()))); } private long permanentDeployments(Instance instance) { diff --git a/document/src/main/java/com/yahoo/document/serialization/VespaDocumentDeserializer6.java b/document/src/main/java/com/yahoo/document/serialization/VespaDocumentDeserializer6.java index 9115a000e20..757356e3096 100644 --- a/document/src/main/java/com/yahoo/document/serialization/VespaDocumentDeserializer6.java +++ b/document/src/main/java/com/yahoo/document/serialization/VespaDocumentDeserializer6.java @@ -58,7 +58,6 @@ import com.yahoo.document.update.RemoveValueUpdate; import com.yahoo.document.update.ValueUpdate; import com.yahoo.document.WeightedSetDataType; import com.yahoo.io.GrowableByteBuffer; -import com.yahoo.tensor.serialization.TypedBinaryFormat; import com.yahoo.text.Utf8; import com.yahoo.text.Utf8Array; import com.yahoo.text.Utf8String; diff --git a/document/src/test/java/com/yahoo/document/DocumentIdTestCase.java b/document/src/test/java/com/yahoo/document/DocumentIdTestCase.java index bee5f78ae83..63a0f8d25ed 100644 --- a/document/src/test/java/com/yahoo/document/DocumentIdTestCase.java +++ b/document/src/test/java/com/yahoo/document/DocumentIdTestCase.java @@ -5,9 +5,7 @@ import com.yahoo.document.idstring.IdIdString; import com.yahoo.document.idstring.IdString; import com.yahoo.vespa.objects.BufferSerializer; import org.junit.Before; -import org.junit.Rule; import org.junit.Test; -import org.junit.rules.ExpectedException; import java.io.BufferedReader; @@ -17,8 +15,10 @@ import java.io.UnsupportedEncodingException; import java.util.regex.Pattern; import java.util.Arrays; +import static org.junit.Assert.assertArrayEquals; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotEquals; import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; @@ -26,9 +26,6 @@ public class DocumentIdTestCase { DocumentTypeManager manager = new DocumentTypeManager(); - @Rule - public ExpectedException expectedException = ExpectedException.none(); - @Before public void setUp() { DocumentType testDocType = new DocumentType("testdoc"); @@ -87,16 +84,22 @@ public class DocumentIdTestCase { @Test public void empty_user_location_value_throws_exception() { - expectedException.expect(IllegalArgumentException.class); - expectedException.expectMessage("ID location value for 'n=' key is empty"); - new DocumentId("id:namespace:type:n=:foo"); + try { + new DocumentId("id:namespace:type:n=:foo"); + fail(); + } catch (IllegalArgumentException e) { + assertEquals("ID location value for 'n=' key is empty", e.getMessage()); + } } @Test public void empty_group_location_value_throws_exception() { - expectedException.expect(IllegalArgumentException.class); - expectedException.expectMessage("ID location value for 'g=' key is empty"); - new DocumentId("id:namespace:type:g=:foo"); + try { + new DocumentId("id:namespace:type:g=:foo"); + fail(); + } catch (IllegalArgumentException e) { + assertEquals("ID location value for 'g=' key is empty", e.getMessage()); + } } //Compares globalId with C++ implementation located in @@ -198,14 +201,10 @@ public class DocumentIdTestCase { DocumentId docId0Copy = new DocumentId("id:blabla:type::0"); byte[] docId0CopyGid = docId0Copy.getGlobalId(); - - //GIDs should be the same - for (int i = 0; i < docId0Gid.length; i++) { - assertEquals(docId0Gid[i], docId0CopyGid[i]); - } + assertArrayEquals(docId0Gid, docId0CopyGid); //straight hashCode() of byte arrays won't be the same - assertFalse(docId0Gid.hashCode() == docId0CopyGid.hashCode()); + assertNotEquals(docId0Gid.hashCode(), docId0CopyGid.hashCode()); //Arrays.hashCode() works better... assertEquals(Arrays.hashCode(docId0Gid), Arrays.hashCode(docId0CopyGid)); @@ -231,7 +230,7 @@ public class DocumentIdTestCase { } @Test - public void testSerializedDocumentIdCanContainNonTextCharacter() throws UnsupportedEncodingException { + public void testSerializedDocumentIdCanContainNonTextCharacter() { String strId = new String(new byte[]{105, 100, 58, 97, 58, 98, 58, 58, 7, 99}); // "id:a:b::0x7c" DocumentId docId = DocumentId.createFromSerialized(strId); { @@ -247,7 +246,7 @@ public class DocumentIdTestCase { } @Test - public void testSerializedDocumentIdCannotContainZeroByte() throws UnsupportedEncodingException { + public void testSerializedDocumentIdCannotContainZeroByte() { String strId = new String(new byte[]{105, 100, 58, 97, 58, 98, 58, 58, 0, 99}); // "id:a:b::0x0c" try { DocumentId.createFromSerialized(strId); diff --git a/document/src/test/java/com/yahoo/document/datatypes/ReferenceFieldValueTestCase.java b/document/src/test/java/com/yahoo/document/datatypes/ReferenceFieldValueTestCase.java index a537f5f5108..9ff15339ab4 100644 --- a/document/src/test/java/com/yahoo/document/datatypes/ReferenceFieldValueTestCase.java +++ b/document/src/test/java/com/yahoo/document/datatypes/ReferenceFieldValueTestCase.java @@ -6,15 +6,14 @@ import com.yahoo.document.DocumentId; import com.yahoo.document.DocumentType; import com.yahoo.document.Field; import com.yahoo.document.ReferenceDataType; -import org.junit.Rule; import org.junit.Test; -import org.junit.rules.ExpectedException; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertNotEquals; import static org.junit.Assert.assertNull; import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; /** * @author vekterli @@ -22,9 +21,6 @@ import static org.junit.Assert.assertTrue; */ public class ReferenceFieldValueTestCase { - @Rule - public final ExpectedException exceptionRule = ExpectedException.none(); - private static DocumentType createDocumentType(String name) { DocumentType type = new DocumentType(name); type.addField(new Field("foo", DataType.STRING)); @@ -126,10 +122,13 @@ public class ReferenceFieldValueTestCase { public void assigning_reference_field_with_different_type_to_existing_reference_throws_exception() { ReferenceFieldValue existing = new ReferenceFieldValue(referenceTypeFoo()); ReferenceFieldValue newValue = new ReferenceFieldValue(referenceTypeBar()); - exceptionRule.expect(IllegalArgumentException.class); - exceptionRule.expectMessage("Can't assign reference of type Reference<bar> " + - "to reference of type Reference<foo>"); - existing.assign(newValue); + try { + existing.assign(newValue); + fail(); + } catch (IllegalArgumentException e) { + assertEquals("Can't assign reference of type Reference<bar> to reference of type Reference<foo>", + e.getMessage()); + } } @Test diff --git a/document/src/test/java/com/yahoo/document/json/JsonReaderTestCase.java b/document/src/test/java/com/yahoo/document/json/JsonReaderTestCase.java index a1c1669ffa1..66ff7a7d4cd 100644 --- a/document/src/test/java/com/yahoo/document/json/JsonReaderTestCase.java +++ b/document/src/test/java/com/yahoo/document/json/JsonReaderTestCase.java @@ -56,9 +56,7 @@ import com.yahoo.text.Utf8; import com.yahoo.yolean.Exceptions; import org.junit.After; import org.junit.Before; -import org.junit.Rule; import org.junit.Test; -import org.junit.rules.ExpectedException; import java.io.ByteArrayInputStream; import java.io.IOException; @@ -96,9 +94,6 @@ public class JsonReaderTestCase { private DocumentTypeManager types; private JsonFactory parserFactory; - @Rule - public ExpectedException exception = ExpectedException.none(); - @Before public void setUp() throws Exception { parserFactory = new JsonFactory(); @@ -202,7 +197,6 @@ public class JsonReaderTestCase { public void tearDown() throws Exception { types = null; parserFactory = null; - exception = ExpectedException.none(); } private JsonReader createReader(String jsonInput) { @@ -963,9 +957,12 @@ public class JsonReaderTestCase { DocumentParseInfo parseInfo = r.parseDocument().get(); DocumentType docType = r.readDocumentType(parseInfo.documentId); DocumentPut put = new DocumentPut(new Document(docType, parseInfo.documentId)); - exception.expect(IllegalArgumentException.class); - exception.expectMessage("No field 'smething' in the structure of type 'smoke'"); - new VespaJsonDocumentReader().readPut(parseInfo.fieldsBuffer, put); + try { + new VespaJsonDocumentReader().readPut(parseInfo.fieldsBuffer, put); + fail(); + } catch (IllegalArgumentException e) { + assertTrue(e.getMessage().startsWith("No field 'smething' in the structure of type 'smoke'")); + } } @Test @@ -975,9 +972,12 @@ public class JsonReaderTestCase { " { 'put': 'id:test:smoke::1', 'fields': { 'something': 'foo' } },", " { 'put': 'id:test:smoke::2', 'fields': { 'something': 'foo' } },", "]")); - exception.expect(RuntimeException.class); - exception.expectMessage("JsonParseException"); - while (r.next() != null); + try { + while (r.next() != null) ; + fail(); + } catch (RuntimeException e) { + assertTrue(e.getMessage().contains("JsonParseException")); + } } @Test @@ -1869,9 +1869,6 @@ public class JsonReaderTestCase { @Test public void requireThatUnknownDocTypeThrowsIllegalArgumentException() { - exception.expect(IllegalArgumentException.class); - exception.expectMessage("Document type walrus does not exist"); - final String jsonData = inputJson( "[", " {", @@ -1881,8 +1878,12 @@ public class JsonReaderTestCase { " }", " }", "]"); - - new JsonReader(types, jsonToInputStream(jsonData), parserFactory).next(); + try { + new JsonReader(types, jsonToInputStream(jsonData), parserFactory).next(); + fail(); + } catch (IllegalArgumentException e) { + assertEquals("Document type walrus does not exist", e.getMessage()); + } } private static final String TENSOR_DOC_ID = "id:unittest:testtensor::0"; @@ -2051,10 +2052,13 @@ public class JsonReaderTestCase { // NOTE: Do not call this method multiple times from a test method as it's using the ExpectedException rule private void assertParserErrorMatches(String expectedError, String... json) { - exception.expect(JsonReaderException.class); - exception.expectMessage(expectedError); String jsonData = inputJson(json); - new JsonReader(types, jsonToInputStream(jsonData), parserFactory).next(); + try { + new JsonReader(types, jsonToInputStream(jsonData), parserFactory).next(); + fail(); + } catch (JsonReaderException e) { + assertEquals(expectedError, e.getMessage()); + } } private void assertCreatePutFails(String tensor, String name, String msg) { diff --git a/document/src/test/java/com/yahoo/document/select/DocumentSelectorTestCase.java b/document/src/test/java/com/yahoo/document/select/DocumentSelectorTestCase.java index 5a75110ca74..0a23c14cf16 100644 --- a/document/src/test/java/com/yahoo/document/select/DocumentSelectorTestCase.java +++ b/document/src/test/java/com/yahoo/document/select/DocumentSelectorTestCase.java @@ -28,9 +28,7 @@ import com.yahoo.document.select.parser.ParseException; import com.yahoo.document.select.parser.TokenMgrException; import com.yahoo.yolean.Exceptions; import org.junit.Before; -import org.junit.Rule; import org.junit.Test; -import org.junit.rules.ExpectedException; import java.util.ArrayList; import java.util.Arrays; @@ -49,9 +47,6 @@ import static org.junit.Assert.fail; */ public class DocumentSelectorTestCase { - @Rule - public final ExpectedException exceptionRule = ExpectedException.none(); - private static final DocumentTypeManager manager = new DocumentTypeManager(); @Before @@ -786,13 +781,15 @@ public class DocumentSelectorTestCase { @Test public void imported_fields_only_supported_for_simple_expressions() throws ParseException { - exceptionRule.expect(IllegalArgumentException.class); // TODO we should probably handle this case specially and give a better exception message - exceptionRule.expectMessage("Field 'my_imported_field' not found in type datatype test"); - var documents = createDocs(); - // Nested field access is NOT considered a simple expression. - evaluate("test.my_imported_field.foo", documents.get(0)); + try { + // Nested field access is NOT considered a simple expression. + evaluate("test.my_imported_field.foo", documents.get(0)); + fail(); + } catch (IllegalArgumentException e) { + assertTrue(e.getMessage().startsWith("Field 'my_imported_field' not found in type datatype test")); + } } @Test diff --git a/document/src/test/java/com/yahoo/document/update/TensorModifyUpdateTest.java b/document/src/test/java/com/yahoo/document/update/TensorModifyUpdateTest.java index 5b7e9ad89d5..60dd5ad1d0d 100644 --- a/document/src/test/java/com/yahoo/document/update/TensorModifyUpdateTest.java +++ b/document/src/test/java/com/yahoo/document/update/TensorModifyUpdateTest.java @@ -5,17 +5,13 @@ import com.yahoo.document.datatypes.TensorFieldValue; import com.yahoo.document.update.TensorModifyUpdate.Operation; import com.yahoo.tensor.Tensor; import com.yahoo.tensor.TensorType; -import org.junit.Rule; import org.junit.Test; -import org.junit.rules.ExpectedException; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.fail; public class TensorModifyUpdateTest { - @Rule - public ExpectedException exception = ExpectedException.none(); - @Test public void convert_to_compatible_type_with_only_mapped_dimensions() { assertConvertToCompatible("tensor(x{})", "tensor(x[])"); @@ -31,10 +27,14 @@ public class TensorModifyUpdateTest { @Test public void use_of_incompatible_tensor_type_throws() { - exception.expect(IllegalArgumentException.class); - exception.expectMessage("Tensor type 'tensor(x[3])' is not compatible as it has no mapped dimensions"); - new TensorModifyUpdate(TensorModifyUpdate.Operation.REPLACE, - new TensorFieldValue(Tensor.from("tensor(x[3])", "{{x:1}:3}"))); + try { + new TensorModifyUpdate(TensorModifyUpdate.Operation.REPLACE, + new TensorFieldValue(Tensor.from("tensor(x[3])", "{{x:1}:3}"))); + fail(); + } catch (IllegalArgumentException e) { + assertEquals("Tensor type 'tensor(x[3])' is not compatible as it has no mapped dimensions", + e.getMessage()); + } } @Test diff --git a/documentapi/abi-spec.json b/documentapi/abi-spec.json index 88ec090d324..2e68d4803cb 100644 --- a/documentapi/abi-spec.json +++ b/documentapi/abi-spec.json @@ -1565,6 +1565,20 @@ "protected final java.util.Random randomizer" ] }, + "com.yahoo.documentapi.messagebus.protocol.ContentPolicy$InstabilityChecker": { + "superClass": "java.lang.Object", + "interfaces": [], + "attributes": [ + "public", + "interface", + "abstract" + ], + "methods": [ + "public abstract boolean tooManyFailures(int)", + "public abstract void addFailure(java.lang.Integer)" + ], + "fields": [] + }, "com.yahoo.documentapi.messagebus.protocol.ContentPolicy$Parameters": { "superClass": "java.lang.Object", "interfaces": [], @@ -1576,7 +1590,8 @@ "public java.lang.String getClusterName()", "public com.yahoo.documentapi.messagebus.protocol.ContentPolicy$SlobrokHostPatternGenerator createPatternGenerator()", "public com.yahoo.documentapi.messagebus.protocol.ContentPolicy$HostFetcher createHostFetcher(com.yahoo.documentapi.messagebus.protocol.SlobrokPolicy, int)", - "public com.yahoo.vdslib.distribution.Distribution createDistribution(com.yahoo.documentapi.messagebus.protocol.SlobrokPolicy)" + "public com.yahoo.vdslib.distribution.Distribution createDistribution(com.yahoo.documentapi.messagebus.protocol.SlobrokPolicy)", + "public com.yahoo.documentapi.messagebus.protocol.ContentPolicy$InstabilityChecker createInstabilityChecker()" ], "fields": [ "protected final java.lang.String clusterName", @@ -1585,6 +1600,21 @@ "protected final com.yahoo.documentapi.messagebus.protocol.ContentPolicy$SlobrokHostPatternGenerator slobrokHostPatternGenerator" ] }, + "com.yahoo.documentapi.messagebus.protocol.ContentPolicy$PerNodeCountingInstabilityChecker": { + "superClass": "java.lang.Object", + "interfaces": [ + "com.yahoo.documentapi.messagebus.protocol.ContentPolicy$InstabilityChecker" + ], + "attributes": [ + "public" + ], + "methods": [ + "public void <init>(int)", + "public boolean tooManyFailures(int)", + "public void addFailure(java.lang.Integer)" + ], + "fields": [] + }, "com.yahoo.documentapi.messagebus.protocol.ContentPolicy$SlobrokHostFetcher": { "superClass": "com.yahoo.documentapi.messagebus.protocol.ContentPolicy$HostFetcher", "interfaces": [], diff --git a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/ContentPolicy.java b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/ContentPolicy.java index 5eaec70ca59..21e621883fe 100644 --- a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/ContentPolicy.java +++ b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/ContentPolicy.java @@ -232,6 +232,47 @@ public class ContentPolicy extends SlobrokPolicy { } + /** + * Tracks "instability" across nodes based on number of failures received versus some + * implementation-specific limit. + * + * Implementations must be thread-safe. + * + * TODO should ideally be protected, but there's a package mismatch between policy classes and its tests + */ + public interface InstabilityChecker { + boolean tooManyFailures(int nodeIndex); + void addFailure(Integer calculatedDistributor); + } + + /** Class that tracks a failure of a given type per node. */ + public static class PerNodeCountingInstabilityChecker implements InstabilityChecker { + private final List<Integer> nodeFailures = new CopyOnWriteArrayList<>(); + private final int failureLimit; + + public PerNodeCountingInstabilityChecker(int failureLimit) { + this.failureLimit = failureLimit; + } + + @Override + public boolean tooManyFailures(int nodeIndex) { + if (nodeFailures.size() > nodeIndex && nodeFailures.get(nodeIndex) > failureLimit) { + nodeFailures.set(nodeIndex, 0); + return true; + } else { + return false; + } + } + + @Override + public void addFailure(Integer calculatedDistributor) { + while (nodeFailures.size() <= calculatedDistributor) { + nodeFailures.add(0); + } + nodeFailures.set(calculatedDistributor, nodeFailures.get(calculatedDistributor) + 1); + } + } + /** Class parsing the semicolon separated parameter string and exposes the appropriate value to the policy. */ public static class Parameters { @@ -271,6 +312,9 @@ public class ContentPolicy extends SlobrokPolicy { return distributionConfig == null ? new Distribution(getDistributionConfigId()) : new Distribution(distributionConfig.cluster(clusterName)); } + public InstabilityChecker createInstabilityChecker() { + return new PerNodeCountingInstabilityChecker(getAttemptRandomOnFailuresLimit()); + } /** * When we have gotten this amount of failures from a node (Any kind of failures). We try to send to a random other node, just to see if the @@ -324,27 +368,6 @@ public class ContentPolicy extends SlobrokPolicy { /** Class handling the logic of picking a distributor */ public static class DistributorSelectionLogic { - /** Class that tracks a failure of a given type per node. */ - static class InstabilityChecker { - private final List<Integer> nodeFailures = new CopyOnWriteArrayList<>(); - private final int failureLimit; - - InstabilityChecker(int failureLimit) { this.failureLimit = failureLimit; } - - boolean tooManyFailures(int nodeIndex) { - if (nodeFailures.size() > nodeIndex && nodeFailures.get(nodeIndex) > failureLimit) { - nodeFailures.set(nodeIndex, 0); - return true; - } else { - return false; - } - } - - void addFailure(Integer calculatedDistributor) { - while (nodeFailures.size() <= calculatedDistributor) nodeFailures.add(0); - nodeFailures.set(calculatedDistributor, nodeFailures.get(calculatedDistributor) + 1); - } - } /** Message context class. Contains data we want to inspect about a request at reply time. */ private static class MessageContext { final Integer calculatedDistributor; @@ -375,7 +398,7 @@ public class ContentPolicy extends SlobrokPolicy { try { hostFetcher = params.createHostFetcher(policy, params.getRequiredUpPercentageToSendToKnownGoodNodes()); distribution = params.createDistribution(policy); - persistentFailureChecker = new InstabilityChecker(params.getAttemptRandomOnFailuresLimit()); + persistentFailureChecker = params.createInstabilityChecker(); maxOldClusterVersionBeforeSendingRandom = params.maxOldClusterStatesSeenBeforeThrowingCachedState(); } catch (Throwable e) { destroy(); @@ -556,10 +579,37 @@ public class ContentPolicy extends SlobrokPolicy { } } + /** + * Returns whether a given error Reply should be counted towards potentially ignoring the cached + * cluster state and triggering a random send (and thus likely WrongDistributionReply with the + * current cluster state). Certain error codes may be used frequently by the content layer for + * purposes that do _not_ indicate that a change in cluster state may have happened, and should + * therefore not be counted for this purpose: + * - ERROR_TEST_AND_SET_CONDITION_FAILED: may happen for any mutating operation that has an + * associated TaS condition. Technically an APP_FATAL_ERROR since resending doesn't make sense. + * - ERROR_BUSY: may happen for concurrent mutations and if distributors are in the process of + * changing bucket ownership and the grace period hasn't passed yet. + */ + private static boolean shouldCountAsErrorForRandomSendTrigger(Reply reply) { + if (reply.getNumErrors() != 1) { + return !reply.hasErrors(); // For simplicity, count any reply with > 1 error. + } + var error = reply.getError(0); + switch (error.getCode()) { + // TODO this feels like a layering violation, but we use DocumentProtocol directly in other places in this policy anyway... + case DocumentProtocol.ERROR_TEST_AND_SET_CONDITION_FAILED: + case DocumentProtocol.ERROR_BUSY: + return false; + default: return true; + } + } + void handleErrorReply(Reply reply, Object untypedContext) { MessageContext messageContext = (MessageContext) untypedContext; if (messageContext.calculatedDistributor != null) { - persistentFailureChecker.addFailure(messageContext.calculatedDistributor); + if (shouldCountAsErrorForRandomSendTrigger(reply)) { + persistentFailureChecker.addFailure(messageContext.calculatedDistributor); + } if (reply.getTrace().shouldTrace(1)) { reply.getTrace().trace(1, "Failed with " + messageContext.toString()); } diff --git a/documentapi/src/test/java/com/yahoo/documentapi/messagebus/protocol/RoutingPolicyRepositoryTest.java b/documentapi/src/test/java/com/yahoo/documentapi/messagebus/protocol/RoutingPolicyRepositoryTest.java index 2bd7c965fdc..43687254498 100644 --- a/documentapi/src/test/java/com/yahoo/documentapi/messagebus/protocol/RoutingPolicyRepositoryTest.java +++ b/documentapi/src/test/java/com/yahoo/documentapi/messagebus/protocol/RoutingPolicyRepositoryTest.java @@ -1,19 +1,16 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.documentapi.messagebus.protocol; -import org.junit.Rule; import org.junit.Test; -import org.junit.rules.ExpectedException; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.fail; import static org.mockito.ArgumentMatchers.anyString; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; public class RoutingPolicyRepositoryTest { - @Rule - public ExpectedException expectedException = ExpectedException.none(); - @Test public void policy_creation_does_not_swallow_exception() { final RoutingPolicyRepository repo = new RoutingPolicyRepository(); @@ -22,10 +19,12 @@ public class RoutingPolicyRepositoryTest { when(factory.createPolicy(anyString())).thenThrow(new IllegalArgumentException("oh no!")); repo.putFactory("foo", factory); - expectedException.expectMessage("oh no!"); - expectedException.expect(IllegalArgumentException.class); - - repo.createPolicy("foo", "bar"); + try { + repo.createPolicy("foo", "bar"); + fail(); + } catch (IllegalArgumentException e) { + assertEquals("oh no!", e.getMessage()); + } } } diff --git a/documentapi/src/test/java/com/yahoo/documentapi/messagebus/protocol/test/PolicyFactoryTestCase.java b/documentapi/src/test/java/com/yahoo/documentapi/messagebus/protocol/test/PolicyFactoryTestCase.java index 9ee58f746a5..473b9c9d5dd 100755 --- a/documentapi/src/test/java/com/yahoo/documentapi/messagebus/protocol/test/PolicyFactoryTestCase.java +++ b/documentapi/src/test/java/com/yahoo/documentapi/messagebus/protocol/test/PolicyFactoryTestCase.java @@ -42,7 +42,7 @@ public class PolicyFactoryTestCase { public void setUp() throws ListenFailedException { slobrok = new Slobrok(); srv = new TestServer(new MessageBusParams().addProtocol(new DocumentProtocol(new DocumentTypeManager())), - new RPCNetworkParams().setSlobrokConfigId(TestServer.getSlobrokConfig(slobrok))); + new RPCNetworkParams().setNumNetworkThreads(1).setSlobrokConfigId(TestServer.getSlobrokConfig(slobrok))); src = srv.mb.createSourceSession(new SourceSessionParams().setReplyHandler(new Receptor())); } diff --git a/documentapi/src/test/java/com/yahoo/documentapi/messagebus/protocol/test/PolicyTestCase.java b/documentapi/src/test/java/com/yahoo/documentapi/messagebus/protocol/test/PolicyTestCase.java index 83f2fbcff5e..07456870977 100755 --- a/documentapi/src/test/java/com/yahoo/documentapi/messagebus/protocol/test/PolicyTestCase.java +++ b/documentapi/src/test/java/com/yahoo/documentapi/messagebus/protocol/test/PolicyTestCase.java @@ -140,13 +140,17 @@ public class PolicyTestCase { assertTrue(new DocumentProtocol(manager).createPolicy("Extern", spec) instanceof ErrorPolicy); assertTrue(new DocumentProtocol(manager).createPolicy("Extern", spec + ";") instanceof ErrorPolicy); assertTrue(new DocumentProtocol(manager).createPolicy("Extern", spec + ";bar") instanceof ErrorPolicy); + slobrok.stop(); } @Test public void requireThatExternPolicyWithUnknownPatternSelectsNone() throws Exception { + var slobrok = new Slobrok(); PolicyTestFrame frame = newPutDocumentFrame("id:ns:testdoc::"); - setupExternPolicy(frame, new Slobrok(), "foo/bar"); + setupExternPolicy(frame, slobrok, "foo/bar"); frame.assertSelect(null); + slobrok.stop(); + frame.destroy(); } @Test @@ -175,6 +179,7 @@ public class PolicyTestCase { server.destroy(); } frame.destroy(); + slobrok.stop(); } @Test @@ -188,6 +193,7 @@ public class PolicyTestCase { frame.assertMergeOneReply(server.net.getConnectionSpec() + "/chain.default"); server.destroy(); frame.destroy(); + slobrok.stop(); } @Test @@ -362,6 +368,9 @@ public class PolicyTestCase { assertNotNull(barFrame.getReceptor().getReply(TIMEOUT)); assertNotNull(fooFrame.getReceptor().getReply(TIMEOUT)); + + fooFrame.destroy(); + barFrame.destroy(); } @Test @@ -386,6 +395,7 @@ public class PolicyTestCase { assertNotNull(reply); assertEquals(DocumentProtocol.REPLY_GETDOCUMENT, reply.getType()); assertEquals(123456L, ((GetDocumentReply)reply).getLastModified()); + frame.destroy(); } private String getDocumentRouteSelectorRawConfig() { @@ -408,6 +418,7 @@ public class PolicyTestCase { frame.setMessage(createRemove("id:ns:other::1")); frame.assertSelect(Arrays.asList("other-route")); + frame.destroy(); } @Test @@ -419,6 +430,7 @@ public class PolicyTestCase { frame.setMessage(createGet("id:ns:other::1")); frame.assertSelect(Arrays.asList("other-route")); + frame.destroy(); } private PolicyTestFrame createFrameWithTwoRoutes() { @@ -538,6 +550,8 @@ public class PolicyTestCase { assertNotNull(barFrame.getReceptor().getReply(TIMEOUT)); assertNotNull(fooFrame.getReceptor().getReply(TIMEOUT)); + fooFrame.destroy(); + barFrame.destroy(); } @Test @@ -654,6 +668,7 @@ public class PolicyTestCase { frame.setHop(new HopSpec("test", "[LoadBalancer:cluster=docproc/cluster.default;session=chain.default]")); assertSelect(frame, 1, Arrays.asList(frame.getNetwork().getConnectionSpec() + "/chain.default")); + frame.destroy(); } @Test @@ -722,6 +737,8 @@ public class PolicyTestCase { assertNotNull(barFrame.getReceptor().getReply(TIMEOUT)); assertNotNull(fooFrame.getReceptor().getReply(TIMEOUT)); + fooFrame.destroy(); + barFrame.destroy(); } /** @@ -773,6 +790,9 @@ public class PolicyTestCase { return; } Thread.sleep(10); + if (i == 42) { + ((Mirror) slobrok).dumpState(); + } } throw new TimeoutException(); } diff --git a/documentapi/src/test/java/com/yahoo/documentapi/messagebus/protocol/test/PolicyTestFrame.java b/documentapi/src/test/java/com/yahoo/documentapi/messagebus/protocol/test/PolicyTestFrame.java index b2fce64f784..a067261ba1a 100755 --- a/documentapi/src/test/java/com/yahoo/documentapi/messagebus/protocol/test/PolicyTestFrame.java +++ b/documentapi/src/test/java/com/yahoo/documentapi/messagebus/protocol/test/PolicyTestFrame.java @@ -80,7 +80,7 @@ public class PolicyTestFrame { e.printStackTrace(); fail(e.getMessage()); } - net = new MyNetwork(new RPCNetworkParams() + net = new MyNetwork(new RPCNetworkParams().setNumNetworkThreads(1) .setIdentity(new Identity(identity)) .setSlobrokConfigId(TestServer.getSlobrokConfig(slobrok))); mbus = new MessageBus(net, new MessageBusParams() diff --git a/documentapi/src/test/java/com/yahoo/documentapi/messagebus/protocol/test/RoutableFactoryTestCase.java b/documentapi/src/test/java/com/yahoo/documentapi/messagebus/protocol/test/RoutableFactoryTestCase.java index cde6a7484ea..51728b67347 100755 --- a/documentapi/src/test/java/com/yahoo/documentapi/messagebus/protocol/test/RoutableFactoryTestCase.java +++ b/documentapi/src/test/java/com/yahoo/documentapi/messagebus/protocol/test/RoutableFactoryTestCase.java @@ -51,11 +51,11 @@ public class RoutableFactoryTestCase { DocumentTypeManager docMan = new DocumentTypeManager(); dstProtocol = new DocumentProtocol(docMan); dstServer = new TestServer(new MessageBusParams().addProtocol(dstProtocol), - new RPCNetworkParams().setIdentity(new Identity("dst")).setSlobrokConfigId(TestServer.getSlobrokConfig(slobrok))); + new RPCNetworkParams().setNumNetworkThreads(1).setIdentity(new Identity("dst")).setSlobrokConfigId(TestServer.getSlobrokConfig(slobrok))); dstSession = dstServer.mb.createDestinationSession(new DestinationSessionParams().setName("session").setMessageHandler(new Receptor())); srcProtocol = new DocumentProtocol(docMan); srcServer = new TestServer(new MessageBusParams().addProtocol(srcProtocol), - new RPCNetworkParams().setSlobrokConfigId(TestServer.getSlobrokConfig(slobrok))); + new RPCNetworkParams().setNumNetworkThreads(1).setSlobrokConfigId(TestServer.getSlobrokConfig(slobrok))); srcSession = srcServer.mb.createSourceSession(new SourceSessionParams().setReplyHandler(new Receptor())); assertTrue(srcServer.waitSlobrok("dst/session", 1)); } diff --git a/documentapi/src/test/java/com/yahoo/documentapi/messagebus/protocol/test/storagepolicy/ContentPolicyTest.java b/documentapi/src/test/java/com/yahoo/documentapi/messagebus/protocol/test/storagepolicy/ContentPolicyTest.java index 42625cf193d..c160f67aa9b 100644 --- a/documentapi/src/test/java/com/yahoo/documentapi/messagebus/protocol/test/storagepolicy/ContentPolicyTest.java +++ b/documentapi/src/test/java/com/yahoo/documentapi/messagebus/protocol/test/storagepolicy/ContentPolicyTest.java @@ -1,9 +1,13 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.documentapi.messagebus.protocol.test.storagepolicy; +import com.yahoo.documentapi.messagebus.protocol.DocumentProtocol; +import com.yahoo.messagebus.routing.RoutingNode; import org.junit.Ignore; import org.junit.Test; +import static org.junit.Assert.assertEquals; + public class ContentPolicyTest extends Simulator { /** @@ -129,6 +133,46 @@ public class ContentPolicyTest extends Simulator { // Note that we use extra requests here as with only 200 requests there was a pretty good chance of not going to any down node on random anyhow. } + private void setUpSingleNodeFixturesWithInitializedPolicy() { + setClusterNodes(new int[]{ 0 }); + // Seed policy with initial, correct cluster state. + { + RoutingNode target = select(); + replyWrongDistribution(target, "foo", null, "version:1234 distributor:1 storage:1"); + } + } + + @Test + public void transient_errors_expected_during_normal_feed_are_not_counted_as_errors_that_may_trigger_random_send() { + setUpSingleNodeFixturesWithInitializedPolicy(); + var checker = policyFactory.getLastParameters().instabilityChecker; + assertEquals(0, checker.recordedFailures); // WrongDistributionReply not counted as regular error + { + frame.setMessage(createMessage("id:ns:testdoc:n=2:foo")); + RoutingNode target = select(); + replyError(target, new com.yahoo.messagebus.Error(DocumentProtocol.ERROR_BUSY, "Get busy livin' or get busy resendin'")); + } + assertEquals(0, checker.recordedFailures); // BUSY not counted as error + { + frame.setMessage(createMessage("id:ns:testdoc:n=3:foo")); + RoutingNode target = select(); + replyError(target, new com.yahoo.messagebus.Error(DocumentProtocol.ERROR_TEST_AND_SET_CONDITION_FAILED, "oh no")); + } + assertEquals(0, checker.recordedFailures); // TaS failures not counted as error + } + + @Test + public void other_errors_during_feed_are_counted_as_errors_that_may_trigger_random_send() { + setUpSingleNodeFixturesWithInitializedPolicy(); + var checker = policyFactory.getLastParameters().instabilityChecker; + { + frame.setMessage(createMessage("id:ns:testdoc:n=1:foo")); + RoutingNode target = select(); + replyError(target, new com.yahoo.messagebus.Error(DocumentProtocol.ERROR_ABORTED, "shop's closing, go home")); + } + assertEquals(1, checker.recordedFailures); + } + // Left to test? // Cluster state down - Not overwrite last good nodes to send random to? diff --git a/documentapi/src/test/java/com/yahoo/documentapi/messagebus/protocol/test/storagepolicy/ContentPolicyTestEnvironment.java b/documentapi/src/test/java/com/yahoo/documentapi/messagebus/protocol/test/storagepolicy/ContentPolicyTestEnvironment.java index c75e1c7832e..b6893d69325 100644 --- a/documentapi/src/test/java/com/yahoo/documentapi/messagebus/protocol/test/storagepolicy/ContentPolicyTestEnvironment.java +++ b/documentapi/src/test/java/com/yahoo/documentapi/messagebus/protocol/test/storagepolicy/ContentPolicyTestEnvironment.java @@ -44,7 +44,7 @@ public abstract class ContentPolicyTestEnvironment { protected PolicyTestFrame frame; private Set<Integer> nodes; protected static int[] bucketOneNodePreference = new int[]{ 3, 5, 7, 6, 8, 0, 9, 2, 1, 4 }; - protected boolean debug = true; + protected boolean debug = false; @Before public void setUp() throws Exception { @@ -145,14 +145,37 @@ public abstract class ContentPolicyTestEnvironment { } } + public static class TestWrappingInstabilityChecker implements ContentPolicy.InstabilityChecker { + + public int recordedFailures = 0; + private final ContentPolicy.InstabilityChecker fwdChecker; + + TestWrappingInstabilityChecker(ContentPolicy.InstabilityChecker fwdChecker) { + this.fwdChecker = fwdChecker; + } + + @Override + public boolean tooManyFailures(int nodeIndex) { + return fwdChecker.tooManyFailures(nodeIndex); + } + + @Override + public void addFailure(Integer calculatedDistributor) { + ++recordedFailures; + fwdChecker.addFailure(calculatedDistributor); + } + } + public static class TestParameters extends ContentPolicy.Parameters { private final TestHostFetcher hostFetcher; private final Distribution distribution; + public final TestWrappingInstabilityChecker instabilityChecker; public TestParameters(String parameters, Set<Integer> nodes) { super(SlobrokPolicy.parse(parameters)); hostFetcher = new TestHostFetcher(getClusterName(), nodes); distribution = new Distribution(Distribution.getDefaultDistributionConfig(2, 10)); + instabilityChecker = new TestWrappingInstabilityChecker(new ContentPolicy.PerNodeCountingInstabilityChecker(5)); } @Override @@ -160,6 +183,9 @@ public abstract class ContentPolicyTestEnvironment { @Override public Distribution createDistribution(SlobrokPolicy policy) { return distribution; } + + @Override + public ContentPolicy.InstabilityChecker createInstabilityChecker() { return instabilityChecker; } } public static class ContentPolicyTestFactory implements RoutingPolicyFactory { @@ -182,8 +208,6 @@ public abstract class ContentPolicyTestEnvironment { } } public TestParameters getLastParameters() { return parameterInstances.getLast(); } - public void destroy() { - } } private int findPreferredAvailableNodeForTestBucket() { diff --git a/documentapi/src/test/java/com/yahoo/documentapi/messagebus/protocol/test/storagepolicy/Simulator.java b/documentapi/src/test/java/com/yahoo/documentapi/messagebus/protocol/test/storagepolicy/Simulator.java index 651a1c15c0b..d40c87e0bd6 100644 --- a/documentapi/src/test/java/com/yahoo/documentapi/messagebus/protocol/test/storagepolicy/Simulator.java +++ b/documentapi/src/test/java/com/yahoo/documentapi/messagebus/protocol/test/storagepolicy/Simulator.java @@ -188,7 +188,7 @@ public abstract class Simulator extends ContentPolicyTestEnvironment { if (badNode != null && randomizer.nextDouble() < badNode.getFailureRate()) { ++failed[half]; switch (badNode.getFailureType()) { - case TRANSIENT_ERROR: replyError(target, new com.yahoo.messagebus.Error(DocumentProtocol.ERROR_BUSY, "Transient error")); break; + case TRANSIENT_ERROR: replyError(target, new com.yahoo.messagebus.Error(DocumentProtocol.ERROR_ABORTED, "Transient error")); break; case FATAL_ERROR: replyError(target, new com.yahoo.messagebus.Error(DocumentProtocol.ERROR_UNPARSEABLE, "Fatal error")); break; case OLD_CLUSTER_STATE: case RESET_CLUSTER_STATE: diff --git a/documentapi/src/test/java/com/yahoo/documentapi/messagebus/test/Destination.java b/documentapi/src/test/java/com/yahoo/documentapi/messagebus/test/Destination.java index 0670358f097..06f82168447 100644 --- a/documentapi/src/test/java/com/yahoo/documentapi/messagebus/test/Destination.java +++ b/documentapi/src/test/java/com/yahoo/documentapi/messagebus/test/Destination.java @@ -47,7 +47,7 @@ public class Destination implements MessageHandler { access = new LocalDocumentAccess(params); local = access.createSyncSession(new SyncParameters.Builder().build()); bus = new RPCMessageBus(Arrays.asList((Protocol)new DocumentProtocol(access.getDocumentTypeManager())), - new RPCNetworkParams() + new RPCNetworkParams().setNumNetworkThreads(1) .setIdentity(new Identity("test/destination")) .setSlobrokConfigId(slobrokConfigId), "file:src/test/cfg/messagebus.cfg"); diff --git a/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/FileReferenceDownloader.java b/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/FileReferenceDownloader.java index 21a9833c3e0..03448b5fa4a 100644 --- a/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/FileReferenceDownloader.java +++ b/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/FileReferenceDownloader.java @@ -67,9 +67,9 @@ public class FileReferenceDownloader { return; retryCount++; - // There is no one connection that will always work for each file reference (each file reference might - // exist on just one config server, and which one could be different for each file reference), so we - // should get a new connection for every retry + // There might not be one connection that works for all file references (each file reference might + // exist on just one config server, and which one could be different for each file reference), so + // switch to a new connection for every retry connection = connectionPool.switchConnection(connection); } while (retryCount < 5 || Instant.now().isAfter(end)); @@ -120,9 +120,8 @@ public class FileReferenceDownloader { return false; } } else { - log.log(logLevel, "Downloading " + fileReference + " from " + connection.getAddress() + " failed: " + - request + ", error: " + request.errorCode() + "(" + request.errorMessage() + - "). Will switch config server for next request" + + log.log(logLevel, "Downloading " + fileReference + " from " + connection.getAddress() + " failed:" + + " error code " + request.errorCode() + " (" + request.errorMessage() + ")." + " (retry " + retryCount + ", rpc timeout " + rpcTimeout + ")"); return false; } @@ -136,7 +135,7 @@ public class FileReferenceDownloader { } private Duration rpcTimeout(int retryCount) { - return Duration.ofSeconds(rpcTimeout.getSeconds()).plus(Duration.ofSeconds(retryCount * 10L)); + return Duration.ofSeconds(rpcTimeout.getSeconds()).plus(Duration.ofSeconds(retryCount * 5L)); } private boolean validateResponse(Request request) { diff --git a/flags/src/main/java/com/yahoo/vespa/flags/Flags.java b/flags/src/main/java/com/yahoo/vespa/flags/Flags.java index 4d436c17100..c3394c9dd76 100644 --- a/flags/src/main/java/com/yahoo/vespa/flags/Flags.java +++ b/flags/src/main/java/com/yahoo/vespa/flags/Flags.java @@ -47,21 +47,28 @@ public class Flags { public static final UnboundDoubleFlag DEFAULT_TERM_WISE_LIMIT = defineDoubleFlag( "default-term-wise-limit", 1.0, - List.of("baldersheim"), "2020-12-02", "2022-01-01", + List.of("baldersheim"), "2020-12-02", "2022-02-01", "Default limit for when to apply termwise query evaluation", "Takes effect at redeployment", ZONE_ID, APPLICATION_ID); + public static final UnboundDoubleFlag TLS_SIZE_FRACTION = defineDoubleFlag( + "tls-size-fraction", 0.07, + List.of("baldersheim"), "2021-12-20", "2022-02-01", + "Fraction of disk available for transaction log", + "Takes effect at redeployment", + ZONE_ID, APPLICATION_ID); + public static final UnboundStringFlag FEED_SEQUENCER_TYPE = defineStringFlag( "feed-sequencer-type", "LATENCY", - List.of("baldersheim"), "2020-12-02", "2022-01-01", + List.of("baldersheim"), "2020-12-02", "2022-02-01", "Selects type of sequenced executor used for feeding in proton, valid values are LATENCY, ADAPTIVE, THROUGHPUT", "Takes effect at redeployment (requires restart)", ZONE_ID, APPLICATION_ID); public static final UnboundIntFlag FEED_TASK_LIMIT = defineIntFlag( "feed-task-limit", 1000, - List.of("geirst, baldersheim"), "2021-10-14", "2022-01-01", + List.of("geirst, baldersheim"), "2021-10-14", "2022-02-01", "The task limit used by the executors handling feed in proton", "Takes effect at redeployment", ZONE_ID, APPLICATION_ID); @@ -83,56 +90,56 @@ public class Flags { public static final UnboundIntFlag MAX_UNCOMMITTED_MEMORY = defineIntFlag( "max-uncommitted-memory", 130000, - List.of("geirst, baldersheim"), "2021-10-21", "2022-01-01", + List.of("geirst, baldersheim"), "2021-10-21", "2022-02-01", "Max amount of memory holding updates to an attribute before we do a commit.", "Takes effect at redeployment", ZONE_ID, APPLICATION_ID); public static final UnboundStringFlag RESPONSE_SEQUENCER_TYPE = defineStringFlag( "response-sequencer-type", "ADAPTIVE", - List.of("baldersheim"), "2020-12-02", "2022-01-01", + List.of("baldersheim"), "2020-12-02", "2022-02-01", "Selects type of sequenced executor used for mbus responses, valid values are LATENCY, ADAPTIVE, THROUGHPUT", "Takes effect at redeployment", ZONE_ID, APPLICATION_ID); public static final UnboundIntFlag RESPONSE_NUM_THREADS = defineIntFlag( "response-num-threads", 2, - List.of("baldersheim"), "2020-12-02", "2022-01-01", + List.of("baldersheim"), "2020-12-02", "2022-02-01", "Number of threads used for mbus responses, default is 2, negative number = numcores/4", "Takes effect at redeployment", ZONE_ID, APPLICATION_ID); public static final UnboundBooleanFlag SKIP_COMMUNICATIONMANAGER_THREAD = defineFeatureFlag( "skip-communicationmanager-thread", false, - List.of("baldersheim"), "2020-12-02", "2022-01-01", + List.of("baldersheim"), "2020-12-02", "2022-02-01", "Should we skip the communicationmanager thread", "Takes effect at redeployment", ZONE_ID, APPLICATION_ID); public static final UnboundBooleanFlag SKIP_MBUS_REQUEST_THREAD = defineFeatureFlag( "skip-mbus-request-thread", false, - List.of("baldersheim"), "2020-12-02", "2022-01-01", + List.of("baldersheim"), "2020-12-02", "2022-02-01", "Should we skip the mbus request thread", "Takes effect at redeployment", ZONE_ID, APPLICATION_ID); public static final UnboundBooleanFlag SKIP_MBUS_REPLY_THREAD = defineFeatureFlag( "skip-mbus-reply-thread", false, - List.of("baldersheim"), "2020-12-02", "2022-01-01", + List.of("baldersheim"), "2020-12-02", "2022-02-01", "Should we skip the mbus reply thread", "Takes effect at redeployment", ZONE_ID, APPLICATION_ID); public static final UnboundBooleanFlag USE_THREE_PHASE_UPDATES = defineFeatureFlag( "use-three-phase-updates", false, - List.of("vekterli"), "2020-12-02", "2022-01-01", + List.of("vekterli"), "2020-12-02", "2022-02-01", "Whether to enable the use of three-phase updates when bucket replicas are out of sync.", "Takes effect at redeployment", ZONE_ID, APPLICATION_ID); public static final UnboundBooleanFlag HIDE_SHARED_ROUTING_ENDPOINT = defineFeatureFlag( "hide-shared-routing-endpoint", false, - List.of("tokle", "bjormel"), "2020-12-02", "2022-01-01", + List.of("tokle", "bjormel"), "2020-12-02", "2022-02-01", "Whether the controller should hide shared routing layer endpoint", "Takes effect immediately", APPLICATION_ID @@ -140,35 +147,35 @@ public class Flags { public static final UnboundBooleanFlag USE_ASYNC_MESSAGE_HANDLING_ON_SCHEDULE = defineFeatureFlag( "async-message-handling-on-schedule", false, - List.of("baldersheim"), "2020-12-02", "2022-01-01", + List.of("baldersheim"), "2020-12-02", "2022-02-01", "Optionally deliver async messages in own thread", "Takes effect at redeployment", ZONE_ID, APPLICATION_ID); public static final UnboundDoubleFlag FEED_CONCURRENCY = defineDoubleFlag( "feed-concurrency", 0.5, - List.of("baldersheim"), "2020-12-02", "2022-01-01", + List.of("baldersheim"), "2020-12-02", "2022-02-01", "How much concurrency should be allowed for feed", "Takes effect at redeployment", ZONE_ID, APPLICATION_ID); public static final UnboundDoubleFlag DISK_BLOAT_FACTOR = defineDoubleFlag( "disk-bloat-factor", 0.2, - List.of("baldersheim"), "2021-10-08", "2022-01-01", + List.of("baldersheim"), "2021-10-08", "2022-02-01", "Amount of bloat allowed before compacting file", "Takes effect at redeployment", ZONE_ID, APPLICATION_ID); public static final UnboundIntFlag DOCSTORE_COMPRESSION_LEVEL = defineIntFlag( "docstore-compression-level", 3, - List.of("baldersheim"), "2021-10-08", "2022-01-01", + List.of("baldersheim"), "2021-10-08", "2022-02-01", "Default compression level used for document store", "Takes effect at redeployment", ZONE_ID, APPLICATION_ID); public static final UnboundIntFlag NUM_DEPLOY_HELPER_THREADS = defineIntFlag( "num-model-builder-threads", -1, - List.of("balder"), "2021-09-09", "2022-01-01", + List.of("balder"), "2021-09-09", "2022-02-01", "Number of threads used for speeding up building of models.", "Takes effect on first (re)start of config server"); @@ -181,14 +188,14 @@ public class Flags { public static final UnboundBooleanFlag CONTAINER_DUMP_HEAP_ON_SHUTDOWN_TIMEOUT = defineFeatureFlag( "container-dump-heap-on-shutdown-timeout", false, - List.of("baldersheim"), "2021-09-25", "2022-01-01", + List.of("baldersheim"), "2021-09-25", "2022-02-01", "Will trigger a heap dump during if container shutdown times out", "Takes effect at redeployment", ZONE_ID, APPLICATION_ID); public static final UnboundDoubleFlag CONTAINER_SHUTDOWN_TIMEOUT = defineDoubleFlag( "container-shutdown-timeout", 50.0, - List.of("baldersheim"), "2021-09-25", "2022-01-01", + List.of("baldersheim"), "2021-09-25", "2022-02-01", "Timeout for shutdown of a jdisc container", "Takes effect at redeployment", ZONE_ID, APPLICATION_ID); @@ -216,14 +223,14 @@ public class Flags { public static final UnboundIntFlag MAX_CONCURRENT_MERGES_PER_NODE = defineIntFlag( "max-concurrent-merges-per-node", 128, - List.of("balder", "vekterli"), "2021-06-06", "2022-01-01", + List.of("balder", "vekterli"), "2021-06-06", "2022-02-01", "Specifies max concurrent merges per content node.", "Takes effect at redeploy", ZONE_ID, APPLICATION_ID); public static final UnboundIntFlag MAX_MERGE_QUEUE_SIZE = defineIntFlag( "max-merge-queue-size", 1024, - List.of("balder", "vekterli"), "2021-06-06", "2022-01-01", + List.of("balder", "vekterli"), "2021-06-06", "2022-02-01", "Specifies max size of merge queue.", "Takes effect at redeploy", ZONE_ID, APPLICATION_ID); @@ -238,7 +245,7 @@ public class Flags { public static final UnboundIntFlag LARGE_RANK_EXPRESSION_LIMIT = defineIntFlag( "large-rank-expression-limit", 8192, - List.of("baldersheim"), "2021-06-09", "2022-01-01", + List.of("baldersheim"), "2021-06-09", "2022-02-01", "Limit for size of rank expressions distributed by filedistribution", "Takes effect on next internal redeployment", APPLICATION_ID); @@ -252,14 +259,14 @@ public class Flags { public static final UnboundIntFlag METRICSPROXY_NUM_THREADS = defineIntFlag( "metricsproxy-num-threads", 2, - List.of("balder"), "2021-09-01", "2022-01-01", + List.of("balder"), "2021-09-01", "2022-02-01", "Number of threads for metrics proxy", "Takes effect at redeployment", ZONE_ID, APPLICATION_ID); public static final UnboundBooleanFlag ENABLED_HORIZON_DASHBOARD = defineFeatureFlag( "enabled-horizon-dashboard", false, - List.of("olaa"), "2021-09-13", "2021-12-31", + List.of("olaa"), "2021-09-13", "2022-02-01", "Enable Horizon dashboard", "Takes effect immediately", TENANT_ID, CONSOLE_USER_EMAIL @@ -267,7 +274,7 @@ public class Flags { public static final UnboundBooleanFlag ENABLE_ONPREM_TENANT_S3_ARCHIVE = defineFeatureFlag( "enable-onprem-tenant-s3-archive", false, - List.of("bjorncs"), "2021-09-14", "2021-12-31", + List.of("bjorncs"), "2021-09-14", "2022-02-01", "Enable tenant S3 buckets in cd/main. Must be set on controller cluster only.", "Takes effect immediately", ZONE_ID, TENANT_ID @@ -275,7 +282,7 @@ public class Flags { public static final UnboundBooleanFlag DELETE_UNMAINTAINED_CERTIFICATES = defineFeatureFlag( "delete-unmaintained-certificates", false, - List.of("andreer"), "2021-09-23", "2021-12-21", + List.of("andreer"), "2021-09-23", "2022-02-01", "Whether to delete certificates that are known by provider but not by controller", "Takes effect on next run of EndpointCertificateMaintainer" ); @@ -289,7 +296,7 @@ public class Flags { public static final UnboundBooleanFlag ENABLE_TENANT_DEVELOPER_ROLE = defineFeatureFlag( "enable-tenant-developer-role", false, - List.of("bjorncs"), "2021-09-23", "2021-12-31", + List.of("bjorncs"), "2021-09-23", "2022-02-01", "Enable tenant developer Athenz role in cd/main. Must be set on controller cluster only.", "Takes effect immediately", TENANT_ID @@ -297,7 +304,7 @@ public class Flags { public static final UnboundBooleanFlag ENABLE_ROUTING_REUSE_PORT = defineFeatureFlag( "enable-routing-reuse-port", false, - List.of("mortent"), "2021-09-29", "2021-12-31", + List.of("mortent"), "2021-09-29", "2022-02-01", "Enable reuse port in routing configuration", "Takes effect on container restart", HOSTNAME @@ -305,7 +312,7 @@ public class Flags { public static final UnboundBooleanFlag ENABLE_TENANT_OPERATOR_ROLE = defineFeatureFlag( "enable-tenant-operator-role", false, - List.of("bjorncs"), "2021-09-29", "2021-12-31", + List.of("bjorncs"), "2021-09-29", "2022-02-01", "Enable tenant specific operator roles in public systems. For controllers only.", "Takes effect on subsequent maintainer invocation", TENANT_ID @@ -367,7 +374,7 @@ public class Flags { public static final UnboundBooleanFlag USE_LEGACY_LB_SERVICES = defineFeatureFlag( "use-legacy-lb-services", false, - List.of("tokle"), "2021-11-22", "2021-12-31", + List.of("tokle"), "2021-11-22", "2022-02-01", "Whether to generate routing table based on legacy lb-services config", "Takes effect on container reboot", ZONE_ID, HOSTNAME); @@ -379,6 +386,20 @@ public class Flags { "Takes effect at redeployment", ZONE_ID, APPLICATION_ID); + public static final UnboundIntFlag MAX_COMPACT_BUFFERS = defineIntFlag( + "max-compact-buffers", 1, + List.of("baldersheim", "geirst", "toregge"), "2021-12-15", "2022-03-31", + "Upper limit of buffers to compact in a data store at the same time for each reason (memory usage, address space usage)", + "Takes effect at redeployment", + ZONE_ID, APPLICATION_ID); + + public static final UnboundBooleanFlag FAIL_DEPLOYMENT_WITH_INVALID_JVM_OPTIONS = defineFeatureFlag( + "fail-deployment-with-invalid-jvm-options", false, + List.of("hmusum"), "2021-12-20", "2022-01-20", + "Whether to fail deployments with invalid JVM options in services.xml", + "Takes effect at redeployment", + ZONE_ID, APPLICATION_ID); + /** WARNING: public for testing: All flags should be defined in {@link Flags}. */ public static UnboundBooleanFlag defineFeatureFlag(String flagId, boolean defaultValue, List<String> owners, String createdAt, String expiresAt, String description, diff --git a/flags/src/main/java/com/yahoo/vespa/flags/PermanentFlags.java b/flags/src/main/java/com/yahoo/vespa/flags/PermanentFlags.java index 34547d14616..b39a3309ad9 100644 --- a/flags/src/main/java/com/yahoo/vespa/flags/PermanentFlags.java +++ b/flags/src/main/java/com/yahoo/vespa/flags/PermanentFlags.java @@ -207,8 +207,8 @@ public class PermanentFlags { // This must be set in a feature flag to avoid flickering between the new and old value during config server upgrade public static final UnboundDoubleFlag HOST_MEMORY = defineDoubleFlag( - "host-memory", 1.0, - "The memory required by a hosts management processes.", + "host-memory", 0.6, + "The memory in GB required by a host's management processes.", "Takes effect immediately", ZONE_ID ); @@ -219,6 +219,13 @@ public class PermanentFlags { "Takes effect immediately", ZONE_ID, APPLICATION_ID); + public static final UnboundBooleanFlag DEACTIVATE_ROUTING = defineFeatureFlag( + "deactivate-routing", false, + "Deactivates routing for an application by removing all reals from its load balancers. Used in " + + "cases where we immediately need to stop serving an application, i.e. in case of service violations", + "Takes effect on next redeployment", + APPLICATION_ID); + private PermanentFlags() {} private static UnboundBooleanFlag defineFeatureFlag( diff --git a/indexinglanguage/pom.xml b/indexinglanguage/pom.xml index cfc7b09a934..32d8068dfcd 100644 --- a/indexinglanguage/pom.xml +++ b/indexinglanguage/pom.xml @@ -44,6 +44,11 @@ <artifactId>predicate-search-core</artifactId> <version>${project.version}</version> </dependency> + <dependency> + <groupId>com.google.guava</groupId> + <artifactId>guava</artifactId> + <scope>test</scope> + </dependency> </dependencies> <build> <plugins> diff --git a/integration/intellij/src/main/java/ai/vespa/intellij/schema/SdChooseByNameContributor.java b/integration/intellij/src/main/java/ai/vespa/intellij/schema/SdChooseByNameContributor.java index 01992ef4a5b..fcf1fd1a69a 100644 --- a/integration/intellij/src/main/java/ai/vespa/intellij/schema/SdChooseByNameContributor.java +++ b/integration/intellij/src/main/java/ai/vespa/intellij/schema/SdChooseByNameContributor.java @@ -10,7 +10,6 @@ import com.intellij.psi.search.FileTypeIndex; import com.intellij.psi.search.GlobalSearchScope; import ai.vespa.intellij.schema.psi.SdDeclaration; import ai.vespa.intellij.schema.psi.SdFile; -import org.jetbrains.annotations.NotNull; import java.util.ArrayList; import java.util.Collection; @@ -24,7 +23,7 @@ import java.util.List; public class SdChooseByNameContributor implements ChooseByNameContributor { @Override - public String @NotNull [] getNames(Project project, boolean includeNonProjectItems) { + public String[] getNames(Project project, boolean includeNonProjectItems) { Collection<VirtualFile> virtualFiles = FileTypeIndex.getFiles(SdFileType.INSTANCE, GlobalSearchScope.allScope(project)); List<SdDeclaration> declarations = new ArrayList<>(); @@ -42,7 +41,7 @@ public class SdChooseByNameContributor implements ChooseByNameContributor { } @Override - public NavigationItem @NotNull [] getItemsByName(String name, String pattern, Project project, boolean includeNonProjectItems) { + public NavigationItem[] getItemsByName(String name, String pattern, Project project, boolean includeNonProjectItems) { Collection<VirtualFile> virtualFiles = FileTypeIndex.getFiles(SdFileType.INSTANCE, GlobalSearchScope.allScope(project)); List<SdDeclaration> declarations = new ArrayList<>(); diff --git a/integration/intellij/src/main/java/ai/vespa/intellij/schema/SdCodeStyleSettingsProvider.java b/integration/intellij/src/main/java/ai/vespa/intellij/schema/SdCodeStyleSettingsProvider.java index b02e8371b77..a8e5827f108 100644 --- a/integration/intellij/src/main/java/ai/vespa/intellij/schema/SdCodeStyleSettingsProvider.java +++ b/integration/intellij/src/main/java/ai/vespa/intellij/schema/SdCodeStyleSettingsProvider.java @@ -8,8 +8,6 @@ import com.intellij.psi.codeStyle.CodeStyleConfigurable; import com.intellij.psi.codeStyle.CodeStyleSettings; import com.intellij.psi.codeStyle.CodeStyleSettingsProvider; import com.intellij.psi.codeStyle.CustomCodeStyleSettings; -import org.jetbrains.annotations.NotNull; -import org.jetbrains.annotations.Nullable; /** * This class is used for the extension (in plugin.xml) to the class SdCodeStyleSettings. @@ -23,14 +21,12 @@ public class SdCodeStyleSettingsProvider extends CodeStyleSettingsProvider { return new SdCodeStyleSettings(settings); } - @Nullable @Override public String getConfigurableDisplayName() { return "Sd"; } - @NotNull - public CodeStyleConfigurable createConfigurable(@NotNull CodeStyleSettings settings, @NotNull CodeStyleSettings modelSettings) { + public CodeStyleConfigurable createConfigurable(CodeStyleSettings settings, CodeStyleSettings modelSettings) { return new CodeStyleAbstractConfigurable(settings, modelSettings, this.getConfigurableDisplayName()) { @Override protected CodeStyleAbstractPanel createPanel(CodeStyleSettings settings) { diff --git a/integration/intellij/src/main/java/ai/vespa/intellij/schema/SdCommenter.java b/integration/intellij/src/main/java/ai/vespa/intellij/schema/SdCommenter.java index a8872f71b63..bb3de625391 100644 --- a/integration/intellij/src/main/java/ai/vespa/intellij/schema/SdCommenter.java +++ b/integration/intellij/src/main/java/ai/vespa/intellij/schema/SdCommenter.java @@ -2,7 +2,6 @@ package ai.vespa.intellij.schema; import com.intellij.lang.Commenter; -import org.jetbrains.annotations.Nullable; /** * This class is used for the extension (in plugin.xml), to enable turning a line into a comment with @@ -12,31 +11,26 @@ import org.jetbrains.annotations.Nullable; */ public class SdCommenter implements Commenter { - @Nullable @Override public String getLineCommentPrefix() { return "#"; } - @Nullable @Override public String getBlockCommentPrefix() { return ""; } - @Nullable @Override public String getBlockCommentSuffix() { return null; } - @Nullable @Override public String getCommentedBlockCommentPrefix() { return null; } - @Nullable @Override public String getCommentedBlockCommentSuffix() { return null; diff --git a/integration/intellij/src/main/java/ai/vespa/intellij/schema/SdCompletionContributor.java b/integration/intellij/src/main/java/ai/vespa/intellij/schema/SdCompletionContributor.java index 3e5b53c87ea..43feb26bbe4 100644 --- a/integration/intellij/src/main/java/ai/vespa/intellij/schema/SdCompletionContributor.java +++ b/integration/intellij/src/main/java/ai/vespa/intellij/schema/SdCompletionContributor.java @@ -10,7 +10,6 @@ import com.intellij.codeInsight.lookup.LookupElementBuilder; import com.intellij.patterns.PlatformPatterns; import com.intellij.util.ProcessingContext; import ai.vespa.intellij.schema.psi.SdTypes; -import org.jetbrains.annotations.NotNull; /** * This class is used for the extension (in plugin.xml) to enables Auto-Complete. Partially works for now. @@ -24,9 +23,9 @@ public class SdCompletionContributor extends CompletionContributor { extend(CompletionType.BASIC, PlatformPatterns.psiElement(SdTypes.IDENTIFIER_VAL), new CompletionProvider<>() { - public void addCompletions(@NotNull CompletionParameters parameters, //completion parameters contain details of the cursor position - @NotNull ProcessingContext context, - @NotNull CompletionResultSet resultSet) { //result set contains completion details to suggest + public void addCompletions(CompletionParameters parameters, //completion parameters contain details of the cursor position + ProcessingContext context, + CompletionResultSet resultSet) { //result set contains completion details to suggest resultSet.addElement(LookupElementBuilder.create("")); } } diff --git a/integration/intellij/src/main/java/ai/vespa/intellij/schema/SdFileType.java b/integration/intellij/src/main/java/ai/vespa/intellij/schema/SdFileType.java index b469d0c0ebb..f2b38a61844 100644 --- a/integration/intellij/src/main/java/ai/vespa/intellij/schema/SdFileType.java +++ b/integration/intellij/src/main/java/ai/vespa/intellij/schema/SdFileType.java @@ -2,8 +2,6 @@ package ai.vespa.intellij.schema; import com.intellij.openapi.fileTypes.LanguageFileType; -import org.jetbrains.annotations.NotNull; -import org.jetbrains.annotations.Nullable; import javax.swing.Icon; @@ -20,25 +18,21 @@ public class SdFileType extends LanguageFileType { super(SdLanguage.INSTANCE); } - @NotNull @Override public String getName() { return "Sd File"; } - @NotNull @Override public String getDescription() { return "Sd language file"; } - @NotNull @Override public String getDefaultExtension() { return "sd"; } - @Nullable @Override public Icon getIcon() { return SdIcons.FILE; diff --git a/integration/intellij/src/main/java/ai/vespa/intellij/schema/SdLanguageCodeStyleSettingsProvider.java b/integration/intellij/src/main/java/ai/vespa/intellij/schema/SdLanguageCodeStyleSettingsProvider.java index 5f618a62d6e..afc7f4b946e 100644 --- a/integration/intellij/src/main/java/ai/vespa/intellij/schema/SdLanguageCodeStyleSettingsProvider.java +++ b/integration/intellij/src/main/java/ai/vespa/intellij/schema/SdLanguageCodeStyleSettingsProvider.java @@ -4,7 +4,6 @@ package ai.vespa.intellij.schema; import com.intellij.lang.Language; import com.intellij.psi.codeStyle.CodeStyleSettingsCustomizable; import com.intellij.psi.codeStyle.LanguageCodeStyleSettingsProvider; -import org.jetbrains.annotations.NotNull; /** * This class is used for the extension (in plugin.xml), to make the IDE use our plugin's code for coding style. @@ -13,14 +12,13 @@ import org.jetbrains.annotations.NotNull; */ public class SdLanguageCodeStyleSettingsProvider extends LanguageCodeStyleSettingsProvider { - @NotNull @Override public Language getLanguage() { return SdLanguage.INSTANCE; } @Override - public void customizeSettings(@NotNull CodeStyleSettingsCustomizable consumer, @NotNull SettingsType settingsType) { + public void customizeSettings(CodeStyleSettingsCustomizable consumer, SettingsType settingsType) { if (settingsType == SettingsType.SPACING_SETTINGS) { consumer.showStandardOptions("SPACE_AFTER_COLON"); // consumer.renameStandardOption("SPACE_AROUND_ASSIGNMENT_OPERATORS", "Separator"); @@ -32,7 +30,7 @@ public class SdLanguageCodeStyleSettingsProvider extends LanguageCodeStyleSettin } @Override - public String getCodeSample(@NotNull SettingsType settingsType) { + public String getCodeSample(SettingsType settingsType) { return "field myField type int {\n indexing: summary\n}"; } diff --git a/integration/intellij/src/main/java/ai/vespa/intellij/schema/SdRefactoringSupportProvider.java b/integration/intellij/src/main/java/ai/vespa/intellij/schema/SdRefactoringSupportProvider.java index 24de7bb6253..88dbf6ce0ff 100644 --- a/integration/intellij/src/main/java/ai/vespa/intellij/schema/SdRefactoringSupportProvider.java +++ b/integration/intellij/src/main/java/ai/vespa/intellij/schema/SdRefactoringSupportProvider.java @@ -4,8 +4,6 @@ package ai.vespa.intellij.schema; import com.intellij.lang.refactoring.RefactoringSupportProvider; import com.intellij.psi.PsiElement; import ai.vespa.intellij.schema.psi.SdIdentifier; -import org.jetbrains.annotations.NotNull; -import org.jetbrains.annotations.Nullable; /** * This class is used for the extension (in plugin.xml), to enable refactoring. @@ -15,7 +13,7 @@ import org.jetbrains.annotations.Nullable; public class SdRefactoringSupportProvider extends RefactoringSupportProvider { @Override - public boolean isMemberInplaceRenameAvailable(@NotNull PsiElement elementToRename, @Nullable PsiElement context) { + public boolean isMemberInplaceRenameAvailable(PsiElement elementToRename, PsiElement context) { return (elementToRename instanceof SdIdentifier); } diff --git a/integration/intellij/src/main/java/ai/vespa/intellij/schema/SdReference.java b/integration/intellij/src/main/java/ai/vespa/intellij/schema/SdReference.java index 6404a738e59..8218d8cc81b 100644 --- a/integration/intellij/src/main/java/ai/vespa/intellij/schema/SdReference.java +++ b/integration/intellij/src/main/java/ai/vespa/intellij/schema/SdReference.java @@ -12,8 +12,6 @@ import com.intellij.psi.PsiReferenceBase; import com.intellij.psi.ResolveResult; import com.intellij.util.IncorrectOperationException; import ai.vespa.intellij.schema.psi.SdDeclaration; -import org.jetbrains.annotations.NotNull; -import org.jetbrains.annotations.Nullable; import java.util.ArrayList; import java.util.List; @@ -27,14 +25,13 @@ public class SdReference extends PsiReferenceBase<PsiElement> implements PsiPoly private final String elementName; - public SdReference(@NotNull PsiElement element, TextRange textRange) { + public SdReference(PsiElement element, TextRange textRange) { super(element, textRange); elementName = element.getText().substring(textRange.getStartOffset(), textRange.getEndOffset()); } - @NotNull @Override - public ResolveResult @NotNull [] multiResolve(boolean incompleteCode) { + public ResolveResult[] multiResolve(boolean incompleteCode) { PsiElement file = myElement.getContainingFile(); @@ -48,7 +45,6 @@ public class SdReference extends PsiReferenceBase<PsiElement> implements PsiPoly return results.toArray(new ResolveResult[results.size()]); } - @Nullable @Override public PsiElement resolve() { ResolveResult[] resolveResults = multiResolve(false); @@ -56,7 +52,7 @@ public class SdReference extends PsiReferenceBase<PsiElement> implements PsiPoly } @Override - public Object @NotNull [] getVariants() { + public Object[] getVariants() { List<SdDeclaration> declarations = SdUtil.findDeclarations(myElement.getContainingFile()); List<LookupElement> variants = new ArrayList<>(); for (final SdDeclaration element : declarations) { @@ -71,7 +67,7 @@ public class SdReference extends PsiReferenceBase<PsiElement> implements PsiPoly } @Override - public PsiElement handleElementRename(@NotNull String newElementName) throws IncorrectOperationException { + public PsiElement handleElementRename(String newElementName) throws IncorrectOperationException { return ((PsiNamedElement) myElement).setName(newElementName); } diff --git a/integration/intellij/src/main/java/ai/vespa/intellij/schema/SdSyntaxHighlighter.java b/integration/intellij/src/main/java/ai/vespa/intellij/schema/SdSyntaxHighlighter.java index df3da81e9fc..666687e56c1 100644 --- a/integration/intellij/src/main/java/ai/vespa/intellij/schema/SdSyntaxHighlighter.java +++ b/integration/intellij/src/main/java/ai/vespa/intellij/schema/SdSyntaxHighlighter.java @@ -12,7 +12,6 @@ import com.intellij.psi.TokenType; import com.intellij.psi.tree.IElementType; import ai.vespa.intellij.schema.lexer.SdLexerAdapter; import ai.vespa.intellij.schema.psi.SdTypes; -import org.jetbrains.annotations.NotNull; import java.util.HashSet; @@ -54,14 +53,13 @@ public class SdSyntaxHighlighter extends SyntaxHighlighterBase { private static final TextAttributesKey[] COMMENT_KEYS = new TextAttributesKey[]{COMMENT}; private static final TextAttributesKey[] EMPTY_KEYS = new TextAttributesKey[0]; - @NotNull @Override public Lexer getHighlightingLexer() { return new SdLexerAdapter(); } @Override - public TextAttributesKey @NotNull [] getTokenHighlights(IElementType tokenType) { + public TextAttributesKey[] getTokenHighlights(IElementType tokenType) { if (tokenType.equals(SdTypes.IDENTIFIER_VAL) || tokenType.equals(SdTypes.IDENTIFIER_WITH_DASH_VAL)) { return IDENTIFIER_KEYS; } else if (keyWordsSet.contains(tokenType)) { diff --git a/integration/intellij/src/main/java/ai/vespa/intellij/schema/SdSyntaxHighlighterFactory.java b/integration/intellij/src/main/java/ai/vespa/intellij/schema/SdSyntaxHighlighterFactory.java index 6cf30ecb54e..9101c85c55d 100644 --- a/integration/intellij/src/main/java/ai/vespa/intellij/schema/SdSyntaxHighlighterFactory.java +++ b/integration/intellij/src/main/java/ai/vespa/intellij/schema/SdSyntaxHighlighterFactory.java @@ -5,7 +5,6 @@ import com.intellij.openapi.fileTypes.SyntaxHighlighter; import com.intellij.openapi.fileTypes.SyntaxHighlighterFactory; import com.intellij.openapi.project.Project; import com.intellij.openapi.vfs.VirtualFile; -import org.jetbrains.annotations.NotNull; /** * This class is used for the extension (in plugin.xml) to the class SdSyntaxHighlighter. @@ -14,7 +13,6 @@ import org.jetbrains.annotations.NotNull; */ public class SdSyntaxHighlighterFactory extends SyntaxHighlighterFactory { - @NotNull @Override public SyntaxHighlighter getSyntaxHighlighter(Project project, VirtualFile virtualFile) { return new SdSyntaxHighlighter(); diff --git a/integration/intellij/src/main/java/ai/vespa/intellij/schema/SdUtil.java b/integration/intellij/src/main/java/ai/vespa/intellij/schema/SdUtil.java index f3f949d645f..d9f769d07cc 100644 --- a/integration/intellij/src/main/java/ai/vespa/intellij/schema/SdUtil.java +++ b/integration/intellij/src/main/java/ai/vespa/intellij/schema/SdUtil.java @@ -30,7 +30,6 @@ import ai.vespa.intellij.schema.psi.SdSchemaAnnotationDefinition; import ai.vespa.intellij.schema.psi.SdSchemaFieldDefinition; import ai.vespa.intellij.schema.psi.SdSummaryDefinition; import ai.vespa.intellij.schema.psi.SdTypes; -import org.jetbrains.annotations.NotNull; import java.util.ArrayList; import java.util.Collection; @@ -44,7 +43,7 @@ import java.util.List; */ public class SdUtil { - public static @NotNull HashMap<String, List<PsiElement>> createMacrosMap(SdFile file) { + public static HashMap<String, List<PsiElement>> createMacrosMap(SdFile file) { HashMap<String, List<PsiElement>> macrosMap = new HashMap<>(); for (SdRankProfileDefinition rankProfile : PsiTreeUtil .findChildrenOfType(file, SdRankProfileDefinition.class)) { diff --git a/integration/intellij/src/main/java/ai/vespa/intellij/schema/findUsages/SdDocumentSummaryGroupingRule.java b/integration/intellij/src/main/java/ai/vespa/intellij/schema/findUsages/SdDocumentSummaryGroupingRule.java index ab463562355..51a2e2490b3 100644 --- a/integration/intellij/src/main/java/ai/vespa/intellij/schema/findUsages/SdDocumentSummaryGroupingRule.java +++ b/integration/intellij/src/main/java/ai/vespa/intellij/schema/findUsages/SdDocumentSummaryGroupingRule.java @@ -10,8 +10,6 @@ import com.intellij.usages.rules.PsiElementUsage; import com.intellij.usages.rules.SingleParentUsageGroupingRule; import ai.vespa.intellij.schema.SdLanguage; import ai.vespa.intellij.schema.psi.SdDocumentSummaryDefinition; -import org.jetbrains.annotations.NotNull; -import org.jetbrains.annotations.Nullable; /** * A Document Summary that groups elements in the "Find Usages" window. @@ -21,7 +19,7 @@ import org.jetbrains.annotations.Nullable; public class SdDocumentSummaryGroupingRule extends SingleParentUsageGroupingRule implements DumbAware { @Override - protected @Nullable UsageGroup getParentGroupFor(@NotNull Usage usage, UsageTarget @NotNull [] targets) { + protected UsageGroup getParentGroupFor(Usage usage, UsageTarget[] targets) { PsiElement psiElement = usage instanceof PsiElementUsage ? ((PsiElementUsage)usage).getElement() : null; if (psiElement == null || psiElement.getLanguage() != SdLanguage.INSTANCE) return null; diff --git a/integration/intellij/src/main/java/ai/vespa/intellij/schema/findUsages/SdDocumentSummaryGroupingRuleProvider.java b/integration/intellij/src/main/java/ai/vespa/intellij/schema/findUsages/SdDocumentSummaryGroupingRuleProvider.java index 76b82d7c857..5c11622446e 100644 --- a/integration/intellij/src/main/java/ai/vespa/intellij/schema/findUsages/SdDocumentSummaryGroupingRuleProvider.java +++ b/integration/intellij/src/main/java/ai/vespa/intellij/schema/findUsages/SdDocumentSummaryGroupingRuleProvider.java @@ -4,8 +4,6 @@ package ai.vespa.intellij.schema.findUsages; import com.intellij.openapi.project.Project; import com.intellij.usages.impl.FileStructureGroupRuleProvider; import com.intellij.usages.rules.UsageGroupingRule; -import org.jetbrains.annotations.NotNull; -import org.jetbrains.annotations.Nullable; /** * This class is used for the extension (in plugin.xml) to the class SdDocumentSummaryGroupingRule. @@ -15,7 +13,7 @@ import org.jetbrains.annotations.Nullable; public class SdDocumentSummaryGroupingRuleProvider implements FileStructureGroupRuleProvider { @Override - public @Nullable UsageGroupingRule getUsageGroupingRule(@NotNull Project project) { + public UsageGroupingRule getUsageGroupingRule(Project project) { return new SdDocumentSummaryGroupingRule(); } } diff --git a/integration/intellij/src/main/java/ai/vespa/intellij/schema/findUsages/SdFindUsagesHandler.java b/integration/intellij/src/main/java/ai/vespa/intellij/schema/findUsages/SdFindUsagesHandler.java index 4609f2e7fbb..07807cd36f8 100644 --- a/integration/intellij/src/main/java/ai/vespa/intellij/schema/findUsages/SdFindUsagesHandler.java +++ b/integration/intellij/src/main/java/ai/vespa/intellij/schema/findUsages/SdFindUsagesHandler.java @@ -16,7 +16,6 @@ import com.intellij.util.Processor; import ai.vespa.intellij.schema.SdUtil; import ai.vespa.intellij.schema.psi.SdFile; import ai.vespa.intellij.schema.psi.SdFunctionDefinition; -import org.jetbrains.annotations.NotNull; import java.util.HashMap; import java.util.List; @@ -30,16 +29,16 @@ public class SdFindUsagesHandler extends FindUsagesHandler { protected HashMap<String, List<PsiElement>> macrosMap; - protected SdFindUsagesHandler(@NotNull PsiElement psiElement) { + protected SdFindUsagesHandler(PsiElement psiElement) { super(psiElement); PsiFile file = psiElement.getContainingFile(); macrosMap = file instanceof SdFile ? SdUtil.createMacrosMap((SdFile) psiElement.getContainingFile()) : new HashMap<>(); } @Override - public boolean processElementUsages(@NotNull final PsiElement elementToSearch, - @NotNull final Processor<? super UsageInfo> processor, - @NotNull final FindUsagesOptions options) { + public boolean processElementUsages(PsiElement elementToSearch, + Processor<? super UsageInfo> processor, + FindUsagesOptions options) { final ReadActionProcessor<PsiReference> refProcessor = new ReadActionProcessor<>() { @Override public boolean processInReadAction(final PsiReference ref) { diff --git a/integration/intellij/src/main/java/ai/vespa/intellij/schema/findUsages/SdFindUsagesHandlerFactory.java b/integration/intellij/src/main/java/ai/vespa/intellij/schema/findUsages/SdFindUsagesHandlerFactory.java index 98be15778e7..4b7ef552fa0 100644 --- a/integration/intellij/src/main/java/ai/vespa/intellij/schema/findUsages/SdFindUsagesHandlerFactory.java +++ b/integration/intellij/src/main/java/ai/vespa/intellij/schema/findUsages/SdFindUsagesHandlerFactory.java @@ -5,8 +5,6 @@ import com.intellij.find.findUsages.FindUsagesHandler; import com.intellij.find.findUsages.FindUsagesHandlerFactory; import com.intellij.psi.PsiElement; import com.intellij.psi.PsiNamedElement; -import org.jetbrains.annotations.NotNull; -import org.jetbrains.annotations.Nullable; /** * This class is used for the extension (in plugin.xml) to the class SdFindUsagesHandler. @@ -16,13 +14,13 @@ import org.jetbrains.annotations.Nullable; public class SdFindUsagesHandlerFactory extends FindUsagesHandlerFactory { @Override - public boolean canFindUsages(@NotNull PsiElement element) { + public boolean canFindUsages(PsiElement element) { return element instanceof PsiNamedElement; } @Override - public @Nullable FindUsagesHandler createFindUsagesHandler(@NotNull PsiElement element, - boolean forHighlightUsages) { + public FindUsagesHandler createFindUsagesHandler(PsiElement element, boolean forHighlightUsages) { return new SdFindUsagesHandler(element); } + } diff --git a/integration/intellij/src/main/java/ai/vespa/intellij/schema/findUsages/SdFindUsagesProvider.java b/integration/intellij/src/main/java/ai/vespa/intellij/schema/findUsages/SdFindUsagesProvider.java index a86fb2b058b..bd79f007fde 100644 --- a/integration/intellij/src/main/java/ai/vespa/intellij/schema/findUsages/SdFindUsagesProvider.java +++ b/integration/intellij/src/main/java/ai/vespa/intellij/schema/findUsages/SdFindUsagesProvider.java @@ -13,8 +13,6 @@ import ai.vespa.intellij.schema.psi.SdIdentifierVal; import ai.vespa.intellij.schema.psi.SdIdentifierWithDashVal; import ai.vespa.intellij.schema.psi.SdTypes; import ai.vespa.intellij.schema.psi.impl.SdNamedElementImpl; -import org.jetbrains.annotations.NotNull; -import org.jetbrains.annotations.Nullable; /** * This class is used for the extension (in plugin.xml), to enable "find Usages" window using the plugin code. @@ -23,8 +21,6 @@ import org.jetbrains.annotations.Nullable; */ public class SdFindUsagesProvider implements FindUsagesProvider { - @Nullable - @Override public WordsScanner getWordsScanner() { return new DefaultWordsScanner(new SdLexerAdapter(), TokenSet.create(SdTypes.ID_REG, SdTypes.ID_WITH_DASH_REG, SdTypes.IDENTIFIER_VAL, @@ -34,19 +30,17 @@ public class SdFindUsagesProvider implements FindUsagesProvider { } @Override - public boolean canFindUsagesFor(@NotNull PsiElement psiElement) { + public boolean canFindUsagesFor(PsiElement psiElement) { return psiElement instanceof PsiNamedElement; } - @Nullable @Override - public String getHelpId(@NotNull PsiElement psiElement) { + public String getHelpId(PsiElement psiElement) { return null; } - @NotNull @Override - public String getType(@NotNull PsiElement element) { + public String getType(PsiElement element) { if (element instanceof SdDeclaration) { return ((SdNamedElementImpl) element).getTypeName(); } else { @@ -54,15 +48,13 @@ public class SdFindUsagesProvider implements FindUsagesProvider { } } - @NotNull @Override - public String getDescriptiveName(@NotNull PsiElement element) { + public String getDescriptiveName(PsiElement element) { return ""; } - @NotNull @Override - public String getNodeText(@NotNull PsiElement element, boolean useFullName) { + public String getNodeText(PsiElement element, boolean useFullName) { if (element instanceof SdIdentifierVal || element instanceof SdIdentifierWithDashVal) { String name = ((PsiNamedElement) element).getName(); return name != null ? name : ""; diff --git a/integration/intellij/src/main/java/ai/vespa/intellij/schema/findUsages/SdRankProfileGroupingRule.java b/integration/intellij/src/main/java/ai/vespa/intellij/schema/findUsages/SdRankProfileGroupingRule.java index fd3073b6016..a9e9f9337dd 100644 --- a/integration/intellij/src/main/java/ai/vespa/intellij/schema/findUsages/SdRankProfileGroupingRule.java +++ b/integration/intellij/src/main/java/ai/vespa/intellij/schema/findUsages/SdRankProfileGroupingRule.java @@ -10,8 +10,6 @@ import com.intellij.usages.rules.PsiElementUsage; import com.intellij.usages.rules.SingleParentUsageGroupingRule; import ai.vespa.intellij.schema.SdLanguage; import ai.vespa.intellij.schema.psi.SdRankProfileDefinition; -import org.jetbrains.annotations.NotNull; -import org.jetbrains.annotations.Nullable; /** * A Rank Profile that groups elements in the "Find Usages" window. @@ -21,7 +19,7 @@ import org.jetbrains.annotations.Nullable; public class SdRankProfileGroupingRule extends SingleParentUsageGroupingRule implements DumbAware { @Override - protected @Nullable UsageGroup getParentGroupFor(@NotNull Usage usage, UsageTarget @NotNull [] targets) { + protected UsageGroup getParentGroupFor(Usage usage, UsageTarget[] targets) { PsiElement psiElement = usage instanceof PsiElementUsage ? ((PsiElementUsage)usage).getElement() : null; if (psiElement == null || psiElement.getLanguage() != SdLanguage.INSTANCE) return null; diff --git a/integration/intellij/src/main/java/ai/vespa/intellij/schema/findUsages/SdRankProfileGroupingRuleProvider.java b/integration/intellij/src/main/java/ai/vespa/intellij/schema/findUsages/SdRankProfileGroupingRuleProvider.java index 0726e103943..62a83500235 100644 --- a/integration/intellij/src/main/java/ai/vespa/intellij/schema/findUsages/SdRankProfileGroupingRuleProvider.java +++ b/integration/intellij/src/main/java/ai/vespa/intellij/schema/findUsages/SdRankProfileGroupingRuleProvider.java @@ -4,8 +4,6 @@ package ai.vespa.intellij.schema.findUsages; import com.intellij.openapi.project.Project; import com.intellij.usages.impl.FileStructureGroupRuleProvider; import com.intellij.usages.rules.UsageGroupingRule; -import org.jetbrains.annotations.NotNull; -import org.jetbrains.annotations.Nullable; /** * This class is used for the extension (in plugin.xml) to the class SdRankProfileGroupingRule. @@ -15,7 +13,7 @@ import org.jetbrains.annotations.Nullable; public class SdRankProfileGroupingRuleProvider implements FileStructureGroupRuleProvider { @Override - public @Nullable UsageGroupingRule getUsageGroupingRule(@NotNull Project project) { + public UsageGroupingRule getUsageGroupingRule(Project project) { return new SdRankProfileGroupingRule(); } diff --git a/integration/intellij/src/main/java/ai/vespa/intellij/schema/findUsages/SdUsageGroup.java b/integration/intellij/src/main/java/ai/vespa/intellij/schema/findUsages/SdUsageGroup.java index 6f24ce27476..7735255a84e 100644 --- a/integration/intellij/src/main/java/ai/vespa/intellij/schema/findUsages/SdUsageGroup.java +++ b/integration/intellij/src/main/java/ai/vespa/intellij/schema/findUsages/SdUsageGroup.java @@ -11,8 +11,6 @@ import com.intellij.psi.SmartPsiElementPointer; import com.intellij.usages.UsageGroup; import com.intellij.usages.UsageView; import ai.vespa.intellij.schema.psi.SdDeclaration; -import org.jetbrains.annotations.NotNull; -import org.jetbrains.annotations.Nullable; import javax.swing.*; @@ -61,7 +59,7 @@ public class SdUsageGroup implements UsageGroup { } @Override - public int compareTo(@NotNull UsageGroup usageGroup) { + public int compareTo(UsageGroup usageGroup) { // return getPresentableGroupText().compareToIgnoreCase(usageGroup.getPresentableGroupText()); return getText(null).compareTo(usageGroup.getText(null)); } @@ -86,12 +84,12 @@ public class SdUsageGroup implements UsageGroup { } @Override - public @NotNull String getPresentableGroupText() { + public String getPresentableGroupText() { return myText; } @Override - public @Nullable Icon getIcon() { + public Icon getIcon() { return myIcon; } diff --git a/integration/intellij/src/main/java/ai/vespa/intellij/schema/hierarchy/SdCallHierarchyBrowser.java b/integration/intellij/src/main/java/ai/vespa/intellij/schema/hierarchy/SdCallHierarchyBrowser.java index 38817e15553..19a47ba15d4 100644 --- a/integration/intellij/src/main/java/ai/vespa/intellij/schema/hierarchy/SdCallHierarchyBrowser.java +++ b/integration/intellij/src/main/java/ai/vespa/intellij/schema/hierarchy/SdCallHierarchyBrowser.java @@ -12,8 +12,6 @@ import com.intellij.openapi.project.Project; import com.intellij.psi.PsiElement; import com.intellij.util.ObjectUtils; import ai.vespa.intellij.schema.psi.SdFunctionDefinition; -import org.jetbrains.annotations.NotNull; -import org.jetbrains.annotations.Nullable; import java.util.Comparator; import java.util.Map; @@ -27,37 +25,33 @@ import javax.swing.JTree; */ public class SdCallHierarchyBrowser extends CallHierarchyBrowserBase { - public SdCallHierarchyBrowser(@NotNull Project project, - @NotNull PsiElement macro) { + public SdCallHierarchyBrowser(Project project, PsiElement macro) { super(project, macro); } - @Nullable @Override - protected PsiElement getElementFromDescriptor(@NotNull HierarchyNodeDescriptor descriptor) { + protected PsiElement getElementFromDescriptor(HierarchyNodeDescriptor descriptor) { return ObjectUtils.tryCast(descriptor.getPsiElement(), PsiElement.class); } @Override - protected void createTrees(@NotNull Map<? super String, ? super JTree> type2TreeMap) { + protected void createTrees(Map<? super String, ? super JTree> type2TreeMap) { ActionGroup group = (ActionGroup) ActionManager.getInstance().getAction(IdeActions.GROUP_CALL_HIERARCHY_POPUP); type2TreeMap.put(getCallerType(), createHierarchyTree(group)); type2TreeMap.put(getCalleeType(), createHierarchyTree(group)); } - private @NotNull JTree createHierarchyTree(ActionGroup group) { + private JTree createHierarchyTree(ActionGroup group) { return createTree(false); } @Override - protected boolean isApplicableElement(@NotNull PsiElement element) { + protected boolean isApplicableElement(PsiElement element) { return element instanceof SdFunctionDefinition; } - - @Nullable @Override - protected HierarchyTreeStructure createHierarchyTreeStructure(@NotNull String typeName, @NotNull PsiElement psiElement) { + protected HierarchyTreeStructure createHierarchyTreeStructure(String typeName, PsiElement psiElement) { if (getCallerType().equals(typeName)) { return new SdCallerTreeStructure(myProject, psiElement, getCurrentScopeType()); } @@ -69,7 +63,6 @@ public class SdCallHierarchyBrowser extends CallHierarchyBrowserBase { } } - @Nullable @Override protected Comparator<NodeDescriptor<?>> getComparator() { return SdHierarchyUtil.getComparator(myProject); diff --git a/integration/intellij/src/main/java/ai/vespa/intellij/schema/hierarchy/SdCallHierarchyNodeDescriptor.java b/integration/intellij/src/main/java/ai/vespa/intellij/schema/hierarchy/SdCallHierarchyNodeDescriptor.java index f99a82dc81a..62734cc1168 100644 --- a/integration/intellij/src/main/java/ai/vespa/intellij/schema/hierarchy/SdCallHierarchyNodeDescriptor.java +++ b/integration/intellij/src/main/java/ai/vespa/intellij/schema/hierarchy/SdCallHierarchyNodeDescriptor.java @@ -16,8 +16,6 @@ import ai.vespa.intellij.schema.psi.SdFirstPhaseDefinition; import ai.vespa.intellij.schema.psi.impl.SdFirstPhaseDefinitionMixin; import ai.vespa.intellij.schema.psi.SdFunctionDefinition; -import org.jetbrains.annotations.NotNull; - import javax.swing.Icon; /** @@ -27,7 +25,7 @@ import javax.swing.Icon; */ public class SdCallHierarchyNodeDescriptor extends HierarchyNodeDescriptor { - public SdCallHierarchyNodeDescriptor(final NodeDescriptor parentDescriptor, @NotNull final PsiElement element, final boolean isBase) { + public SdCallHierarchyNodeDescriptor(NodeDescriptor parentDescriptor, PsiElement element, boolean isBase) { super(element.getProject(), parentDescriptor, element, isBase); CompositeAppearance.DequeEnd beginning = myHighlightedText.getBeginning(); if (element instanceof SdFunctionDefinition) { diff --git a/integration/intellij/src/main/java/ai/vespa/intellij/schema/hierarchy/SdCallHierarchyProvider.java b/integration/intellij/src/main/java/ai/vespa/intellij/schema/hierarchy/SdCallHierarchyProvider.java index f7f243356b9..8319e5c46ac 100644 --- a/integration/intellij/src/main/java/ai/vespa/intellij/schema/hierarchy/SdCallHierarchyProvider.java +++ b/integration/intellij/src/main/java/ai/vespa/intellij/schema/hierarchy/SdCallHierarchyProvider.java @@ -12,8 +12,6 @@ import com.intellij.psi.PsiFile; import com.intellij.psi.PsiReference; import ai.vespa.intellij.schema.psi.SdFunctionDefinition; import ai.vespa.intellij.schema.psi.SdIdentifierVal; -import org.jetbrains.annotations.NotNull; -import org.jetbrains.annotations.Nullable; import com.intellij.openapi.actionSystem.CommonDataKeys; import com.intellij.openapi.project.Project; @@ -25,7 +23,7 @@ import com.intellij.openapi.project.Project; public class SdCallHierarchyProvider implements HierarchyProvider { @Override - public @Nullable PsiElement getTarget(@NotNull DataContext dataContext) { + public PsiElement getTarget(DataContext dataContext) { final Project project = CommonDataKeys.PROJECT.getData(dataContext); final Editor editor = CommonDataKeys.EDITOR.getData(dataContext); if (project == null || editor == null) return null; @@ -53,12 +51,12 @@ public class SdCallHierarchyProvider implements HierarchyProvider { } @Override - public @NotNull HierarchyBrowser createHierarchyBrowser(@NotNull PsiElement target) { + public HierarchyBrowser createHierarchyBrowser(PsiElement target) { return new SdCallHierarchyBrowser(target.getProject(), target); } @Override - public void browserActivated(@NotNull HierarchyBrowser hierarchyBrowser) { + public void browserActivated(HierarchyBrowser hierarchyBrowser) { ((SdCallHierarchyBrowser) hierarchyBrowser).changeView(CallHierarchyBrowserBase.getCallerType()); } diff --git a/integration/intellij/src/main/java/ai/vespa/intellij/schema/hierarchy/SdCallTreeStructure.java b/integration/intellij/src/main/java/ai/vespa/intellij/schema/hierarchy/SdCallTreeStructure.java index 053e7bd20f9..3dae0d8c818 100644 --- a/integration/intellij/src/main/java/ai/vespa/intellij/schema/hierarchy/SdCallTreeStructure.java +++ b/integration/intellij/src/main/java/ai/vespa/intellij/schema/hierarchy/SdCallTreeStructure.java @@ -12,7 +12,6 @@ import ai.vespa.intellij.schema.SdUtil; import ai.vespa.intellij.schema.psi.SdFile; import ai.vespa.intellij.schema.psi.SdFunctionDefinition; import ai.vespa.intellij.schema.psi.SdRankProfileDefinition; -import org.jetbrains.annotations.NotNull; import java.util.ArrayList; import java.util.HashMap; @@ -39,11 +38,10 @@ public abstract class SdCallTreeStructure extends HierarchyTreeStructure { ranksHeritageMap = new HashMap<>(); } - @NotNull - protected abstract HashSet<PsiElement> getChildren(@NotNull SdFunctionDefinition element); + protected abstract HashSet<PsiElement> getChildren(SdFunctionDefinition element); @Override - protected Object @NotNull [] buildChildren(@NotNull HierarchyNodeDescriptor descriptor) { + protected Object[] buildChildren(HierarchyNodeDescriptor descriptor) { final List<SdCallHierarchyNodeDescriptor> descriptors = new ArrayList<>(); if (descriptor instanceof SdCallHierarchyNodeDescriptor) { diff --git a/integration/intellij/src/main/java/ai/vespa/intellij/schema/hierarchy/SdCalleeTreeStructure.java b/integration/intellij/src/main/java/ai/vespa/intellij/schema/hierarchy/SdCalleeTreeStructure.java index 37fd336b615..59e6ca7add3 100644 --- a/integration/intellij/src/main/java/ai/vespa/intellij/schema/hierarchy/SdCalleeTreeStructure.java +++ b/integration/intellij/src/main/java/ai/vespa/intellij/schema/hierarchy/SdCalleeTreeStructure.java @@ -10,7 +10,6 @@ import ai.vespa.intellij.schema.psi.SdExpressionDefinition; import ai.vespa.intellij.schema.psi.SdFunctionDefinition; import ai.vespa.intellij.schema.psi.SdIdentifier; import ai.vespa.intellij.schema.psi.SdRankProfileDefinition; -import org.jetbrains.annotations.NotNull; import java.util.HashMap; import java.util.HashSet; @@ -27,13 +26,12 @@ public class SdCalleeTreeStructure extends SdCallTreeStructure { super(project, element, currentScopeType); } - @NotNull @Override - protected HashSet<PsiElement> getChildren(@NotNull SdFunctionDefinition element) { + protected HashSet<PsiElement> getChildren(SdFunctionDefinition element) { return getCallees(element, macrosMap); } - private HashSet<PsiElement> getCallees(@NotNull SdFunctionDefinition macro, HashMap<String, List<PsiElement>> macrosMap) { + private HashSet<PsiElement> getCallees(SdFunctionDefinition macro, HashMap<String, List<PsiElement>> macrosMap) { final HashSet<PsiElement> results = new HashSet<>(); SdExpressionDefinition expression = PsiTreeUtil.findChildOfType(macro, SdExpressionDefinition.class); if (expression == null) { @@ -69,5 +67,4 @@ public class SdCalleeTreeStructure extends SdCallTreeStructure { return results; } - } diff --git a/integration/intellij/src/main/java/ai/vespa/intellij/schema/hierarchy/SdCallerTreeStructure.java b/integration/intellij/src/main/java/ai/vespa/intellij/schema/hierarchy/SdCallerTreeStructure.java index 444e71978c9..f442c4ea5bc 100644 --- a/integration/intellij/src/main/java/ai/vespa/intellij/schema/hierarchy/SdCallerTreeStructure.java +++ b/integration/intellij/src/main/java/ai/vespa/intellij/schema/hierarchy/SdCallerTreeStructure.java @@ -11,7 +11,6 @@ import com.intellij.psi.util.PsiTreeUtil; import com.intellij.util.containers.ContainerUtil; import ai.vespa.intellij.schema.psi.SdFirstPhaseDefinition; import ai.vespa.intellij.schema.psi.SdFunctionDefinition; -import org.jetbrains.annotations.NotNull; import java.util.HashMap; import java.util.HashSet; @@ -32,13 +31,12 @@ public class SdCallerTreeStructure extends SdCallTreeStructure { macroTreeChildren = new HashMap<>(); } - @NotNull @Override - protected HashSet<PsiElement> getChildren(@NotNull SdFunctionDefinition element) { + protected HashSet<PsiElement> getChildren(SdFunctionDefinition element) { return getCallers(element, macrosMap); } - private HashSet<PsiElement> getCallers(@NotNull SdFunctionDefinition macro, @NotNull HashMap<String, List<PsiElement>> macrosMap) { + private HashSet<PsiElement> getCallers(SdFunctionDefinition macro, HashMap<String, List<PsiElement>> macrosMap) { String macroName = macro.getName(); if (macroTreeChildren.containsKey(macroName)) { diff --git a/integration/intellij/src/main/java/ai/vespa/intellij/schema/parser/SdParserDefinition.java b/integration/intellij/src/main/java/ai/vespa/intellij/schema/parser/SdParserDefinition.java index d5b76c039d0..afca26eb5cb 100644 --- a/integration/intellij/src/main/java/ai/vespa/intellij/schema/parser/SdParserDefinition.java +++ b/integration/intellij/src/main/java/ai/vespa/intellij/schema/parser/SdParserDefinition.java @@ -16,7 +16,6 @@ import ai.vespa.intellij.schema.SdLanguage; import ai.vespa.intellij.schema.lexer.SdLexerAdapter; import ai.vespa.intellij.schema.psi.SdFile; import ai.vespa.intellij.schema.psi.SdTypes; -import org.jetbrains.annotations.NotNull; /** * This class is used for the extension (in plugin.xml), to make the parsing process use the plugin code. @@ -31,55 +30,46 @@ public class SdParserDefinition implements ParserDefinition { public static final IFileElementType FILE = new IFileElementType(SdLanguage.INSTANCE); - @NotNull @Override public Lexer createLexer(Project project) { return new SdLexerAdapter(); } - @NotNull @Override public PsiParser createParser(final Project project) { return new SdParser(); } - @NotNull @Override public TokenSet getWhitespaceTokens() { return WHITE_SPACES; } - @NotNull @Override public TokenSet getCommentTokens() { return COMMENTS; } - @NotNull @Override public TokenSet getStringLiteralElements() { return STRINGS; } - @NotNull @Override public IFileElementType getFileNodeType() { return FILE; } - @NotNull @Override - public PsiFile createFile(@NotNull FileViewProvider viewProvider) { + public PsiFile createFile(FileViewProvider viewProvider) { return new SdFile(viewProvider); } - @NotNull @Override public SpaceRequirements spaceExistenceTypeBetweenTokens(ASTNode left, ASTNode right) { return SpaceRequirements.MAY; } - @NotNull @Override public PsiElement createElement(ASTNode node) { return SdTypes.Factory.createElement(node); diff --git a/integration/intellij/src/main/java/ai/vespa/intellij/schema/psi/SdElementDescriptionProvider.java b/integration/intellij/src/main/java/ai/vespa/intellij/schema/psi/SdElementDescriptionProvider.java index 7cd2d7f9331..028a034c08f 100644 --- a/integration/intellij/src/main/java/ai/vespa/intellij/schema/psi/SdElementDescriptionProvider.java +++ b/integration/intellij/src/main/java/ai/vespa/intellij/schema/psi/SdElementDescriptionProvider.java @@ -5,8 +5,6 @@ import com.intellij.psi.ElementDescriptionLocation; import com.intellij.psi.ElementDescriptionProvider; import com.intellij.psi.PsiElement; import ai.vespa.intellij.schema.psi.impl.SdNamedElementImpl; -import org.jetbrains.annotations.NotNull; -import org.jetbrains.annotations.Nullable; /** * This class is used for the extension (in plugin.xml), to enable "find Usages" window take the element description from @@ -22,9 +20,8 @@ public class SdElementDescriptionProvider implements ElementDescriptionProvider * @param psiElement the element to describe * @return a string with the description to write in the headline */ - @Nullable @Override - public String getElementDescription(@NotNull PsiElement psiElement, @NotNull ElementDescriptionLocation elementDescriptionLocation) { + public String getElementDescription(PsiElement psiElement, ElementDescriptionLocation elementDescriptionLocation) { if (psiElement instanceof SdDeclaration) { return ((SdNamedElementImpl) psiElement).getTypeName(); } else { diff --git a/integration/intellij/src/main/java/ai/vespa/intellij/schema/psi/SdElementType.java b/integration/intellij/src/main/java/ai/vespa/intellij/schema/psi/SdElementType.java index e8154bce8ad..2f044168a0a 100644 --- a/integration/intellij/src/main/java/ai/vespa/intellij/schema/psi/SdElementType.java +++ b/integration/intellij/src/main/java/ai/vespa/intellij/schema/psi/SdElementType.java @@ -3,8 +3,6 @@ package ai.vespa.intellij.schema.psi; import com.intellij.psi.tree.IElementType; import ai.vespa.intellij.schema.SdLanguage; -import org.jetbrains.annotations.NonNls; -import org.jetbrains.annotations.NotNull; /** * An SdElementType. @@ -13,7 +11,7 @@ import org.jetbrains.annotations.NotNull; */ public class SdElementType extends IElementType { - public SdElementType(@NotNull @NonNls String debugName) { + public SdElementType(String debugName) { super(debugName, SdLanguage.INSTANCE); } diff --git a/integration/intellij/src/main/java/ai/vespa/intellij/schema/psi/SdFile.java b/integration/intellij/src/main/java/ai/vespa/intellij/schema/psi/SdFile.java index 1260883c96f..fae8d7aa7e4 100644 --- a/integration/intellij/src/main/java/ai/vespa/intellij/schema/psi/SdFile.java +++ b/integration/intellij/src/main/java/ai/vespa/intellij/schema/psi/SdFile.java @@ -6,7 +6,6 @@ import com.intellij.openapi.fileTypes.FileType; import com.intellij.psi.FileViewProvider; import ai.vespa.intellij.schema.SdFileType; import ai.vespa.intellij.schema.SdLanguage; -import org.jetbrains.annotations.NotNull; /** * An SD file. @@ -15,11 +14,10 @@ import org.jetbrains.annotations.NotNull; */ public class SdFile extends PsiFileBase { - public SdFile(@NotNull FileViewProvider viewProvider) { + public SdFile(FileViewProvider viewProvider) { super(viewProvider, SdLanguage.INSTANCE); } - @NotNull @Override public FileType getFileType() { return SdFileType.INSTANCE; diff --git a/integration/intellij/src/main/java/ai/vespa/intellij/schema/psi/SdTokenType.java b/integration/intellij/src/main/java/ai/vespa/intellij/schema/psi/SdTokenType.java index e42906530b0..f68fcf1ce59 100644 --- a/integration/intellij/src/main/java/ai/vespa/intellij/schema/psi/SdTokenType.java +++ b/integration/intellij/src/main/java/ai/vespa/intellij/schema/psi/SdTokenType.java @@ -3,8 +3,6 @@ package ai.vespa.intellij.schema.psi; import com.intellij.psi.tree.IElementType; import ai.vespa.intellij.schema.SdLanguage; -import org.jetbrains.annotations.NonNls; -import org.jetbrains.annotations.NotNull; /** * An SdTokenType. @@ -13,7 +11,7 @@ import org.jetbrains.annotations.NotNull; */ public class SdTokenType extends IElementType { - public SdTokenType(@NotNull @NonNls String debugName) { + public SdTokenType(String debugName) { super(debugName, SdLanguage.INSTANCE); } diff --git a/integration/intellij/src/main/java/ai/vespa/intellij/schema/psi/impl/SdFirstPhaseDefinitionMixin.java b/integration/intellij/src/main/java/ai/vespa/intellij/schema/psi/impl/SdFirstPhaseDefinitionMixin.java index 43c02ea4fcf..e02f5d13779 100644 --- a/integration/intellij/src/main/java/ai/vespa/intellij/schema/psi/impl/SdFirstPhaseDefinitionMixin.java +++ b/integration/intellij/src/main/java/ai/vespa/intellij/schema/psi/impl/SdFirstPhaseDefinitionMixin.java @@ -7,8 +7,6 @@ import com.intellij.navigation.ItemPresentation; import com.intellij.psi.util.PsiTreeUtil; import ai.vespa.intellij.schema.SdIcons; import ai.vespa.intellij.schema.psi.SdRankProfileDefinition; -import org.jetbrains.annotations.NotNull; -import org.jetbrains.annotations.Nullable; import javax.swing.Icon; @@ -20,11 +18,10 @@ import javax.swing.Icon; */ public class SdFirstPhaseDefinitionMixin extends ASTWrapperPsiElement { - public SdFirstPhaseDefinitionMixin(@NotNull ASTNode node) { + public SdFirstPhaseDefinitionMixin(ASTNode node) { super(node); } - @NotNull public String getName() { SdRankProfileDefinition rankProfile = PsiTreeUtil.getParentOfType(this, SdRankProfileDefinition.class); if (rankProfile == null) { @@ -45,7 +42,6 @@ public class SdFirstPhaseDefinitionMixin extends ASTWrapperPsiElement { return "first-phase of " + rankProfile.getName(); } - @Nullable @Override public String getLocationString() { return element.getContainingFile() != null ? element.getContainingFile().getName() : null; diff --git a/integration/intellij/src/main/java/ai/vespa/intellij/schema/psi/impl/SdIdentifierMixin.java b/integration/intellij/src/main/java/ai/vespa/intellij/schema/psi/impl/SdIdentifierMixin.java index 897217c082c..e7ea4bbcb59 100644 --- a/integration/intellij/src/main/java/ai/vespa/intellij/schema/psi/impl/SdIdentifierMixin.java +++ b/integration/intellij/src/main/java/ai/vespa/intellij/schema/psi/impl/SdIdentifierMixin.java @@ -7,7 +7,6 @@ import com.intellij.psi.PsiNamedElement; import ai.vespa.intellij.schema.psi.SdElementFactory; import ai.vespa.intellij.schema.psi.SdIdentifier; import ai.vespa.intellij.schema.psi.SdIdentifierVal; -import org.jetbrains.annotations.NotNull; /** * This abstract class is used for methods' implementations for SdIdentifier. Connected with "mixin" to IdentifierVal and @@ -17,18 +16,16 @@ import org.jetbrains.annotations.NotNull; */ public abstract class SdIdentifierMixin extends SdIdentifierMixinImpl implements PsiNamedElement { - public SdIdentifierMixin(@NotNull ASTNode node) { + public SdIdentifierMixin(ASTNode node) { super(node); } - @NotNull public String getName() { // IMPORTANT: Convert embedded escaped spaces to simple spaces return this.getText().replaceAll("\\\\ ", " "); } - @NotNull - public PsiElement setName(@NotNull String newName) { + public PsiElement setName(String newName) { ASTNode node = this.getNode().getFirstChildNode(); if (node != null) { SdIdentifier elementName; diff --git a/integration/intellij/src/main/java/ai/vespa/intellij/schema/psi/impl/SdIdentifierMixinImpl.java b/integration/intellij/src/main/java/ai/vespa/intellij/schema/psi/impl/SdIdentifierMixinImpl.java index 3f69952d0b6..f557ae2618b 100644 --- a/integration/intellij/src/main/java/ai/vespa/intellij/schema/psi/impl/SdIdentifierMixinImpl.java +++ b/integration/intellij/src/main/java/ai/vespa/intellij/schema/psi/impl/SdIdentifierMixinImpl.java @@ -7,7 +7,6 @@ import com.intellij.openapi.util.TextRange; import com.intellij.psi.PsiReference; import ai.vespa.intellij.schema.SdReference; import ai.vespa.intellij.schema.psi.SdIdentifier; -import org.jetbrains.annotations.NotNull; /** * This class is used for methods' implementations for SdIdentifier. The abstract class SdIdentifierMixin extents it. @@ -16,11 +15,10 @@ import org.jetbrains.annotations.NotNull; */ public class SdIdentifierMixinImpl extends ASTWrapperPsiElement implements SdIdentifier { - public SdIdentifierMixinImpl(@NotNull ASTNode node) { + public SdIdentifierMixinImpl(ASTNode node) { super(node); } - @NotNull public PsiReference getReference() { return new SdReference(this, new TextRange(0, getNode().getText().length())); } diff --git a/integration/intellij/src/main/java/ai/vespa/intellij/schema/psi/impl/SdNamedElementImpl.java b/integration/intellij/src/main/java/ai/vespa/intellij/schema/psi/impl/SdNamedElementImpl.java index ee328f43075..04a8b981fb9 100644 --- a/integration/intellij/src/main/java/ai/vespa/intellij/schema/psi/impl/SdNamedElementImpl.java +++ b/integration/intellij/src/main/java/ai/vespa/intellij/schema/psi/impl/SdNamedElementImpl.java @@ -30,8 +30,6 @@ import ai.vespa.intellij.schema.psi.SdSchemaAnnotationDefinition; import ai.vespa.intellij.schema.psi.SdSchemaFieldDefinition; import ai.vespa.intellij.schema.psi.SdStructFieldDefinition; import ai.vespa.intellij.schema.psi.SdTypes; -import org.jetbrains.annotations.NotNull; -import org.jetbrains.annotations.Nullable; import javax.swing.Icon; @@ -42,11 +40,10 @@ import javax.swing.Icon; */ public abstract class SdNamedElementImpl extends ASTWrapperPsiElement implements SdNamedElement { - public SdNamedElementImpl(@NotNull ASTNode node) { + public SdNamedElementImpl(ASTNode node) { super(node); } - @NotNull public String getName() { ASTNode node; if (this instanceof SdImportFieldDefinition) { @@ -106,8 +103,7 @@ public abstract class SdNamedElementImpl extends ASTWrapperPsiElement implements } } - @NotNull - public PsiElement setName(@NotNull String newName) { + public PsiElement setName(String newName) { ASTNode node; if (this instanceof SdImportFieldDefinition) { ASTNode asNode = this.getNode().findChildByType(SdTypes.AS); @@ -144,7 +140,6 @@ public abstract class SdNamedElementImpl extends ASTWrapperPsiElement implements public ItemPresentation getPresentation() { final SdNamedElement element = this; return new ItemPresentation() { - @Nullable @Override public String getPresentableText() { if (element instanceof SdFunctionDefinition) { @@ -160,13 +155,11 @@ public abstract class SdNamedElementImpl extends ASTWrapperPsiElement implements return element.getName(); } - @Nullable @Override public String getLocationString() { return element.getContainingFile() != null ? element.getContainingFile().getName() : null; } - @Nullable @Override public Icon getIcon(boolean unused) { if (element instanceof SdSchemaFieldDefinition || element instanceof SdDocumentFieldDefinition || diff --git a/integration/intellij/src/main/java/ai/vespa/intellij/schema/psi/impl/SdSummaryDefinitionMixin.java b/integration/intellij/src/main/java/ai/vespa/intellij/schema/psi/impl/SdSummaryDefinitionMixin.java index 71d93641939..8c7167d6e50 100644 --- a/integration/intellij/src/main/java/ai/vespa/intellij/schema/psi/impl/SdSummaryDefinitionMixin.java +++ b/integration/intellij/src/main/java/ai/vespa/intellij/schema/psi/impl/SdSummaryDefinitionMixin.java @@ -6,8 +6,6 @@ import com.intellij.lang.ASTNode; import com.intellij.navigation.ItemPresentation; import ai.vespa.intellij.schema.SdIcons; import ai.vespa.intellij.schema.psi.SdTypes; -import org.jetbrains.annotations.NotNull; -import org.jetbrains.annotations.Nullable; import javax.swing.Icon; @@ -19,11 +17,10 @@ import javax.swing.Icon; */ public abstract class SdSummaryDefinitionMixin extends ASTWrapperPsiElement { - public SdSummaryDefinitionMixin(@NotNull ASTNode node) { + public SdSummaryDefinitionMixin(ASTNode node) { super(node); } - @NotNull public String getName() { ASTNode node; node = this.getNode().findChildByType(SdTypes.IDENTIFIER_WITH_DASH_VAL); @@ -44,7 +41,6 @@ public abstract class SdSummaryDefinitionMixin extends ASTWrapperPsiElement { return getName(); } - @Nullable @Override public String getLocationString() { return element.getContainingFile() != null ? element.getContainingFile().getName() : null; diff --git a/integration/intellij/src/main/java/ai/vespa/intellij/schema/structure/SdStructureViewElement.java b/integration/intellij/src/main/java/ai/vespa/intellij/schema/structure/SdStructureViewElement.java index 76662c5cd31..10384c2a34c 100644 --- a/integration/intellij/src/main/java/ai/vespa/intellij/schema/structure/SdStructureViewElement.java +++ b/integration/intellij/src/main/java/ai/vespa/intellij/schema/structure/SdStructureViewElement.java @@ -16,7 +16,6 @@ import ai.vespa.intellij.schema.psi.SdDocumentSummaryDefinition; import ai.vespa.intellij.schema.psi.SdFile; import ai.vespa.intellij.schema.psi.SdRankProfileDefinition; import ai.vespa.intellij.schema.psi.SdSchemaAnnotationDefinition; -import org.jetbrains.annotations.NotNull; import java.util.ArrayList; import java.util.List; @@ -54,14 +53,12 @@ public class SdStructureViewElement implements StructureViewTreeElement, Sortabl return myElement.canNavigateToSource(); } - @NotNull @Override public String getAlphaSortKey() { String name = myElement.getName(); return name != null ? name : ""; } - @NotNull @Override public ItemPresentation getPresentation() { ItemPresentation presentation = myElement.getPresentation(); @@ -69,7 +66,7 @@ public class SdStructureViewElement implements StructureViewTreeElement, Sortabl } @Override - public TreeElement @NotNull [] getChildren() { + public TreeElement[] getChildren() { List<PsiElement> children = new ArrayList<>(); if (myElement instanceof SdFile) { children = SdUtil.findSchemaChildren(myElement); diff --git a/integration/intellij/src/main/java/ai/vespa/intellij/schema/structure/SdStructureViewFactory.java b/integration/intellij/src/main/java/ai/vespa/intellij/schema/structure/SdStructureViewFactory.java index 3d7b3a1a275..9e4fd596a5a 100644 --- a/integration/intellij/src/main/java/ai/vespa/intellij/schema/structure/SdStructureViewFactory.java +++ b/integration/intellij/src/main/java/ai/vespa/intellij/schema/structure/SdStructureViewFactory.java @@ -7,8 +7,6 @@ import com.intellij.ide.structureView.TreeBasedStructureViewBuilder; import com.intellij.lang.PsiStructureViewFactory; import com.intellij.openapi.editor.Editor; import com.intellij.psi.PsiFile; -import org.jetbrains.annotations.NotNull; -import org.jetbrains.annotations.Nullable; /** * This class is used for the extension (in plugin.xml) to the class SdStructureViewModel. It make the IDE use our @@ -18,13 +16,10 @@ import org.jetbrains.annotations.Nullable; */ public class SdStructureViewFactory implements PsiStructureViewFactory { - @Nullable - @Override - public StructureViewBuilder getStructureViewBuilder(@NotNull final PsiFile psiFile) { + public StructureViewBuilder getStructureViewBuilder(PsiFile psiFile) { return new TreeBasedStructureViewBuilder() { - @NotNull @Override - public StructureViewModel createStructureViewModel(@Nullable Editor editor) { + public StructureViewModel createStructureViewModel(Editor editor) { return new SdStructureViewModel(psiFile); } }; diff --git a/integration/intellij/src/main/java/ai/vespa/intellij/schema/structure/SdStructureViewModel.java b/integration/intellij/src/main/java/ai/vespa/intellij/schema/structure/SdStructureViewModel.java index 1f1c271dcf6..7282c74288f 100644 --- a/integration/intellij/src/main/java/ai/vespa/intellij/schema/structure/SdStructureViewModel.java +++ b/integration/intellij/src/main/java/ai/vespa/intellij/schema/structure/SdStructureViewModel.java @@ -6,7 +6,6 @@ import com.intellij.ide.structureView.StructureViewModelBase; import com.intellij.ide.structureView.StructureViewTreeElement; import com.intellij.ide.util.treeView.smartTree.Sorter; import com.intellij.psi.PsiFile; -import org.jetbrains.annotations.NotNull; /** * This class represents the "Structure View" window. @@ -14,16 +13,15 @@ import org.jetbrains.annotations.NotNull; * @author Shahar Ariel */ public class SdStructureViewModel extends StructureViewModelBase implements StructureViewModel.ElementInfoProvider { + public SdStructureViewModel(PsiFile psiFile) { super(psiFile, new SdStructureViewElement(psiFile)); } - - - public Sorter @NotNull [] getSorters() { + + public Sorter[] getSorters() { return new Sorter[]{Sorter.ALPHA_SORTER}; } - @Override public boolean isAlwaysShowsPlus(StructureViewTreeElement element) { return false; diff --git a/jdisc-security-filters/src/test/java/com/yahoo/jdisc/http/filter/security/athenz/AthenzAuthorizationFilterTest.java b/jdisc-security-filters/src/test/java/com/yahoo/jdisc/http/filter/security/athenz/AthenzAuthorizationFilterTest.java index 8be552c1da0..63223f3c221 100644 --- a/jdisc-security-filters/src/test/java/com/yahoo/jdisc/http/filter/security/athenz/AthenzAuthorizationFilterTest.java +++ b/jdisc-security-filters/src/test/java/com/yahoo/jdisc/http/filter/security/athenz/AthenzAuthorizationFilterTest.java @@ -39,14 +39,10 @@ import static com.yahoo.jdisc.http.filter.security.athenz.AthenzAuthorizationFil import static com.yahoo.security.SignatureAlgorithm.SHA256_WITH_ECDSA; import static com.yahoo.security.SubjectAlternativeName.Type.RFC822_NAME; import static com.yahoo.vespa.athenz.zpe.AuthorizationResult.Type; -import static org.hamcrest.CoreMatchers.containsString; -import static org.hamcrest.CoreMatchers.equalTo; -import static org.hamcrest.CoreMatchers.hasItem; -import static org.hamcrest.CoreMatchers.notNullValue; -import static org.hamcrest.CoreMatchers.nullValue; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNotNull; -import static org.junit.Assert.assertThat; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertTrue; import static org.mockito.ArgumentMatchers.any; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.never; @@ -240,13 +236,13 @@ public class AthenzAuthorizationFilterTest { } private void assertMetrics(MetricMock metric, String metricName, Map<String, String> dimensions) { - assertThat(metric.addInvocations.keySet(), hasItem(metricName)); + assertTrue(metric.addInvocations.keySet().contains(metricName)); SimpleMetricContext metricContext = metric.addInvocations.get(metricName); assertNotNull("Metric not found " + metricName, metricName); for (Map.Entry<String, String> entry : dimensions.entrySet()) { String dimensionName = entry.getKey(); String expected = entry.getValue(); - assertThat(metricContext.dimensions.keySet(), hasItem(dimensionName)); + assertTrue(metricContext.dimensions.keySet().contains(dimensionName)); assertEquals(expected, metricContext.dimensions.get(dimensionName)); } } @@ -307,8 +303,8 @@ public class AthenzAuthorizationFilterTest { private static void assertStatusCode(MockResponseHandler responseHandler, int statusCode) { Response response = responseHandler.getResponse(); - assertThat(response, notNullValue()); - assertThat(response.getStatus(), equalTo(statusCode)); + assertNotNull(response); + assertEquals(statusCode, response.getStatus()); } private static void assertMatchedCredentialType(DiscFilterRequest request, EnabledCredentials.Enum expectedType) { @@ -316,7 +312,7 @@ public class AthenzAuthorizationFilterTest { } private static void assertRequestNotFiltered(MockResponseHandler responseHandler) { - assertThat(responseHandler.getResponse(), nullValue()); + assertNull(responseHandler.getResponse()); } private static void assertMatchedRole(DiscFilterRequest request, AthenzRole role) { @@ -325,9 +321,9 @@ public class AthenzAuthorizationFilterTest { private static void assertErrorMessage(MockResponseHandler responseHandler, String errorMessage) { Response response = responseHandler.getResponse(); - assertThat(response, notNullValue()); + assertNotNull(response); String content = responseHandler.readAll(); - assertThat(content, containsString(errorMessage)); + assertTrue(content.contains(errorMessage)); } private static class AllowingZpe implements Zpe { diff --git a/jdisc-security-filters/src/test/java/com/yahoo/jdisc/http/filter/security/athenz/AthenzPrincipalFilterTest.java b/jdisc-security-filters/src/test/java/com/yahoo/jdisc/http/filter/security/athenz/AthenzPrincipalFilterTest.java index 23b1b7c569e..d670c969e5c 100644 --- a/jdisc-security-filters/src/test/java/com/yahoo/jdisc/http/filter/security/athenz/AthenzPrincipalFilterTest.java +++ b/jdisc-security-filters/src/test/java/com/yahoo/jdisc/http/filter/security/athenz/AthenzPrincipalFilterTest.java @@ -31,11 +31,10 @@ import static com.yahoo.security.SignatureAlgorithm.SHA256_WITH_ECDSA; import static java.util.Collections.emptyList; import static java.util.Collections.singletonList; import static java.util.stream.Collectors.joining; -import static org.hamcrest.CoreMatchers.containsString; -import static org.hamcrest.CoreMatchers.equalTo; -import static org.hamcrest.CoreMatchers.notNullValue; -import static org.hamcrest.CoreMatchers.nullValue; -import static org.junit.Assert.assertThat; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertTrue; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; @@ -91,7 +90,7 @@ public class AthenzPrincipalFilterTest { AthenzPrincipalFilter filter = createFilter(true); filter.filter(request, responseHandler); - assertThat(responseHandler.response, nullValue()); + assertNull(responseHandler.response); } private DiscFilterRequest createRequestMock() { @@ -103,9 +102,9 @@ public class AthenzPrincipalFilterTest { } private static void assertUnauthorized(DiscFilterRequest request, ResponseHandlerMock responseHandler, String expectedMessageSubstring) { - assertThat(responseHandler.response, notNullValue()); - assertThat(responseHandler.response.getStatus(), equalTo(UNAUTHORIZED)); - assertThat(responseHandler.getResponseContent(), containsString(expectedMessageSubstring)); + assertNotNull(responseHandler.response); + assertEquals(UNAUTHORIZED, responseHandler.response.getStatus()); + assertTrue(responseHandler.getResponseContent().contains(expectedMessageSubstring)); verify(request).setAttribute(AthenzPrincipalFilter.RESULT_ERROR_CODE_ATTRIBUTE, UNAUTHORIZED); } diff --git a/jdisc-security-filters/src/test/java/com/yahoo/jdisc/http/filter/security/base/JsonSecurityRequestFilterBaseTest.java b/jdisc-security-filters/src/test/java/com/yahoo/jdisc/http/filter/security/base/JsonSecurityRequestFilterBaseTest.java index a2746e84e1c..0f5512a6ec5 100644 --- a/jdisc-security-filters/src/test/java/com/yahoo/jdisc/http/filter/security/base/JsonSecurityRequestFilterBaseTest.java +++ b/jdisc-security-filters/src/test/java/com/yahoo/jdisc/http/filter/security/base/JsonSecurityRequestFilterBaseTest.java @@ -11,9 +11,8 @@ import org.junit.Test; import java.io.IOException; import java.util.Optional; -import static org.hamcrest.CoreMatchers.equalTo; -import static org.hamcrest.CoreMatchers.notNullValue; -import static org.junit.Assert.assertThat; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; import static org.mockito.Mockito.mock; /** @@ -34,12 +33,12 @@ public class JsonSecurityRequestFilterBaseTest { filter.filter(request, responseHandler); Response response = responseHandler.getResponse(); - assertThat(response, notNullValue()); - assertThat(response.getStatus(), equalTo(statusCode)); + assertNotNull(response); + assertEquals(statusCode, response.getStatus()); JsonNode jsonNode = mapper.readTree(responseHandler.readAll()); - assertThat(jsonNode.get("message").asText(), equalTo(message)); - assertThat(jsonNode.get("code").asInt(), equalTo(statusCode)); + assertEquals(message, jsonNode.get("message").asText()); + assertEquals(statusCode, jsonNode.get("code").asInt()); } private static class SimpleSecurityRequestFilter extends JsonSecurityRequestFilterBase { diff --git a/jrt/src/com/yahoo/jrt/TransportThread.java b/jrt/src/com/yahoo/jrt/TransportThread.java index 4f951ad8259..a3f1773c814 100644 --- a/jrt/src/com/yahoo/jrt/TransportThread.java +++ b/jrt/src/com/yahoo/jrt/TransportThread.java @@ -269,6 +269,23 @@ public class TransportThread { } /** + * Wake up this transport thread explicitly. + **/ + public void wakeup() { + selector.wakeup(); + } + + /** + * Wake up this transport thread explicitly, but only if the + * calling thread is not the transport thread itself. + **/ + public void wakeup_if_not_self() { + if (Thread.currentThread() != thread) { + wakeup(); + } + } + + /** * Synchronize with the transport thread. This method will block * until all commands issued before this method was invoked has * completed. If the transport thread has been shut down (or is in diff --git a/jrt/src/com/yahoo/jrt/slobrok/api/Mirror.java b/jrt/src/com/yahoo/jrt/slobrok/api/Mirror.java index 51a3c6b6146..63de287486e 100644 --- a/jrt/src/com/yahoo/jrt/slobrok/api/Mirror.java +++ b/jrt/src/com/yahoo/jrt/slobrok/api/Mirror.java @@ -67,9 +67,10 @@ public class Mirror implements IMirror { public void handleRequestDone(Request req) { requestDone = true; updateTask.scheduleNow(); + transportThread.wakeup_if_not_self(); } }; - updateTask.scheduleNow(); + startFetchRequest(); } /** @@ -174,9 +175,11 @@ public class Mirror implements IMirror { if (requestDone) { handleUpdate(); requestDone = false; - return; } + startFetchRequest(); + } + private void startFetchRequest() { if (target != null && ! slobroks.contains(currSlobrok)) { log.log(Level.INFO, "location broker "+currSlobrok+" removed, will disconnect and use one of: "+slobroks); target.close(); @@ -204,7 +207,7 @@ public class Mirror implements IMirror { req.parameters().add(new Int32Value(5000)); // mstimeout target.invokeAsync(req, 40.0, reqWait); } - + private void handleUpdate() { if (!req.checkReturnTypes("iSSSi") || (req.returnValues().get(2).count() != @@ -215,8 +218,7 @@ public class Mirror implements IMirror { " (error code " + req.errorCode() + ")"); } target.close(); - target = null; - updateTask.scheduleNow(); // try next slobrok + target = null; // try next slobrok return; } Values answer = req.returnValues(); @@ -235,7 +237,6 @@ public class Mirror implements IMirror { Entry[] newSpecs; if (diffFromGeneration == 0) { newSpecs = new Entry[numNames]; - for (int idx = 0; idx < numNames; idx++) { newSpecs[idx] = new Entry(n[idx], s[idx]); } @@ -270,7 +271,6 @@ public class Mirror implements IMirror { updates = u; } backOff.reset(); - updateTask.schedule(0.1); // be nice } /** diff --git a/jrt/src/com/yahoo/jrt/slobrok/api/Register.java b/jrt/src/com/yahoo/jrt/slobrok/api/Register.java index a02527eee25..14afea396bf 100644 --- a/jrt/src/com/yahoo/jrt/slobrok/api/Register.java +++ b/jrt/src/com/yahoo/jrt/slobrok/api/Register.java @@ -87,6 +87,7 @@ public class Register { public void handleRequestDone(Request req) { reqDone = true; updateTask.scheduleNow(); + transportThread.wakeup_if_not_self(); } }; m_list = new Method("slobrok.callback.listNamesServed", @@ -159,6 +160,7 @@ public class Register { pending.add(name); discard(unreg, name); updateTask.scheduleNow(); + transportThread.wakeup(); } /** @@ -171,6 +173,7 @@ public class Register { discard(pending, name); unreg.add(name); updateTask.scheduleNow(); + transportThread.wakeup(); } /** diff --git a/linguistics-components/abi-spec.json b/linguistics-components/abi-spec.json index f1deac67dc2..39666fd93a3 100644 --- a/linguistics-components/abi-spec.json +++ b/linguistics-components/abi-spec.json @@ -153,10 +153,12 @@ "superClass": "java.lang.Object", "interfaces": [], "attributes": [ - "public" + "public", + "final" ], "methods": [ "public void <init>()", + "public void <init>(java.lang.String)", "public void addModel(com.yahoo.language.Language, java.nio.file.Path)", "public com.yahoo.language.sentencepiece.SentencePieceEmbedder$Builder addDefaultModel(java.nio.file.Path)", "public java.util.Map getModels()", @@ -186,5 +188,139 @@ "public java.lang.String normalize(java.lang.String)" ], "fields": [] + }, + "com.yahoo.language.wordpiece.WordPieceConfig$Builder": { + "superClass": "java.lang.Object", + "interfaces": [ + "com.yahoo.config.ConfigInstance$Builder" + ], + "attributes": [ + "public" + ], + "methods": [ + "public void <init>()", + "public void <init>(com.yahoo.language.wordpiece.WordPieceConfig)", + "public com.yahoo.language.wordpiece.WordPieceConfig$Builder subwordPrefix(java.lang.String)", + "public com.yahoo.language.wordpiece.WordPieceConfig$Builder model(com.yahoo.language.wordpiece.WordPieceConfig$Model$Builder)", + "public com.yahoo.language.wordpiece.WordPieceConfig$Builder model(java.util.function.Consumer)", + "public com.yahoo.language.wordpiece.WordPieceConfig$Builder model(java.util.List)", + "public final boolean dispatchGetConfig(com.yahoo.config.ConfigInstance$Producer)", + "public final java.lang.String getDefMd5()", + "public final java.lang.String getDefName()", + "public final java.lang.String getDefNamespace()", + "public final boolean getApplyOnRestart()", + "public final void setApplyOnRestart(boolean)", + "public com.yahoo.language.wordpiece.WordPieceConfig build()" + ], + "fields": [ + "public java.util.List model" + ] + }, + "com.yahoo.language.wordpiece.WordPieceConfig$Model$Builder": { + "superClass": "java.lang.Object", + "interfaces": [ + "com.yahoo.config.ConfigBuilder" + ], + "attributes": [ + "public" + ], + "methods": [ + "public void <init>()", + "public void <init>(com.yahoo.language.wordpiece.WordPieceConfig$Model)", + "public com.yahoo.language.wordpiece.WordPieceConfig$Model$Builder language(java.lang.String)", + "public com.yahoo.language.wordpiece.WordPieceConfig$Model$Builder path(com.yahoo.config.FileReference)", + "public com.yahoo.language.wordpiece.WordPieceConfig$Model build()" + ], + "fields": [] + }, + "com.yahoo.language.wordpiece.WordPieceConfig$Model": { + "superClass": "com.yahoo.config.InnerNode", + "interfaces": [], + "attributes": [ + "public", + "final" + ], + "methods": [ + "public void <init>(com.yahoo.language.wordpiece.WordPieceConfig$Model$Builder)", + "public java.lang.String language()", + "public java.nio.file.Path path()" + ], + "fields": [] + }, + "com.yahoo.language.wordpiece.WordPieceConfig$Producer": { + "superClass": "java.lang.Object", + "interfaces": [ + "com.yahoo.config.ConfigInstance$Producer" + ], + "attributes": [ + "public", + "interface", + "abstract" + ], + "methods": [ + "public abstract void getConfig(com.yahoo.language.wordpiece.WordPieceConfig$Builder)" + ], + "fields": [] + }, + "com.yahoo.language.wordpiece.WordPieceConfig": { + "superClass": "com.yahoo.config.ConfigInstance", + "interfaces": [], + "attributes": [ + "public", + "final" + ], + "methods": [ + "public static java.lang.String getDefMd5()", + "public static java.lang.String getDefName()", + "public static java.lang.String getDefNamespace()", + "public static java.lang.String getDefVersion()", + "public void <init>(com.yahoo.language.wordpiece.WordPieceConfig$Builder)", + "public java.lang.String subwordPrefix()", + "public java.util.List model()", + "public com.yahoo.language.wordpiece.WordPieceConfig$Model model(int)" + ], + "fields": [ + "public static final java.lang.String CONFIG_DEF_MD5", + "public static final java.lang.String CONFIG_DEF_NAME", + "public static final java.lang.String CONFIG_DEF_NAMESPACE", + "public static final java.lang.String CONFIG_DEF_VERSION", + "public static final java.lang.String[] CONFIG_DEF_SCHEMA" + ] + }, + "com.yahoo.language.wordpiece.WordPieceEmbedder$Builder": { + "superClass": "java.lang.Object", + "interfaces": [], + "attributes": [ + "public", + "final" + ], + "methods": [ + "public void <init>()", + "public void <init>(java.lang.String)", + "public com.yahoo.language.wordpiece.WordPieceEmbedder$Builder setSubwordPrefix(java.lang.String)", + "public java.lang.String getSubwordPrefix()", + "public void addModel(com.yahoo.language.Language, java.nio.file.Path)", + "public com.yahoo.language.wordpiece.WordPieceEmbedder$Builder addDefaultModel(java.nio.file.Path)", + "public java.util.Map getModels()", + "public com.yahoo.language.wordpiece.WordPieceEmbedder build()" + ], + "fields": [] + }, + "com.yahoo.language.wordpiece.WordPieceEmbedder": { + "superClass": "java.lang.Object", + "interfaces": [ + "com.yahoo.language.process.Embedder", + "com.yahoo.language.process.Segmenter" + ], + "attributes": [ + "public" + ], + "methods": [ + "public void <init>(com.yahoo.language.wordpiece.WordPieceConfig)", + "public java.util.List segment(java.lang.String, com.yahoo.language.Language)", + "public java.util.List embed(java.lang.String, com.yahoo.language.process.Embedder$Context)", + "public com.yahoo.tensor.Tensor embed(java.lang.String, com.yahoo.language.process.Embedder$Context, com.yahoo.tensor.TensorType)" + ], + "fields": [] } }
\ No newline at end of file diff --git a/linguistics-components/src/main/java/com/yahoo/language/sentencepiece/SentencePieceEmbedder.java b/linguistics-components/src/main/java/com/yahoo/language/sentencepiece/SentencePieceEmbedder.java index 3afc85300d4..31964eac514 100644 --- a/linguistics-components/src/main/java/com/yahoo/language/sentencepiece/SentencePieceEmbedder.java +++ b/linguistics-components/src/main/java/com/yahoo/language/sentencepiece/SentencePieceEmbedder.java @@ -3,17 +3,18 @@ package com.yahoo.language.sentencepiece; import com.yahoo.api.annotations.Beta; import com.google.inject.Inject; +import com.yahoo.language.tools.Embed; import com.yahoo.language.Language; import com.yahoo.language.process.Embedder; import com.yahoo.language.process.Segmenter; import com.yahoo.tensor.Tensor; -import com.yahoo.tensor.TensorAddress; import com.yahoo.tensor.TensorType; +import java.io.File; import java.nio.file.Path; import java.util.ArrayList; import java.util.Collections; -import java.util.HashMap; +import java.util.EnumMap; import java.util.List; import java.util.Map; import java.util.stream.Collectors; @@ -38,7 +39,7 @@ public class SentencePieceEmbedder implements Segmenter, Embedder { } public SentencePieceEmbedder(Builder builder) { - algorithm = new SentencePieceAlgorithm(builder.collapseUnknowns, builder.getScoring()); + algorithm = new SentencePieceAlgorithm(builder.getCollapseUnknowns(), builder.getScoring()); models = builder.getModels().entrySet() .stream() @@ -94,9 +95,6 @@ public class SentencePieceEmbedder implements Segmenter, Embedder { * they were encountered in the text. If the dimension is bound and too large it will be zero padded, if too small * it will be truncated.</p> * - * <p>If the tensor type is1-d sparse this will return a tensor containing the token strings as keys and the token - * position as value.</p> - * * <p>If the tensor is any other type IllegalArgumentException is thrown.</p> * * @param rawInput the text to segment. Any sequence of BMP (Unicode-16 the True Unicode) is supported. @@ -105,40 +103,15 @@ public class SentencePieceEmbedder implements Segmenter, Embedder { */ @Override public Tensor embed(String rawInput, Embedder.Context context, TensorType type) { - if (type.dimensions().size() == 1 && type.dimensions().get(0).isIndexed()) { - // Build to a list first since we can't reverse a tensor builder - List<Integer> values = embed(rawInput, context); - - long maxSize = values.size(); - if (type.dimensions().get(0).size().isPresent()) - maxSize = Math.min(maxSize, type.dimensions().get(0).size().get()); - - Tensor.Builder builder = Tensor.Builder.of(type); - for (int i = 0; i < maxSize; i++) - builder.cell(values.get(i), i); - return builder.build(); - } - else if (type.dimensions().size() == 1 && type.dimensions().get(0).isMapped()) { - // Build to a list first since we can't reverse a tensor builder - List<String> values = segment(rawInput, context.getLanguage()); - - Tensor.Builder builder = Tensor.Builder.of(type); - for (int i = 0; i < values.size(); i++) - builder.cell(TensorAddress.ofLabels(values.get(i)), i); - return builder.build(); - } - else { - throw new IllegalArgumentException("Don't know how to embed with SentencePiece into " + type); - } + return Embed.asTensor(rawInput, this, context, type); } private <RESULTTYPE> void segment(String input, Language language, ResultBuilder<RESULTTYPE> resultBuilder) { - Model model = resolveFrom(language); - algorithm.segment(input, resultBuilder, model); + algorithm.segment(input, resultBuilder, resolveModelFrom(language)); } - private Model resolveFrom(Language language) { + private Model resolveModelFrom(Language language) { // Disregard language if there is default model if (models.size() == 1 && models.containsKey(Language.UNKNOWN)) return models.get(Language.UNKNOWN); if (models.containsKey(language)) return models.get(language); @@ -164,22 +137,24 @@ public class SentencePieceEmbedder implements Segmenter, Embedder { return b.toString(); } - public static class Builder { + public static final class Builder { - private final Map<Language, Path> models = new HashMap<>(); + private final Map<Language, Path> models = new EnumMap<>(Language.class); private boolean collapseUnknowns = true; private Scoring scoring = Scoring.fewestSegments; - public Builder() { + public Builder() {} + + public Builder(String defaultModelFile) { + addDefaultModel(new File(defaultModelFile).toPath()); } private Builder(SentencePieceConfig config) { collapseUnknowns = config.collapseUnknowns(); scoring = config.scoring() == SentencePieceConfig.Scoring.fewestSegments ? Scoring.fewestSegments : Scoring.highestScore; - for (SentencePieceConfig.Model model : config.model()) { + for (SentencePieceConfig.Model model : config.model()) addModel(Language.fromLanguageTag(model.language()), model.path()); - } } public void addModel(Language language, Path model) { diff --git a/linguistics-components/src/main/java/com/yahoo/language/tools/Embed.java b/linguistics-components/src/main/java/com/yahoo/language/tools/Embed.java new file mode 100644 index 00000000000..401347cc452 --- /dev/null +++ b/linguistics-components/src/main/java/com/yahoo/language/tools/Embed.java @@ -0,0 +1,43 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.language.tools; + +import com.yahoo.language.process.Embedder; +import com.yahoo.tensor.Tensor; +import com.yahoo.tensor.TensorType; + +import java.util.List; + +/** + * Component internal helpers for embedding + * + * @author bratseth + */ +public class Embed { + + /** + * Convenience function which embeds the given string into the given tensor type (if possible), + * using the given embedder. + */ + public static Tensor asTensor(String text, + Embedder embedder, + Embedder.Context context, + TensorType type) { + if (type.dimensions().size() == 1 && type.dimensions().get(0).isIndexed()) { + // Build to a list first since we can't reverse a tensor builder + List<Integer> values = embedder.embed(text, context); + + long maxSize = values.size(); + if (type.dimensions().get(0).size().isPresent()) + maxSize = Math.min(maxSize, type.dimensions().get(0).size().get()); + + Tensor.Builder builder = Tensor.Builder.of(type); + for (int i = 0; i < maxSize; i++) + builder.cell(values.get(i), i); + return builder.build(); + } + else { + throw new IllegalArgumentException("Don't know how to embed into " + type); + } + } + +} diff --git a/linguistics-components/src/main/java/com/yahoo/language/wordpiece/Model.java b/linguistics-components/src/main/java/com/yahoo/language/wordpiece/Model.java new file mode 100644 index 00000000000..ce996b85313 --- /dev/null +++ b/linguistics-components/src/main/java/com/yahoo/language/wordpiece/Model.java @@ -0,0 +1,114 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.language.wordpiece; + +import com.yahoo.collections.Tuple2; +import com.yahoo.language.Language; +import com.yahoo.language.process.StemMode; +import com.yahoo.language.process.Token; +import com.yahoo.language.process.Tokenizer; + +import java.io.BufferedReader; +import java.io.FileInputStream; +import java.io.IOException; +import java.io.InputStreamReader; +import java.nio.charset.StandardCharsets; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.NavigableMap; +import java.util.TreeMap; +import java.util.stream.Collectors; + +/** + * A WordPiece embedder "model" - just a vocabulary of strings with a fixed id (index). + * + * Adapted from + * https://github.com/eclipse/deeplearning4j/blob/master/deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/main/java/org/deeplearning4j/text/tokenization/tokenizer/BertWordPieceTokenizer.java + * licensed under the Apache License, Version 2.0 + * + * @author bergum + * @author bratseth + */ +class Model { + + private final String subwordPrefix; + private final Path source; + private final Language language; + private final NavigableMap<String, Integer> vocabulary; + private final Map<Integer, String> tokenId2Token; + + Model(String subwordPrefix, Language language, Path path) { + this.subwordPrefix = subwordPrefix; + this.source = path; + this.language = language; + + this.vocabulary = new TreeMap<>(Collections.reverseOrder()); + this.tokenId2Token = new HashMap<>(); + try (BufferedReader reader = new BufferedReader(new InputStreamReader(new FileInputStream(path.toFile()), + StandardCharsets.UTF_8))) { + String token; + int i = 0; + while ((token = reader.readLine()) != null) { + this.vocabulary.put(token, i); + this.tokenId2Token.put(i, token); + i++; + } + } + catch (IOException e) { + throw new IllegalArgumentException("Could not read a WordPiece model from " + path, e); + } + + } + + Language language() { return language; } + + List<Integer> embed(String text, Tokenizer tokenizer) { + List<Integer> ids = new ArrayList<>(); + text = text.toLowerCase(); + for (Token t : tokenizer.tokenize(text, language, StemMode.NONE, true)) { + String originalToken = t.getTokenString(); + String candidate = originalToken; + int count = 0; + while (candidate.length() > 0 && !candidate.equals(subwordPrefix)) { + Tuple2<String, Integer> entry = findLongestSubstring(candidate); + if (entry == null) break; + ids.add(entry.second); + candidate = subwordPrefix + candidate.substring(entry.first.length()); + if (count++ > originalToken.length()) break; + } + } + + return ids; + } + + List<String> segment(String text, Tokenizer tokenizer) { + return embed(text, tokenizer).stream().map(tokenId -> tokenId2Token.get(tokenId)).collect(Collectors.toList()); + } + + private Tuple2<String, Integer> findLongestSubstring(String candidate) { + NavigableMap<String, Integer> tailMap = this.vocabulary.tailMap(candidate, true); + if (tailMap.isEmpty()) + return null; + String longestSubstring = tailMap.firstKey(); + Integer id = tailMap.firstEntry().getValue(); + int subStringLength = Math.min(candidate.length(), longestSubstring.length()); + while (!candidate.startsWith(longestSubstring)) { + subStringLength--; + tailMap = tailMap.tailMap(candidate.substring(0, subStringLength), true); + if (tailMap.isEmpty()) + return null; + longestSubstring = tailMap.firstKey(); + id = tailMap.firstEntry().getValue(); + } + return new Tuple2<>(longestSubstring, id); + } + + @Override + public String toString() { + return "WordPiece model for " + language + ": '" + source + "'"; + } + +} diff --git a/linguistics-components/src/main/java/com/yahoo/language/wordpiece/WordPieceEmbedder.java b/linguistics-components/src/main/java/com/yahoo/language/wordpiece/WordPieceEmbedder.java new file mode 100644 index 00000000000..08de05f351a --- /dev/null +++ b/linguistics-components/src/main/java/com/yahoo/language/wordpiece/WordPieceEmbedder.java @@ -0,0 +1,147 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.language.wordpiece; + +import com.google.inject.Inject; +import com.yahoo.language.tools.Embed; +import com.yahoo.language.Language; +import com.yahoo.language.process.Embedder; +import com.yahoo.language.process.Segmenter; +import com.yahoo.language.process.Tokenizer; +import com.yahoo.language.simple.SimpleLinguistics; +import com.yahoo.tensor.Tensor; +import com.yahoo.tensor.TensorType; +import com.yahoo.language.wordpiece.WordPieceConfig; + +import java.io.File; +import java.nio.file.Path; +import java.util.EnumMap; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; + +/** + * An implementation of the WordPiece embedder, usually used with BERT models, + * see https://arxiv.org/pdf/1609.08144v2.pdf + * Text is tokenized into tokens from a configured vocabulary, + * and optionally returned as a 1-d dense tensor of token ids. + * + * @author bratseth + */ +public class WordPieceEmbedder implements Embedder, Segmenter { + + private final Map<Language, Model> models; + + private final Tokenizer tokenizer; + + @Inject + public WordPieceEmbedder(WordPieceConfig config) { + this(new Builder(config)); + } + + private WordPieceEmbedder(Builder builder) { + super(); + this.tokenizer = new SimpleLinguistics().getTokenizer(); // always just split on spaces etc. and lowercase + models = builder.getModels().entrySet() + .stream() + .map(e -> new Model(builder.getSubwordPrefix(), e.getKey(), e.getValue())) + .collect(Collectors.toUnmodifiableMap(m -> m.language(), m -> m)); + if (models.isEmpty()) + throw new IllegalArgumentException("WordPieceEmbedder requires at least one model configured"); + } + + /** + * Segments the given text into token segments from the WordPiece vocabulary. + * + * @param text the text to segment. The text should be of a language using space-separated words. + * @return the list of zero or more token ids resulting from segmenting the input text + */ + @Override + public List<String> segment(String text, Language language) { + return resolveModelFrom(language).segment(text, tokenizer); + } + + /** + * Segments the given text into token segments from the WordPiece vocabulary and returns the token ids. + * + * @param text the text to segment. The text should be of a language using space-separated words. + * @param context the context which specifies the language used to select a model + * @return the list of zero or more token ids resulting from segmenting the input text + */ + @Override + public List<Integer> embed(String text, Context context) { + return resolveModelFrom(context.getLanguage()).embed(text, tokenizer); + } + + /** + * <p>Embeds text into a tensor.</p> + * + * <p>If the tensor type is indexed 1-d (bound or unbound) this will return a tensor containing the token ids in the order + * they were encountered in the text. If the dimension is bound and too large it will be zero padded, if too small + * it will be truncated.</p> + * + * <p>If the tensor is any other type IllegalArgumentException is thrown.</p> + * + * @param text the text to segment. The text should be of a language using space-separated words. + * @param context the context which specifies the language used to select a model + * @return the list of zero or more token ids resulting from segmenting the input text + */ + @Override + public Tensor embed(String text, Context context, TensorType type) { + return Embed.asTensor(text, this, context, type); + } + + private Model resolveModelFrom(Language language) { + // Disregard language if there is default model + if (models.size() == 1 && models.containsKey(Language.UNKNOWN)) return models.get(Language.UNKNOWN); + if (models.containsKey(language)) return models.get(language); + throw new IllegalArgumentException("No WordPiece model for language " + language + " is configured"); + } + + public static final class Builder { + + private String subwordPrefix = "##"; + private final Map<Language, Path> models = new EnumMap<>(Language.class); + + public Builder() {} + + public Builder(String defaultModelFile) { + addDefaultModel(new File(defaultModelFile).toPath()); + } + + private Builder(WordPieceConfig config) { + this.subwordPrefix = config.subwordPrefix(); + for (WordPieceConfig.Model model : config.model()) + addModel(Language.fromLanguageTag(model.language()), model.path()); + } + + public Builder setSubwordPrefix(String prefix) { + this.subwordPrefix = subwordPrefix; + return this; + } + + public String getSubwordPrefix() { return subwordPrefix; } + + public void addModel(Language language, Path model) { + models.put(language, model); + } + + /** + * Adds the model that will be used if the language is unknown, OR only one model is specified. + * The same as addModel(Language.UNKNOWN, model). + */ + public WordPieceEmbedder.Builder addDefaultModel(Path model) { + addModel(Language.UNKNOWN, model); + return this; + } + + public Map<Language, Path> getModels() { return models; } + + public WordPieceEmbedder build() { + if (models.isEmpty()) throw new IllegalStateException("At least one model must be supplied"); + return new WordPieceEmbedder(this); + } + + } + +} + diff --git a/linguistics-components/src/main/java/com/yahoo/language/wordpiece/package-info.java b/linguistics-components/src/main/java/com/yahoo/language/wordpiece/package-info.java new file mode 100644 index 00000000000..0bbb6f001f5 --- /dev/null +++ b/linguistics-components/src/main/java/com/yahoo/language/wordpiece/package-info.java @@ -0,0 +1,7 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +@ExportPackage +@PublicApi +package com.yahoo.language.wordpiece; + +import com.yahoo.api.annotations.PublicApi; +import com.yahoo.osgi.annotation.ExportPackage; diff --git a/linguistics-components/src/main/resources/configdefinitions/language.wordpiece.word-piece.def b/linguistics-components/src/main/resources/configdefinitions/language.wordpiece.word-piece.def new file mode 100644 index 00000000000..08592250eb5 --- /dev/null +++ b/linguistics-components/src/main/resources/configdefinitions/language.wordpiece.word-piece.def @@ -0,0 +1,14 @@ +# Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +# Configures com.yahoo.language.wordpiece.WordPieceEmbedder + +namespace=language.wordpiece + +# The prefix to prepend to subword tokens +subwordPrefix string default="##" + +# The language a model is for, one of the language tags in com.yahoo.language.Language. +# Use "unknown" for a model to be used for any language (i.e by default). +model[].language string +# The path to the model relative to the application package root +model[].path path diff --git a/linguistics-components/src/test/java/com/yahoo/language/sentencepiece/SentencePieceConfigurationTest.java b/linguistics-components/src/test/java/com/yahoo/language/sentencepiece/SentencePieceConfigurationTest.java index 1ed2271f774..19cb2267655 100644 --- a/linguistics-components/src/test/java/com/yahoo/language/sentencepiece/SentencePieceConfigurationTest.java +++ b/linguistics-components/src/test/java/com/yahoo/language/sentencepiece/SentencePieceConfigurationTest.java @@ -4,6 +4,7 @@ package com.yahoo.language.sentencepiece; import com.yahoo.config.FileReference; import com.yahoo.language.Language; +import com.yahoo.language.tools.EmbedderTester; import org.junit.Test; /** @@ -15,7 +16,7 @@ public class SentencePieceConfigurationTest { public void testEnglishTokenization() { var b = new SentencePieceConfig.Builder(); addModel("unknown", "src/test/models/sentencepiece/en.wiki.bpe.vs10000.model", b); - var tester = new SentencePieceTester(new SentencePieceEmbedder(b.build())); + var tester = new EmbedderTester(new SentencePieceEmbedder(b.build())); tester.assertSegmented("this is another sentence", "▁this", "▁is", "▁another", "▁sentence"); tester.assertSegmented("KHJKJHHKJHHSH hello", "▁", "KHJKJHHKJHHSH", "▁hel", "lo"); } @@ -25,7 +26,7 @@ public class SentencePieceConfigurationTest { var b = new SentencePieceConfig.Builder(); addModel("unknown", "src/test/models/sentencepiece/en.wiki.bpe.vs10000.model", b); b.collapseUnknowns(false); - var tester = new SentencePieceTester(new SentencePieceEmbedder(b.build())); + var tester = new EmbedderTester(new SentencePieceEmbedder(b.build())); tester.assertSegmented("KHJ hello", "▁", "K", "H", "J", "▁hel", "lo"); } @@ -34,7 +35,7 @@ public class SentencePieceConfigurationTest { var b = new SentencePieceConfig.Builder(); addModel("unknown", "src/test/models/sentencepiece/en.wiki.bpe.vs10000.model", b); b.scoring(SentencePieceConfig.Scoring.highestScore); - var tester = new SentencePieceTester(new SentencePieceEmbedder(b.build())); + var tester = new EmbedderTester(new SentencePieceEmbedder(b.build())); tester.assertSegmented("hello", "▁h", "el", "lo"); } @@ -43,7 +44,7 @@ public class SentencePieceConfigurationTest { var b = new SentencePieceConfig.Builder(); addModel("ja", "src/test/models/sentencepiece/ja.wiki.bpe.vs5000.model", b); addModel("en", "src/test/models/sentencepiece/en.wiki.bpe.vs10000.model", b); - var tester = new SentencePieceTester(new SentencePieceEmbedder(b.build())); + var tester = new EmbedderTester(new SentencePieceEmbedder(b.build())); tester.assertSegmented(Language.JAPANESE, "いくつかの通常のテキスト", "▁", "いく", "つか", "の", "通常", "の", "テ", "キ", "スト"); tester.assertSegmented(Language.ENGLISH, "hello", "▁hel", "lo"); tester.assertSegmented(Language.JAPANESE, "hello", "▁h", "ell", "o"); diff --git a/linguistics-components/src/test/java/com/yahoo/language/sentencepiece/SentencePieceTest.java b/linguistics-components/src/test/java/com/yahoo/language/sentencepiece/SentencePieceTest.java index 939f8ebe9d3..2fbafb23485 100644 --- a/linguistics-components/src/test/java/com/yahoo/language/sentencepiece/SentencePieceTest.java +++ b/linguistics-components/src/test/java/com/yahoo/language/sentencepiece/SentencePieceTest.java @@ -3,6 +3,7 @@ package com.yahoo.language.sentencepiece; import com.yahoo.language.Language; +import com.yahoo.language.tools.EmbedderTester; import org.junit.Test; import java.io.File; @@ -13,8 +14,8 @@ import java.io.File; public class SentencePieceTest { @Test - public void testEnglishTokenization() { - var tester = new SentencePieceTester(new File("src/test/models/sentencepiece/en.wiki.bpe.vs10000.model").toPath()); + public void testEnglishSegmenting() { + var tester = new EmbedderTester(new SentencePieceEmbedder.Builder("src/test/models/sentencepiece/en.wiki.bpe.vs10000.model").build()); tester.assertSegmented("h", "▁h"); tester.assertSegmented("he", "▁he"); tester.assertSegmented("hel", "▁hel"); @@ -36,39 +37,28 @@ public class SentencePieceTest { } @Test - public void testIntegerListEncoding() { - var tester = new SentencePieceTester(new File("src/test/models/sentencepiece/en.wiki.bpe.vs10000.model").toPath()); - tester.assertEmbedded("hello, world!", 908, 1418, 9934, 501, 9960); - tester.assertEmbedded("Hello, world!", 9912, 0, 6595, 9934, 501, 9960); - } - - @Test - public void testDenseTensorEncoding() { - var tester = new SentencePieceTester(new File("src/test/models/sentencepiece/en.wiki.bpe.vs10000.model").toPath()); - tester.assertEmbedded("hello, world!", "tensor(d[10])", "[908,1418,9934,501,9960,0,0,0,0,0]"); - tester.assertEmbedded("Hello, world!", "tensor(d[10])", "[9912,0,6595,9934,501,9960,0,0,0,0]"); - tester.assertEmbedded("hello, world!", "tensor(d[2])", "[908,1418]"); - } - - @Test - public void testSparseTensorEncoding() { - var tester = new SentencePieceTester(new File("src/test/models/sentencepiece/en.wiki.bpe.vs10000.model").toPath()); - tester.assertEmbedded("hello", "tensor(token{})", "{lo:1.0,'▁hel':0.0}"); + public void testEnglishEmbedding() { + var tester = new EmbedderTester(new SentencePieceEmbedder.Builder("src/test/models/sentencepiece/en.wiki.bpe.vs10000.model").build()); + tester.assertEmbedded("hello, world!", "tensor(d[10])", 908, 1418, 9934, 501, 9960); + tester.assertEmbedded("Hello, world!", "tensor(d[10])", 9912, 0, 6595, 9934, 501, 9960); + tester.assertEmbedded("hello, world!", "tensor(d[2])", 908, 1418, 9934, 501, 9960); } @Test public void testNoCollapse() { - var tester = new SentencePieceTester(new SentencePieceEmbedder.Builder() - .addDefaultModel(new File("src/test/models/sentencepiece/en.wiki.bpe.vs10000.model").toPath()) - .setCollapseUnknowns(false)); + var builder = new SentencePieceEmbedder.Builder() + .addDefaultModel(new File("src/test/models/sentencepiece/en.wiki.bpe.vs10000.model").toPath()) + .setCollapseUnknowns(false); + var tester = new EmbedderTester(builder.build()); tester.assertSegmented("KHJ hello", "▁", "K", "H", "J", "▁hel", "lo"); } @Test public void testHighestScore() { - var tester = new SentencePieceTester(new SentencePieceEmbedder.Builder() - .addDefaultModel(new File("src/test/models/sentencepiece/en.wiki.bpe.vs10000.model").toPath()) - .setScoring(Scoring.highestScore)); + var builder = new SentencePieceEmbedder.Builder() + .addDefaultModel(new File("src/test/models/sentencepiece/en.wiki.bpe.vs10000.model").toPath()) + .setScoring(Scoring.highestScore); + var tester = new EmbedderTester(builder.build()); tester.assertSegmented("h", "▁h"); tester.assertSegmented("he", "▁he"); tester.assertSegmented("hel", "▁h", "el"); @@ -80,7 +70,7 @@ public class SentencePieceTest { SentencePieceEmbedder.Builder builder = new SentencePieceEmbedder.Builder(); builder.addModel(Language.JAPANESE, new File("src/test/models/sentencepiece/ja.wiki.bpe.vs5000.model").toPath()); builder.addModel(Language.ENGLISH, new File("src/test/models/sentencepiece/en.wiki.bpe.vs10000.model").toPath()); - var tester = new SentencePieceTester(builder); + var tester = new EmbedderTester(builder.build()); tester.assertSegmented(Language.JAPANESE, "いくつかの通常のテキスト", "▁", "いく", "つか", "の", "通常", "の", "テ", "キ", "スト"); tester.assertSegmented(Language.ENGLISH, "hello", "▁hel", "lo"); tester.assertSegmented(Language.JAPANESE, "hello", "▁h", "ell", "o"); diff --git a/linguistics-components/src/test/java/com/yahoo/language/sentencepiece/SentencePieceTester.java b/linguistics-components/src/test/java/com/yahoo/language/sentencepiece/SentencePieceTester.java deleted file mode 100644 index 4dae53c60df..00000000000 --- a/linguistics-components/src/test/java/com/yahoo/language/sentencepiece/SentencePieceTester.java +++ /dev/null @@ -1,50 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -// - -package com.yahoo.language.sentencepiece; - -import com.yahoo.language.Language; -import com.yahoo.language.process.Embedder; -import com.yahoo.tensor.Tensor; -import com.yahoo.tensor.TensorType; - -import java.nio.file.Path; - -import static org.junit.Assert.assertArrayEquals; -import static org.junit.Assert.assertEquals; - -class SentencePieceTester { - - private final SentencePieceEmbedder embedder; - - public SentencePieceTester(Path model) { - this(new SentencePieceEmbedder.Builder().addDefaultModel(model)); - } - - public SentencePieceTester(SentencePieceEmbedder.Builder builder) { - this(builder.build()); - } - - public SentencePieceTester(SentencePieceEmbedder embedder) { - this.embedder = embedder; - } - - public void assertEmbedded(String input, Integer... expectedCodes) { - assertArrayEquals(expectedCodes, embedder.embed(input, new Embedder.Context("test")).toArray()); - } - - public void assertEmbedded(String input, String tensorType, String tensor) { - TensorType type = TensorType.fromSpec(tensorType); - Tensor expected = Tensor.from(type, tensor); - assertEquals(expected, embedder.embed(input, new Embedder.Context("test"), type)); - } - - public void assertSegmented(String input, String... expectedSegments) { - assertSegmented(Language.UNKNOWN, input, expectedSegments); - } - - public void assertSegmented(Language language, String input, String... expectedSegments) { - assertArrayEquals(expectedSegments, embedder.segment(input, language).toArray()); - } - -} diff --git a/linguistics-components/src/test/java/com/yahoo/language/tools/EmbedderTester.java b/linguistics-components/src/test/java/com/yahoo/language/tools/EmbedderTester.java new file mode 100644 index 00000000000..638efba2480 --- /dev/null +++ b/linguistics-components/src/test/java/com/yahoo/language/tools/EmbedderTester.java @@ -0,0 +1,62 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +package com.yahoo.language.tools; + +import com.yahoo.language.Language; +import com.yahoo.language.process.Embedder; +import com.yahoo.language.process.Segmenter; +import com.yahoo.tensor.Tensor; +import com.yahoo.tensor.TensorType; + +import java.util.Arrays; +import java.util.List; + +import static org.junit.Assert.assertArrayEquals; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +/** + * Tester of embedders. + * + * @author bratseth + */ +public class EmbedderTester { + + private final Embedder embedder; + + public EmbedderTester(Embedder embedder) { + this.embedder = embedder; + } + + /** + * Tests both embedding to a list of id's and encoding the same ids to a vector of the given type. + * + * @param expectedCodes all the expected codes of the given input, not including any trailing 0-paddings + * required for the tensor only + */ + public void assertEmbedded(String input, String tensorType, Integer... expectedCodes) { + TensorType type = TensorType.fromSpec(tensorType); + assertEquals(1, type.dimensions().size()); + assertTrue(type.dimensions().get(0).isIndexed()); + + int tensorSize = type.dimensions().get(0).size().get().intValue(); + + assertArrayEquals(expectedCodes, embedder.embed(input, new Embedder.Context("test")).toArray()); + + var builder = Tensor.Builder.of(type); + for (int i = 0; i < tensorSize; i++) + builder.cell(i < expectedCodes.length ? expectedCodes[i] : 0, i); + assertEquals(builder.build(), embedder.embed(input, new Embedder.Context("destination"), type)); + } + + public void assertSegmented(String input, String... expectedSegments) { + assertSegmented(Language.UNKNOWN, input, expectedSegments); + } + + public void assertSegmented(Language language, String input, String... expectedSegments) { + List<String> segments = ((Segmenter)embedder).segment(input, language); + assertArrayEquals("Actual segments: " + segments, + expectedSegments, ((Segmenter)embedder).segment(input, language).toArray()); + } + +} diff --git a/linguistics-components/src/test/java/com/yahoo/language/wordpiece/WordPieceEmbedderTest.java b/linguistics-components/src/test/java/com/yahoo/language/wordpiece/WordPieceEmbedderTest.java new file mode 100644 index 00000000000..13e0cbce10d --- /dev/null +++ b/linguistics-components/src/test/java/com/yahoo/language/wordpiece/WordPieceEmbedderTest.java @@ -0,0 +1,44 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.language.wordpiece; + +import com.yahoo.config.FileReference; +import com.yahoo.language.tools.EmbedderTester; +import org.junit.Test; + +/** + * Tests the WordPiece embedder + * + * @author bratseth + */ +public class WordPieceEmbedderTest { + + private static final String vocabulary = "src/test/models/wordpiece/bert-base-uncased-vocab.txt"; + + @Test + public void testWordPieceSegmentation() { + var tester = new EmbedderTester(new WordPieceEmbedder.Builder(vocabulary).build()); + tester.assertSegmented("what was the impact of the manhattan project", + "what", "was", "the", "impact", "of", "the", "manhattan", "project"); + tester.assertSegmented("overcommunication", "over", "##com", "##mun", "##ication"); + } + + @Test + public void testWordPieceEmbedding() { + var tester = new EmbedderTester(new WordPieceEmbedder.Builder(vocabulary).build()); + tester.assertEmbedded("what was the impact of the manhattan project", + "tensor(x[8])", + 2054, 2001, 1996, 4254, 1997, 1996, 7128, 2622); + } + + @Test + public void testWordPieceEmbedderConfiguration() { + var config = new WordPieceConfig.Builder().model(new WordPieceConfig.Model.Builder() + .language("unknown") + .path(new FileReference(vocabulary))) + .build(); + var tester = new EmbedderTester(new WordPieceEmbedder(config)); + tester.assertSegmented("what was the impact of the manhattan project", + "what", "was", "the", "impact", "of", "the", "manhattan", "project"); + } + +} diff --git a/linguistics-components/src/test/models/wordpiece/bert-base-uncased-vocab.txt b/linguistics-components/src/test/models/wordpiece/bert-base-uncased-vocab.txt new file mode 100644 index 00000000000..fb140275c15 --- /dev/null +++ b/linguistics-components/src/test/models/wordpiece/bert-base-uncased-vocab.txt @@ -0,0 +1,30522 @@ +[PAD] +[unused0] +[unused1] +[unused2] +[unused3] +[unused4] +[unused5] +[unused6] +[unused7] +[unused8] +[unused9] +[unused10] +[unused11] +[unused12] +[unused13] +[unused14] +[unused15] +[unused16] +[unused17] +[unused18] +[unused19] +[unused20] +[unused21] +[unused22] +[unused23] +[unused24] +[unused25] +[unused26] +[unused27] +[unused28] +[unused29] +[unused30] +[unused31] +[unused32] +[unused33] +[unused34] +[unused35] +[unused36] +[unused37] +[unused38] +[unused39] +[unused40] +[unused41] +[unused42] +[unused43] +[unused44] +[unused45] +[unused46] +[unused47] +[unused48] +[unused49] +[unused50] +[unused51] +[unused52] +[unused53] +[unused54] +[unused55] +[unused56] +[unused57] +[unused58] +[unused59] +[unused60] +[unused61] +[unused62] +[unused63] +[unused64] +[unused65] +[unused66] +[unused67] +[unused68] +[unused69] +[unused70] +[unused71] +[unused72] +[unused73] +[unused74] +[unused75] +[unused76] +[unused77] +[unused78] +[unused79] +[unused80] +[unused81] +[unused82] +[unused83] +[unused84] +[unused85] +[unused86] +[unused87] +[unused88] +[unused89] +[unused90] +[unused91] +[unused92] +[unused93] +[unused94] +[unused95] +[unused96] +[unused97] +[unused98] +[UNK] +[CLS] +[SEP] +[MASK] +[unused99] +[unused100] +[unused101] +[unused102] +[unused103] +[unused104] +[unused105] +[unused106] +[unused107] +[unused108] +[unused109] +[unused110] +[unused111] +[unused112] +[unused113] +[unused114] +[unused115] +[unused116] +[unused117] +[unused118] +[unused119] +[unused120] +[unused121] +[unused122] +[unused123] +[unused124] +[unused125] +[unused126] +[unused127] +[unused128] +[unused129] +[unused130] +[unused131] +[unused132] +[unused133] +[unused134] +[unused135] +[unused136] +[unused137] +[unused138] +[unused139] +[unused140] +[unused141] +[unused142] +[unused143] +[unused144] +[unused145] +[unused146] +[unused147] +[unused148] +[unused149] +[unused150] +[unused151] +[unused152] +[unused153] +[unused154] +[unused155] +[unused156] +[unused157] +[unused158] +[unused159] +[unused160] +[unused161] +[unused162] +[unused163] +[unused164] +[unused165] +[unused166] +[unused167] +[unused168] +[unused169] +[unused170] +[unused171] +[unused172] +[unused173] +[unused174] +[unused175] +[unused176] +[unused177] +[unused178] +[unused179] +[unused180] +[unused181] +[unused182] +[unused183] +[unused184] +[unused185] +[unused186] +[unused187] +[unused188] +[unused189] +[unused190] +[unused191] +[unused192] +[unused193] +[unused194] +[unused195] +[unused196] +[unused197] +[unused198] +[unused199] +[unused200] +[unused201] +[unused202] +[unused203] +[unused204] +[unused205] +[unused206] +[unused207] +[unused208] +[unused209] +[unused210] +[unused211] +[unused212] +[unused213] +[unused214] +[unused215] +[unused216] +[unused217] +[unused218] +[unused219] +[unused220] +[unused221] +[unused222] +[unused223] +[unused224] +[unused225] +[unused226] +[unused227] +[unused228] +[unused229] +[unused230] +[unused231] +[unused232] +[unused233] +[unused234] +[unused235] +[unused236] +[unused237] +[unused238] +[unused239] +[unused240] +[unused241] +[unused242] +[unused243] +[unused244] +[unused245] +[unused246] +[unused247] +[unused248] +[unused249] +[unused250] +[unused251] +[unused252] +[unused253] +[unused254] +[unused255] +[unused256] +[unused257] +[unused258] +[unused259] +[unused260] +[unused261] +[unused262] +[unused263] +[unused264] +[unused265] +[unused266] +[unused267] +[unused268] +[unused269] +[unused270] +[unused271] +[unused272] +[unused273] +[unused274] +[unused275] +[unused276] +[unused277] +[unused278] +[unused279] +[unused280] +[unused281] +[unused282] +[unused283] +[unused284] +[unused285] +[unused286] +[unused287] +[unused288] +[unused289] +[unused290] +[unused291] +[unused292] +[unused293] +[unused294] +[unused295] +[unused296] +[unused297] +[unused298] +[unused299] +[unused300] +[unused301] +[unused302] +[unused303] +[unused304] +[unused305] +[unused306] +[unused307] +[unused308] +[unused309] +[unused310] +[unused311] +[unused312] +[unused313] +[unused314] +[unused315] +[unused316] +[unused317] +[unused318] +[unused319] +[unused320] +[unused321] +[unused322] +[unused323] +[unused324] +[unused325] +[unused326] +[unused327] +[unused328] +[unused329] +[unused330] +[unused331] +[unused332] +[unused333] +[unused334] +[unused335] +[unused336] +[unused337] +[unused338] +[unused339] +[unused340] +[unused341] +[unused342] +[unused343] +[unused344] +[unused345] +[unused346] +[unused347] +[unused348] +[unused349] +[unused350] +[unused351] +[unused352] +[unused353] +[unused354] +[unused355] +[unused356] +[unused357] +[unused358] +[unused359] +[unused360] +[unused361] +[unused362] +[unused363] +[unused364] +[unused365] +[unused366] +[unused367] +[unused368] +[unused369] +[unused370] +[unused371] +[unused372] +[unused373] +[unused374] +[unused375] +[unused376] +[unused377] +[unused378] +[unused379] +[unused380] +[unused381] +[unused382] +[unused383] +[unused384] +[unused385] +[unused386] +[unused387] +[unused388] +[unused389] +[unused390] +[unused391] +[unused392] +[unused393] +[unused394] +[unused395] +[unused396] +[unused397] +[unused398] +[unused399] +[unused400] +[unused401] +[unused402] +[unused403] +[unused404] +[unused405] +[unused406] +[unused407] +[unused408] +[unused409] +[unused410] +[unused411] +[unused412] +[unused413] +[unused414] +[unused415] +[unused416] +[unused417] +[unused418] +[unused419] +[unused420] +[unused421] +[unused422] +[unused423] +[unused424] +[unused425] +[unused426] +[unused427] +[unused428] +[unused429] +[unused430] +[unused431] +[unused432] +[unused433] +[unused434] +[unused435] +[unused436] +[unused437] +[unused438] +[unused439] +[unused440] +[unused441] +[unused442] +[unused443] +[unused444] +[unused445] +[unused446] +[unused447] +[unused448] +[unused449] +[unused450] +[unused451] +[unused452] +[unused453] +[unused454] +[unused455] +[unused456] +[unused457] +[unused458] +[unused459] +[unused460] +[unused461] +[unused462] +[unused463] +[unused464] +[unused465] +[unused466] +[unused467] +[unused468] +[unused469] +[unused470] +[unused471] +[unused472] +[unused473] +[unused474] +[unused475] +[unused476] +[unused477] +[unused478] +[unused479] +[unused480] +[unused481] +[unused482] +[unused483] +[unused484] +[unused485] +[unused486] +[unused487] +[unused488] +[unused489] +[unused490] +[unused491] +[unused492] +[unused493] +[unused494] +[unused495] +[unused496] +[unused497] +[unused498] +[unused499] +[unused500] +[unused501] +[unused502] +[unused503] +[unused504] +[unused505] +[unused506] +[unused507] +[unused508] +[unused509] +[unused510] +[unused511] +[unused512] +[unused513] +[unused514] +[unused515] +[unused516] +[unused517] +[unused518] +[unused519] +[unused520] +[unused521] +[unused522] +[unused523] +[unused524] +[unused525] +[unused526] +[unused527] +[unused528] +[unused529] +[unused530] +[unused531] +[unused532] +[unused533] +[unused534] +[unused535] +[unused536] +[unused537] +[unused538] +[unused539] +[unused540] +[unused541] +[unused542] +[unused543] +[unused544] +[unused545] +[unused546] +[unused547] +[unused548] +[unused549] +[unused550] +[unused551] +[unused552] +[unused553] +[unused554] +[unused555] +[unused556] +[unused557] +[unused558] +[unused559] +[unused560] +[unused561] +[unused562] +[unused563] +[unused564] +[unused565] +[unused566] +[unused567] +[unused568] +[unused569] +[unused570] +[unused571] +[unused572] +[unused573] +[unused574] +[unused575] +[unused576] +[unused577] +[unused578] +[unused579] +[unused580] +[unused581] +[unused582] +[unused583] +[unused584] +[unused585] +[unused586] +[unused587] +[unused588] +[unused589] +[unused590] +[unused591] +[unused592] +[unused593] +[unused594] +[unused595] +[unused596] +[unused597] +[unused598] +[unused599] +[unused600] +[unused601] +[unused602] +[unused603] +[unused604] +[unused605] +[unused606] +[unused607] +[unused608] +[unused609] +[unused610] +[unused611] +[unused612] +[unused613] +[unused614] +[unused615] +[unused616] +[unused617] +[unused618] +[unused619] +[unused620] +[unused621] +[unused622] +[unused623] +[unused624] +[unused625] +[unused626] +[unused627] +[unused628] +[unused629] +[unused630] +[unused631] +[unused632] +[unused633] +[unused634] +[unused635] +[unused636] +[unused637] +[unused638] +[unused639] +[unused640] +[unused641] +[unused642] +[unused643] +[unused644] +[unused645] +[unused646] +[unused647] +[unused648] +[unused649] +[unused650] +[unused651] +[unused652] +[unused653] +[unused654] +[unused655] +[unused656] +[unused657] +[unused658] +[unused659] +[unused660] +[unused661] +[unused662] +[unused663] +[unused664] +[unused665] +[unused666] +[unused667] +[unused668] +[unused669] +[unused670] +[unused671] +[unused672] +[unused673] +[unused674] +[unused675] +[unused676] +[unused677] +[unused678] +[unused679] +[unused680] +[unused681] +[unused682] +[unused683] +[unused684] +[unused685] +[unused686] +[unused687] +[unused688] +[unused689] +[unused690] +[unused691] +[unused692] +[unused693] +[unused694] +[unused695] +[unused696] +[unused697] +[unused698] +[unused699] +[unused700] +[unused701] +[unused702] +[unused703] +[unused704] +[unused705] +[unused706] +[unused707] +[unused708] +[unused709] +[unused710] +[unused711] +[unused712] +[unused713] +[unused714] +[unused715] +[unused716] +[unused717] +[unused718] +[unused719] +[unused720] +[unused721] +[unused722] +[unused723] +[unused724] +[unused725] +[unused726] +[unused727] +[unused728] +[unused729] +[unused730] +[unused731] +[unused732] +[unused733] +[unused734] +[unused735] +[unused736] +[unused737] +[unused738] +[unused739] +[unused740] +[unused741] +[unused742] +[unused743] +[unused744] +[unused745] +[unused746] +[unused747] +[unused748] +[unused749] +[unused750] +[unused751] +[unused752] +[unused753] +[unused754] +[unused755] +[unused756] +[unused757] +[unused758] +[unused759] +[unused760] +[unused761] +[unused762] +[unused763] +[unused764] +[unused765] +[unused766] +[unused767] +[unused768] +[unused769] +[unused770] +[unused771] +[unused772] +[unused773] +[unused774] +[unused775] +[unused776] +[unused777] +[unused778] +[unused779] +[unused780] +[unused781] +[unused782] +[unused783] +[unused784] +[unused785] +[unused786] +[unused787] +[unused788] +[unused789] +[unused790] +[unused791] +[unused792] +[unused793] +[unused794] +[unused795] +[unused796] +[unused797] +[unused798] +[unused799] +[unused800] +[unused801] +[unused802] +[unused803] +[unused804] +[unused805] +[unused806] +[unused807] +[unused808] +[unused809] +[unused810] +[unused811] +[unused812] +[unused813] +[unused814] +[unused815] +[unused816] +[unused817] +[unused818] +[unused819] +[unused820] +[unused821] +[unused822] +[unused823] +[unused824] +[unused825] +[unused826] +[unused827] +[unused828] +[unused829] +[unused830] +[unused831] +[unused832] +[unused833] +[unused834] +[unused835] +[unused836] +[unused837] +[unused838] +[unused839] +[unused840] +[unused841] +[unused842] +[unused843] +[unused844] +[unused845] +[unused846] +[unused847] +[unused848] +[unused849] +[unused850] +[unused851] +[unused852] +[unused853] +[unused854] +[unused855] +[unused856] +[unused857] +[unused858] +[unused859] +[unused860] +[unused861] +[unused862] +[unused863] +[unused864] +[unused865] +[unused866] +[unused867] +[unused868] +[unused869] +[unused870] +[unused871] +[unused872] +[unused873] +[unused874] +[unused875] +[unused876] +[unused877] +[unused878] +[unused879] +[unused880] +[unused881] +[unused882] +[unused883] +[unused884] +[unused885] +[unused886] +[unused887] +[unused888] +[unused889] +[unused890] +[unused891] +[unused892] +[unused893] +[unused894] +[unused895] +[unused896] +[unused897] +[unused898] +[unused899] +[unused900] +[unused901] +[unused902] +[unused903] +[unused904] +[unused905] +[unused906] +[unused907] +[unused908] +[unused909] +[unused910] +[unused911] +[unused912] +[unused913] +[unused914] +[unused915] +[unused916] +[unused917] +[unused918] +[unused919] +[unused920] +[unused921] +[unused922] +[unused923] +[unused924] +[unused925] +[unused926] +[unused927] +[unused928] +[unused929] +[unused930] +[unused931] +[unused932] +[unused933] +[unused934] +[unused935] +[unused936] +[unused937] +[unused938] +[unused939] +[unused940] +[unused941] +[unused942] +[unused943] +[unused944] +[unused945] +[unused946] +[unused947] +[unused948] +[unused949] +[unused950] +[unused951] +[unused952] +[unused953] +[unused954] +[unused955] +[unused956] +[unused957] +[unused958] +[unused959] +[unused960] +[unused961] +[unused962] +[unused963] +[unused964] +[unused965] +[unused966] +[unused967] +[unused968] +[unused969] +[unused970] +[unused971] +[unused972] +[unused973] +[unused974] +[unused975] +[unused976] +[unused977] +[unused978] +[unused979] +[unused980] +[unused981] +[unused982] +[unused983] +[unused984] +[unused985] +[unused986] +[unused987] +[unused988] +[unused989] +[unused990] +[unused991] +[unused992] +[unused993] +! +" +# +$ +% +& +' +( +) +* ++ +, +- +. +/ +0 +1 +2 +3 +4 +5 +6 +7 +8 +9 +: +; +< += +> +? +@ +[ +\ +] +^ +_ +` +a +b +c +d +e +f +g +h +i +j +k +l +m +n +o +p +q +r +s +t +u +v +w +x +y +z +{ +| +} +~ +¡ +¢ +£ +¤ +¥ +¦ +§ +¨ +© +ª +« +¬ +® +° +± +² +³ +´ +µ +¶ +· +¹ +º +» +¼ +½ +¾ +¿ +× +ß +æ +ð +÷ +ø +þ +đ +ħ +ı +ł +ŋ +œ +ƒ +ɐ +ɑ +ɒ +ɔ +ɕ +ə +ɛ +ɡ +ɣ +ɨ +ɪ +ɫ +ɬ +ɯ +ɲ +ɴ +ɹ +ɾ +ʀ +ʁ +ʂ +ʃ +ʉ +ʊ +ʋ +ʌ +ʎ +ʐ +ʑ +ʒ +ʔ +ʰ +ʲ +ʳ +ʷ +ʸ +ʻ +ʼ +ʾ +ʿ +ˈ +ː +ˡ +ˢ +ˣ +ˤ +α +β +γ +δ +ε +ζ +η +θ +ι +κ +λ +μ +ν +ξ +ο +π +ρ +ς +σ +τ +υ +φ +χ +ψ +ω +а +б +в +г +д +е +ж +з +и +к +л +м +н +о +п +р +с +т +у +ф +х +ц +ч +ш +щ +ъ +ы +ь +э +ю +я +ђ +є +і +ј +љ +њ +ћ +ӏ +ա +բ +գ +դ +ե +թ +ի +լ +կ +հ +մ +յ +ն +ո +պ +ս +վ +տ +ր +ւ +ք +־ +א +ב +ג +ד +ה +ו +ז +ח +ט +י +ך +כ +ל +ם +מ +ן +נ +ס +ע +ף +פ +ץ +צ +ק +ר +ש +ת +، +ء +ا +ب +ة +ت +ث +ج +ح +خ +د +ذ +ر +ز +س +ش +ص +ض +ط +ظ +ع +غ +ـ +ف +ق +ك +ل +م +ن +ه +و +ى +ي +ٹ +پ +چ +ک +گ +ں +ھ +ہ +ی +ے +अ +आ +उ +ए +क +ख +ग +च +ज +ट +ड +ण +त +थ +द +ध +न +प +ब +भ +म +य +र +ल +व +श +ष +स +ह +ा +ि +ी +ो +। +॥ +ং +অ +আ +ই +উ +এ +ও +ক +খ +গ +চ +ছ +জ +ট +ড +ণ +ত +থ +দ +ধ +ন +প +ব +ভ +ম +য +র +ল +শ +ষ +স +হ +া +ি +ী +ে +க +ச +ட +த +ந +ன +ப +ம +ய +ர +ல +ள +வ +ா +ி +ு +ே +ை +ನ +ರ +ಾ +ක +ය +ර +ල +ව +ා +ก +ง +ต +ท +น +พ +ม +ย +ร +ล +ว +ส +อ +า +เ +་ +། +ག +ང +ད +ན +པ +བ +མ +འ +ར +ལ +ས +မ +ა +ბ +გ +დ +ე +ვ +თ +ი +კ +ლ +მ +ნ +ო +რ +ს +ტ +უ +ᄀ +ᄂ +ᄃ +ᄅ +ᄆ +ᄇ +ᄉ +ᄊ +ᄋ +ᄌ +ᄎ +ᄏ +ᄐ +ᄑ +ᄒ +ᅡ +ᅢ +ᅥ +ᅦ +ᅧ +ᅩ +ᅪ +ᅭ +ᅮ +ᅯ +ᅲ +ᅳ +ᅴ +ᅵ +ᆨ +ᆫ +ᆯ +ᆷ +ᆸ +ᆼ +ᴬ +ᴮ +ᴰ +ᴵ +ᴺ +ᵀ +ᵃ +ᵇ +ᵈ +ᵉ +ᵍ +ᵏ +ᵐ +ᵒ +ᵖ +ᵗ +ᵘ +ᵢ +ᵣ +ᵤ +ᵥ +ᶜ +ᶠ +‐ +‑ +‒ +– +— +― +‖ +‘ +’ +‚ +“ +” +„ +† +‡ +• +… +‰ +′ +″ +› +‿ +⁄ +⁰ +ⁱ +⁴ +⁵ +⁶ +⁷ +⁸ +⁹ +⁺ +⁻ +ⁿ +₀ +₁ +₂ +₃ +₄ +₅ +₆ +₇ +₈ +₉ +₊ +₍ +₎ +ₐ +ₑ +ₒ +ₓ +ₕ +ₖ +ₗ +ₘ +ₙ +ₚ +ₛ +ₜ +₤ +₩ +€ +₱ +₹ +ℓ +№ +ℝ +™ +⅓ +⅔ +← +↑ +→ +↓ +↔ +↦ +⇄ +⇌ +⇒ +∂ +∅ +∆ +∇ +∈ +− +∗ +∘ +√ +∞ +∧ +∨ +∩ +∪ +≈ +≡ +≤ +≥ +⊂ +⊆ +⊕ +⊗ +⋅ +─ +│ +■ +▪ +● +★ +☆ +☉ +♠ +♣ +♥ +♦ +♭ +♯ +⟨ +⟩ +ⱼ +⺩ +⺼ +⽥ +、 +。 +〈 +〉 +《 +》 +「 +」 +『 +』 +〜 +あ +い +う +え +お +か +き +く +け +こ +さ +し +す +せ +そ +た +ち +っ +つ +て +と +な +に +ぬ +ね +の +は +ひ +ふ +へ +ほ +ま +み +む +め +も +や +ゆ +よ +ら +り +る +れ +ろ +を +ん +ァ +ア +ィ +イ +ウ +ェ +エ +オ +カ +キ +ク +ケ +コ +サ +シ +ス +セ +タ +チ +ッ +ツ +テ +ト +ナ +ニ +ノ +ハ +ヒ +フ +ヘ +ホ +マ +ミ +ム +メ +モ +ャ +ュ +ョ +ラ +リ +ル +レ +ロ +ワ +ン +・ +ー +一 +三 +上 +下 +不 +世 +中 +主 +久 +之 +也 +事 +二 +五 +井 +京 +人 +亻 +仁 +介 +代 +仮 +伊 +会 +佐 +侍 +保 +信 +健 +元 +光 +八 +公 +内 +出 +分 +前 +劉 +力 +加 +勝 +北 +区 +十 +千 +南 +博 +原 +口 +古 +史 +司 +合 +吉 +同 +名 +和 +囗 +四 +国 +國 +土 +地 +坂 +城 +堂 +場 +士 +夏 +外 +大 +天 +太 +夫 +奈 +女 +子 +学 +宀 +宇 +安 +宗 +定 +宣 +宮 +家 +宿 +寺 +將 +小 +尚 +山 +岡 +島 +崎 +川 +州 +巿 +帝 +平 +年 +幸 +广 +弘 +張 +彳 +後 +御 +德 +心 +忄 +志 +忠 +愛 +成 +我 +戦 +戸 +手 +扌 +政 +文 +新 +方 +日 +明 +星 +春 +昭 +智 +曲 +書 +月 +有 +朝 +木 +本 +李 +村 +東 +松 +林 +森 +楊 +樹 +橋 +歌 +止 +正 +武 +比 +氏 +民 +水 +氵 +氷 +永 +江 +沢 +河 +治 +法 +海 +清 +漢 +瀬 +火 +版 +犬 +王 +生 +田 +男 +疒 +発 +白 +的 +皇 +目 +相 +省 +真 +石 +示 +社 +神 +福 +禾 +秀 +秋 +空 +立 +章 +竹 +糹 +美 +義 +耳 +良 +艹 +花 +英 +華 +葉 +藤 +行 +街 +西 +見 +訁 +語 +谷 +貝 +貴 +車 +軍 +辶 +道 +郎 +郡 +部 +都 +里 +野 +金 +鈴 +镇 +長 +門 +間 +阝 +阿 +陳 +陽 +雄 +青 +面 +風 +食 +香 +馬 +高 +龍 +龸 +fi +fl +！ +（ +） +， +－ +． +／ +： +？ +～ +the +of +and +in +to +was +he +is +as +for +on +with +that +it +his +by +at +from +her +##s +she +you +had +an +were +but +be +this +are +not +my +they +one +which +or +have +him +me +first +all +also +their +has +up +who +out +been +when +after +there +into +new +two +its +##a +time +would +no +what +about +said +we +over +then +other +so +more +##e +can +if +like +back +them +only +some +could +##i +where +just +##ing +during +before +##n +do +##o +made +school +through +than +now +years +most +world +may +between +down +well +three +##d +year +while +will +##ed +##r +##y +later +##t +city +under +around +did +such +being +used +state +people +part +know +against +your +many +second +university +both +national +##er +these +don +known +off +way +until +re +how +even +get +head +... +didn +##ly +team +american +because +de +##l +born +united +film +since +still +long +work +south +us +became +any +high +again +day +family +see +right +man +eyes +house +season +war +states +including +took +life +north +same +each +called +name +much +place +however +go +four +group +another +found +won +area +here +going +10 +away +series +left +home +music +best +make +hand +number +company +several +never +last +john +000 +very +album +take +end +good +too +following +released +game +played +little +began +district +##m +old +want +those +side +held +own +early +county +ll +league +use +west +##u +face +think +##es +2010 +government +##h +march +came +small +general +town +june +##on +line +based +something +##k +september +thought +looked +along +international +2011 +air +july +club +went +january +october +our +august +april +york +12 +few +2012 +2008 +east +show +member +college +2009 +father +public +##us +come +men +five +set +station +church +##c +next +former +november +room +party +located +december +2013 +age +got +2007 +##g +system +let +love +2006 +though +every +2014 +look +song +water +century +without +body +black +night +within +great +women +single +ve +building +large +population +river +named +band +white +started +##an +once +15 +20 +should +18 +2015 +service +top +built +british +open +death +king +moved +local +times +children +february +book +why +11 +door +need +president +order +final +road +wasn +although +due +major +died +village +third +knew +2016 +asked +turned +st +wanted +say +##p +together +received +main +son +served +different +##en +behind +himself +felt +members +power +football +law +voice +play +##in +near +park +history +30 +having +2005 +16 +##man +saw +mother +##al +army +point +front +help +english +street +art +late +hands +games +award +##ia +young +14 +put +published +country +division +across +told +13 +often +ever +french +london +center +six +red +2017 +led +days +include +light +25 +find +tell +among +species +really +according +central +half +2004 +form +original +gave +office +making +enough +lost +full +opened +must +included +live +given +german +player +run +business +woman +community +cup +might +million +land +2000 +court +development +17 +short +round +ii +km +seen +class +story +always +become +sure +research +almost +director +council +la +##2 +career +things +using +island +##z +couldn +car +##is +24 +close +force +##1 +better +free +support +control +field +students +2003 +education +married +##b +nothing +worked +others +record +big +inside +level +anything +continued +give +james +##3 +military +established +non +returned +feel +does +title +written +thing +feet +william +far +co +association +hard +already +2002 +##ra +championship +human +western +100 +##na +department +hall +role +various +production +21 +19 +heart +2001 +living +fire +version +##ers +##f +television +royal +##4 +produced +working +act +case +society +region +present +radio +period +looking +least +total +keep +england +wife +program +per +brother +mind +special +22 +##le +am +works +soon +##6 +political +george +services +taken +created +##7 +further +able +reached +david +union +joined +upon +done +important +social +information +either +##ic +##x +appeared +position +ground +lead +rock +dark +election +23 +board +france +hair +course +arms +site +police +girl +instead +real +sound +##v +words +moment +##te +someone +##8 +summer +project +announced +san +less +wrote +past +followed +##5 +blue +founded +al +finally +india +taking +records +america +##ne +1999 +design +considered +northern +god +stop +battle +toward +european +outside +described +track +today +playing +language +28 +call +26 +heard +professional +low +australia +miles +california +win +yet +green +##ie +trying +blood +##ton +southern +science +maybe +everything +match +square +27 +mouth +video +race +recorded +leave +above +##9 +daughter +points +space +1998 +museum +change +middle +common +##0 +move +tv +post +##ta +lake +seven +tried +elected +closed +ten +paul +minister +##th +months +start +chief +return +canada +person +sea +release +similar +modern +brought +rest +hit +formed +mr +##la +1997 +floor +event +doing +thomas +1996 +robert +care +killed +training +star +week +needed +turn +finished +railway +rather +news +health +sent +example +ran +term +michael +coming +currently +yes +forces +despite +gold +areas +50 +stage +fact +29 +dead +says +popular +2018 +originally +germany +probably +developed +result +pulled +friend +stood +money +running +mi +signed +word +songs +child +eventually +met +tour +average +teams +minutes +festival +current +deep +kind +1995 +decided +usually +eastern +seemed +##ness +episode +bed +added +table +indian +private +charles +route +available +idea +throughout +centre +addition +appointed +style +1994 +books +eight +construction +press +mean +wall +friends +remained +schools +study +##ch +##um +institute +oh +chinese +sometimes +events +possible +1992 +australian +type +brown +forward +talk +process +food +debut +seat +performance +committee +features +character +arts +herself +else +lot +strong +russian +range +hours +peter +arm +##da +morning +dr +sold +##ry +quickly +directed +1993 +guitar +china +##w +31 +list +##ma +performed +media +uk +players +smile +##rs +myself +40 +placed +coach +province +towards +wouldn +leading +whole +boy +official +designed +grand +census +##el +europe +attack +japanese +henry +1991 +##re +##os +cross +getting +alone +action +lower +network +wide +washington +japan +1990 +hospital +believe +changed +sister +##ar +hold +gone +sir +hadn +ship +##ka +studies +academy +shot +rights +below +base +bad +involved +kept +largest +##ist +bank +future +especially +beginning +mark +movement +section +female +magazine +plan +professor +lord +longer +##ian +sat +walked +hill +actually +civil +energy +model +families +size +thus +aircraft +completed +includes +data +captain +##or +fight +vocals +featured +richard +bridge +fourth +1989 +officer +stone +hear +##ism +means +medical +groups +management +self +lips +competition +entire +lived +technology +leaving +federal +tournament +bit +passed +hot +independent +awards +kingdom +mary +spent +fine +doesn +reported +##ling +jack +fall +raised +itself +stay +true +studio +1988 +sports +replaced +paris +systems +saint +leader +theatre +whose +market +capital +parents +spanish +canadian +earth +##ity +cut +degree +writing +bay +christian +awarded +natural +higher +bill +##as +coast +provided +previous +senior +ft +valley +organization +stopped +onto +countries +parts +conference +queen +security +interest +saying +allowed +master +earlier +phone +matter +smith +winning +try +happened +moving +campaign +los +##ley +breath +nearly +mid +1987 +certain +girls +date +italian +african +standing +fell +artist +##ted +shows +deal +mine +industry +1986 +##ng +everyone +republic +provide +collection +library +student +##ville +primary +owned +older +via +heavy +1st +makes +##able +attention +anyone +africa +##ri +stated +length +ended +fingers +command +staff +skin +foreign +opening +governor +okay +medal +kill +sun +cover +job +1985 +introduced +chest +hell +feeling +##ies +success +meet +reason +standard +meeting +novel +1984 +trade +source +buildings +##land +rose +guy +goal +##ur +chapter +native +husband +previously +unit +limited +entered +weeks +producer +operations +mountain +takes +covered +forced +related +roman +complete +successful +key +texas +cold +##ya +channel +1980 +traditional +films +dance +clear +approximately +500 +nine +van +prince +question +active +tracks +ireland +regional +silver +author +personal +sense +operation +##ine +economic +1983 +holding +twenty +isbn +additional +speed +hour +edition +regular +historic +places +whom +shook +movie +km² +secretary +prior +report +chicago +read +foundation +view +engine +scored +1982 +units +ask +airport +property +ready +immediately +lady +month +listed +contract +##de +manager +themselves +lines +##ki +navy +writer +meant +##ts +runs +##ro +practice +championships +singer +glass +commission +required +forest +starting +culture +generally +giving +access +attended +test +couple +stand +catholic +martin +caught +executive +##less +eye +##ey +thinking +chair +quite +shoulder +1979 +hope +decision +plays +defeated +municipality +whether +structure +offered +slowly +pain +ice +direction +##ion +paper +mission +1981 +mostly +200 +noted +individual +managed +nature +lives +plant +##ha +helped +except +studied +computer +figure +relationship +issue +significant +loss +die +smiled +gun +ago +highest +1972 +##am +male +bring +goals +mexico +problem +distance +commercial +completely +location +annual +famous +drive +1976 +neck +1978 +surface +caused +italy +understand +greek +highway +wrong +hotel +comes +appearance +joseph +double +issues +musical +companies +castle +income +review +assembly +bass +initially +parliament +artists +experience +1974 +particular +walk +foot +engineering +talking +window +dropped +##ter +miss +baby +boys +break +1975 +stars +edge +remember +policy +carried +train +stadium +bar +sex +angeles +evidence +##ge +becoming +assistant +soviet +1977 +upper +step +wing +1970 +youth +financial +reach +##ll +actor +numerous +##se +##st +nodded +arrived +##ation +minute +##nt +believed +sorry +complex +beautiful +victory +associated +temple +1968 +1973 +chance +perhaps +metal +##son +1945 +bishop +##et +lee +launched +particularly +tree +le +retired +subject +prize +contains +yeah +theory +empire +##ce +suddenly +waiting +trust +recording +##to +happy +terms +camp +champion +1971 +religious +pass +zealand +names +2nd +port +ancient +tom +corner +represented +watch +legal +anti +justice +cause +watched +brothers +45 +material +changes +simply +response +louis +fast +##ting +answer +60 +historical +1969 +stories +straight +create +feature +increased +rate +administration +virginia +el +activities +cultural +overall +winner +programs +basketball +legs +guard +beyond +cast +doctor +mm +flight +results +remains +cost +effect +winter +##ble +larger +islands +problems +chairman +grew +commander +isn +1967 +pay +failed +selected +hurt +fort +box +regiment +majority +journal +35 +edward +plans +##ke +##ni +shown +pretty +irish +characters +directly +scene +likely +operated +allow +spring +##j +junior +matches +looks +mike +houses +fellow +##tion +beach +marriage +##ham +##ive +rules +oil +65 +florida +expected +nearby +congress +sam +peace +recent +iii +wait +subsequently +cell +##do +variety +serving +agreed +please +poor +joe +pacific +attempt +wood +democratic +piece +prime +##ca +rural +mile +touch +appears +township +1964 +1966 +soldiers +##men +##ized +1965 +pennsylvania +closer +fighting +claimed +score +jones +physical +editor +##ous +filled +genus +specific +sitting +super +mom +##va +therefore +supported +status +fear +cases +store +meaning +wales +minor +spain +tower +focus +vice +frank +follow +parish +separate +golden +horse +fifth +remaining +branch +32 +presented +stared +##id +uses +secret +forms +##co +baseball +exactly +##ck +choice +note +discovered +travel +composed +truth +russia +ball +color +kiss +dad +wind +continue +ring +referred +numbers +digital +greater +##ns +metres +slightly +direct +increase +1960 +responsible +crew +rule +trees +troops +##no +broke +goes +individuals +hundred +weight +creek +sleep +memory +defense +provides +ordered +code +value +jewish +windows +1944 +safe +judge +whatever +corps +realized +growing +pre +##ga +cities +alexander +gaze +lies +spread +scott +letter +showed +situation +mayor +transport +watching +workers +extended +##li +expression +normal +##ment +chart +multiple +border +##ba +host +##ner +daily +mrs +walls +piano +##ko +heat +cannot +##ate +earned +products +drama +era +authority +seasons +join +grade +##io +sign +difficult +machine +1963 +territory +mainly +##wood +stations +squadron +1962 +stepped +iron +19th +##led +serve +appear +sky +speak +broken +charge +knowledge +kilometres +removed +ships +article +campus +simple +##ty +pushed +britain +##ve +leaves +recently +cd +soft +boston +latter +easy +acquired +poland +##sa +quality +officers +presence +planned +nations +mass +broadcast +jean +share +image +influence +wild +offer +emperor +electric +reading +headed +ability +promoted +yellow +ministry +1942 +throat +smaller +politician +##by +latin +spoke +cars +williams +males +lack +pop +80 +##ier +acting +seeing +consists +##ti +estate +1961 +pressure +johnson +newspaper +jr +chris +olympics +online +conditions +beat +elements +walking +vote +##field +needs +carolina +text +featuring +global +block +shirt +levels +francisco +purpose +females +et +dutch +duke +ahead +gas +twice +safety +serious +turning +highly +lieutenant +firm +maria +amount +mixed +daniel +proposed +perfect +agreement +affairs +3rd +seconds +contemporary +paid +1943 +prison +save +kitchen +label +administrative +intended +constructed +academic +nice +teacher +races +1956 +formerly +corporation +ben +nation +issued +shut +1958 +drums +housing +victoria +seems +opera +1959 +graduated +function +von +mentioned +picked +build +recognized +shortly +protection +picture +notable +exchange +elections +1980s +loved +percent +racing +fish +elizabeth +garden +volume +hockey +1941 +beside +settled +##ford +1940 +competed +replied +drew +1948 +actress +marine +scotland +steel +glanced +farm +steve +1957 +risk +tonight +positive +magic +singles +effects +gray +screen +dog +##ja +residents +bus +sides +none +secondary +literature +polish +destroyed +flying +founder +households +1939 +lay +reserve +usa +gallery +##ler +1946 +industrial +younger +approach +appearances +urban +ones +1950 +finish +avenue +powerful +fully +growth +page +honor +jersey +projects +advanced +revealed +basic +90 +infantry +pair +equipment +visit +33 +evening +search +grant +effort +solo +treatment +buried +republican +primarily +bottom +owner +1970s +israel +gives +jim +dream +bob +remain +spot +70 +notes +produce +champions +contact +ed +soul +accepted +ways +del +##ally +losing +split +price +capacity +basis +trial +questions +##ina +1955 +20th +guess +officially +memorial +naval +initial +##ization +whispered +median +engineer +##ful +sydney +##go +columbia +strength +300 +1952 +tears +senate +00 +card +asian +agent +1947 +software +44 +draw +warm +supposed +com +pro +##il +transferred +leaned +##at +candidate +escape +mountains +asia +potential +activity +entertainment +seem +traffic +jackson +murder +36 +slow +product +orchestra +haven +agency +bbc +taught +website +comedy +unable +storm +planning +albums +rugby +environment +scientific +grabbed +protect +##hi +boat +typically +1954 +1953 +damage +principal +divided +dedicated +mount +ohio +##berg +pick +fought +driver +##der +empty +shoulders +sort +thank +berlin +prominent +account +freedom +necessary +efforts +alex +headquarters +follows +alongside +des +simon +andrew +suggested +operating +learning +steps +1949 +sweet +technical +begin +easily +34 +teeth +speaking +settlement +scale +##sh +renamed +ray +max +enemy +semi +joint +compared +##rd +scottish +leadership +analysis +offers +georgia +pieces +captured +animal +deputy +guest +organized +##lin +tony +combined +method +challenge +1960s +huge +wants +battalion +sons +rise +crime +types +facilities +telling +path +1951 +platform +sit +1990s +##lo +tells +assigned +rich +pull +##ot +commonly +alive +##za +letters +concept +conducted +wearing +happen +bought +becomes +holy +gets +ocean +defeat +languages +purchased +coffee +occurred +titled +##q +declared +applied +sciences +concert +sounds +jazz +brain +##me +painting +fleet +tax +nick +##ius +michigan +count +animals +leaders +episodes +##line +content +##den +birth +##it +clubs +64 +palace +critical +refused +fair +leg +laughed +returning +surrounding +participated +formation +lifted +pointed +connected +rome +medicine +laid +taylor +santa +powers +adam +tall +shared +focused +knowing +yards +entrance +falls +##wa +calling +##ad +sources +chosen +beneath +resources +yard +##ite +nominated +silence +zone +defined +##que +gained +thirty +38 +bodies +moon +##ard +adopted +christmas +widely +register +apart +iran +premier +serves +du +unknown +parties +##les +generation +##ff +continues +quick +fields +brigade +quiet +teaching +clothes +impact +weapons +partner +flat +theater +supreme +1938 +37 +relations +##tor +plants +suffered +1936 +wilson +kids +begins +##age +1918 +seats +armed +internet +models +worth +laws +400 +communities +classes +background +knows +thanks +quarter +reaching +humans +carry +killing +format +kong +hong +setting +75 +architecture +disease +railroad +inc +possibly +wish +arthur +thoughts +harry +doors +density +##di +crowd +illinois +stomach +tone +unique +reports +anyway +##ir +liberal +der +vehicle +thick +dry +drug +faced +largely +facility +theme +holds +creation +strange +colonel +##mi +revolution +bell +politics +turns +silent +rail +relief +independence +combat +shape +write +determined +sales +learned +4th +finger +oxford +providing +1937 +heritage +fiction +situated +designated +allowing +distribution +hosted +##est +sight +interview +estimated +reduced +##ria +toronto +footballer +keeping +guys +damn +claim +motion +sport +sixth +stayed +##ze +en +rear +receive +handed +twelve +dress +audience +granted +brazil +##well +spirit +##ated +noticed +etc +olympic +representative +eric +tight +trouble +reviews +drink +vampire +missing +roles +ranked +newly +household +finals +wave +critics +##ee +phase +massachusetts +pilot +unlike +philadelphia +bright +guns +crown +organizations +roof +42 +respectively +clearly +tongue +marked +circle +fox +korea +bronze +brian +expanded +sexual +supply +yourself +inspired +labour +fc +##ah +reference +vision +draft +connection +brand +reasons +1935 +classic +driving +trip +jesus +cells +entry +1920 +neither +trail +claims +atlantic +orders +labor +nose +afraid +identified +intelligence +calls +cancer +attacked +passing +stephen +positions +imperial +grey +jason +39 +sunday +48 +swedish +avoid +extra +uncle +message +covers +allows +surprise +materials +fame +hunter +##ji +1930 +citizens +figures +davis +environmental +confirmed +shit +titles +di +performing +difference +acts +attacks +##ov +existing +votes +opportunity +nor +shop +entirely +trains +opposite +pakistan +##pa +develop +resulted +representatives +actions +reality +pressed +##ish +barely +wine +conversation +faculty +northwest +ends +documentary +nuclear +stock +grace +sets +eat +alternative +##ps +bag +resulting +creating +surprised +cemetery +1919 +drop +finding +sarah +cricket +streets +tradition +ride +1933 +exhibition +target +ear +explained +rain +composer +injury +apartment +municipal +educational +occupied +netherlands +clean +billion +constitution +learn +1914 +maximum +classical +francis +lose +opposition +jose +ontario +bear +core +hills +rolled +ending +drawn +permanent +fun +##tes +##lla +lewis +sites +chamber +ryan +##way +scoring +height +1934 +##house +lyrics +staring +55 +officials +1917 +snow +oldest +##tic +orange +##ger +qualified +interior +apparently +succeeded +thousand +dinner +lights +existence +fans +heavily +41 +greatest +conservative +send +bowl +plus +enter +catch +##un +economy +duty +1929 +speech +authorities +princess +performances +versions +shall +graduate +pictures +effective +remembered +poetry +desk +crossed +starring +starts +passenger +sharp +##ant +acres +ass +weather +falling +rank +fund +supporting +check +adult +publishing +heads +cm +southeast +lane +##burg +application +bc +##ura +les +condition +transfer +prevent +display +ex +regions +earl +federation +cool +relatively +answered +besides +1928 +obtained +portion +##town +mix +##ding +reaction +liked +dean +express +peak +1932 +##tte +counter +religion +chain +rare +miller +convention +aid +lie +vehicles +mobile +perform +squad +wonder +lying +crazy +sword +##ping +attempted +centuries +weren +philosophy +category +##ize +anna +interested +47 +sweden +wolf +frequently +abandoned +kg +literary +alliance +task +entitled +##ay +threw +promotion +factory +tiny +soccer +visited +matt +fm +achieved +52 +defence +internal +persian +43 +methods +##ging +arrested +otherwise +cambridge +programming +villages +elementary +districts +rooms +criminal +conflict +worry +trained +1931 +attempts +waited +signal +bird +truck +subsequent +programme +##ol +ad +49 +communist +details +faith +sector +patrick +carrying +laugh +##ss +controlled +korean +showing +origin +fuel +evil +1927 +##ent +brief +identity +darkness +address +pool +missed +publication +web +planet +ian +anne +wings +invited +##tt +briefly +standards +kissed +##be +ideas +climate +causing +walter +worse +albert +articles +winners +desire +aged +northeast +dangerous +gate +doubt +1922 +wooden +multi +##ky +poet +rising +funding +46 +communications +communication +violence +copies +prepared +ford +investigation +skills +1924 +pulling +electronic +##ak +##ial +##han +containing +ultimately +offices +singing +understanding +restaurant +tomorrow +fashion +christ +ward +da +pope +stands +5th +flow +studios +aired +commissioned +contained +exist +fresh +americans +##per +wrestling +approved +kid +employed +respect +suit +1925 +angel +asking +increasing +frame +angry +selling +1950s +thin +finds +##nd +temperature +statement +ali +explain +inhabitants +towns +extensive +narrow +51 +jane +flowers +images +promise +somewhere +object +fly +closely +##ls +1912 +bureau +cape +1926 +weekly +presidential +legislative +1921 +##ai +##au +launch +founding +##ny +978 +##ring +artillery +strike +un +institutions +roll +writers +landing +chose +kevin +anymore +pp +##ut +attorney +fit +dan +billboard +receiving +agricultural +breaking +sought +dave +admitted +lands +mexican +##bury +charlie +specifically +hole +iv +howard +credit +moscow +roads +accident +1923 +proved +wear +struck +hey +guards +stuff +slid +expansion +1915 +cat +anthony +##kin +melbourne +opposed +sub +southwest +architect +failure +plane +1916 +##ron +map +camera +tank +listen +regarding +wet +introduction +metropolitan +link +ep +fighter +inch +grown +gene +anger +fixed +buy +dvd +khan +domestic +worldwide +chapel +mill +functions +examples +##head +developing +1910 +turkey +hits +pocket +antonio +papers +grow +unless +circuit +18th +concerned +attached +journalist +selection +journey +converted +provincial +painted +hearing +aren +bands +negative +aside +wondered +knight +lap +survey +ma +##ow +noise +billy +##ium +shooting +guide +bedroom +priest +resistance +motor +homes +sounded +giant +##mer +150 +scenes +equal +comic +patients +hidden +solid +actual +bringing +afternoon +touched +funds +wedding +consisted +marie +canal +sr +kim +treaty +turkish +recognition +residence +cathedral +broad +knees +incident +shaped +fired +norwegian +handle +cheek +contest +represent +##pe +representing +beauty +##sen +birds +advantage +emergency +wrapped +drawing +notice +pink +broadcasting +##ong +somehow +bachelor +seventh +collected +registered +establishment +alan +assumed +chemical +personnel +roger +retirement +jeff +portuguese +wore +tied +device +threat +progress +advance +##ised +banks +hired +manchester +nfl +teachers +structures +forever +##bo +tennis +helping +saturday +sale +applications +junction +hip +incorporated +neighborhood +dressed +ceremony +##ds +influenced +hers +visual +stairs +decades +inner +kansas +hung +hoped +gain +scheduled +downtown +engaged +austria +clock +norway +certainly +pale +protected +1913 +victor +employees +plate +putting +surrounded +##ists +finishing +blues +tropical +##ries +minnesota +consider +philippines +accept +54 +retrieved +1900 +concern +anderson +properties +institution +gordon +successfully +vietnam +##dy +backing +outstanding +muslim +crossing +folk +producing +usual +demand +occurs +observed +lawyer +educated +##ana +kelly +string +pleasure +budget +items +quietly +colorado +philip +typical +##worth +derived +600 +survived +asks +mental +##ide +56 +jake +jews +distinguished +ltd +1911 +sri +extremely +53 +athletic +loud +thousands +worried +shadow +transportation +horses +weapon +arena +importance +users +tim +objects +contributed +dragon +douglas +aware +senator +johnny +jordan +sisters +engines +flag +investment +samuel +shock +capable +clark +row +wheel +refers +session +familiar +biggest +wins +hate +maintained +drove +hamilton +request +expressed +injured +underground +churches +walker +wars +tunnel +passes +stupid +agriculture +softly +cabinet +regarded +joining +indiana +##ea +##ms +push +dates +spend +behavior +woods +protein +gently +chase +morgan +mention +burning +wake +combination +occur +mirror +leads +jimmy +indeed +impossible +singapore +paintings +covering +##nes +soldier +locations +attendance +sell +historian +wisconsin +invasion +argued +painter +diego +changing +egypt +##don +experienced +inches +##ku +missouri +vol +grounds +spoken +switzerland +##gan +reform +rolling +ha +forget +massive +resigned +burned +allen +tennessee +locked +values +improved +##mo +wounded +universe +sick +dating +facing +pack +purchase +user +##pur +moments +##ul +merged +anniversary +1908 +coal +brick +understood +causes +dynasty +queensland +establish +stores +crisis +promote +hoping +views +cards +referee +extension +##si +raise +arizona +improve +colonial +formal +charged +##rt +palm +lucky +hide +rescue +faces +95 +feelings +candidates +juan +##ell +goods +6th +courses +weekend +59 +luke +cash +fallen +##om +delivered +affected +installed +carefully +tries +swiss +hollywood +costs +lincoln +responsibility +##he +shore +file +proper +normally +maryland +assistance +jump +constant +offering +friendly +waters +persons +realize +contain +trophy +800 +partnership +factor +58 +musicians +cry +bound +oregon +indicated +hero +houston +medium +##ure +consisting +somewhat +##ara +57 +cycle +##che +beer +moore +frederick +gotten +eleven +worst +weak +approached +arranged +chin +loan +universal +bond +fifteen +pattern +disappeared +##ney +translated +##zed +lip +arab +capture +interests +insurance +##chi +shifted +cave +prix +warning +sections +courts +coat +plot +smell +feed +golf +favorite +maintain +knife +vs +voted +degrees +finance +quebec +opinion +translation +manner +ruled +operate +productions +choose +musician +discovery +confused +tired +separated +stream +techniques +committed +attend +ranking +kings +throw +passengers +measure +horror +fan +mining +sand +danger +salt +calm +decade +dam +require +runner +##ik +rush +associate +greece +##ker +rivers +consecutive +matthew +##ski +sighed +sq +documents +steam +edited +closing +tie +accused +1905 +##ini +islamic +distributed +directors +organisation +bruce +7th +breathing +mad +lit +arrival +concrete +taste +08 +composition +shaking +faster +amateur +adjacent +stating +1906 +twin +flew +##ran +tokyo +publications +##tone +obviously +ridge +storage +1907 +carl +pages +concluded +desert +driven +universities +ages +terminal +sequence +borough +250 +constituency +creative +cousin +economics +dreams +margaret +notably +reduce +montreal +mode +17th +ears +saved +jan +vocal +##ica +1909 +andy +##jo +riding +roughly +threatened +##ise +meters +meanwhile +landed +compete +repeated +grass +czech +regularly +charges +tea +sudden +appeal +##ung +solution +describes +pierre +classification +glad +parking +##ning +belt +physics +99 +rachel +add +hungarian +participate +expedition +damaged +gift +childhood +85 +fifty +##red +mathematics +jumped +letting +defensive +mph +##ux +##gh +testing +##hip +hundreds +shoot +owners +matters +smoke +israeli +kentucky +dancing +mounted +grandfather +emma +designs +profit +argentina +##gs +truly +li +lawrence +cole +begun +detroit +willing +branches +smiling +decide +miami +enjoyed +recordings +##dale +poverty +ethnic +gay +##bi +gary +arabic +09 +accompanied +##one +##ons +fishing +determine +residential +acid +##ary +alice +returns +starred +mail +##ang +jonathan +strategy +##ue +net +forty +cook +businesses +equivalent +commonwealth +distinct +ill +##cy +seriously +##ors +##ped +shift +harris +replace +rio +imagine +formula +ensure +##ber +additionally +scheme +conservation +occasionally +purposes +feels +favor +##and +##ore +1930s +contrast +hanging +hunt +movies +1904 +instruments +victims +danish +christopher +busy +demon +sugar +earliest +colony +studying +balance +duties +##ks +belgium +slipped +carter +05 +visible +stages +iraq +fifa +##im +commune +forming +zero +07 +continuing +talked +counties +legend +bathroom +option +tail +clay +daughters +afterwards +severe +jaw +visitors +##ded +devices +aviation +russell +kate +##vi +entering +subjects +##ino +temporary +swimming +forth +smooth +ghost +audio +bush +operates +rocks +movements +signs +eddie +##tz +ann +voices +honorary +06 +memories +dallas +pure +measures +racial +promised +66 +harvard +ceo +16th +parliamentary +indicate +benefit +flesh +dublin +louisiana +1902 +1901 +patient +sleeping +1903 +membership +coastal +medieval +wanting +element +scholars +rice +62 +limit +survive +makeup +rating +definitely +collaboration +obvious +##tan +boss +ms +baron +birthday +linked +soil +diocese +##lan +ncaa +##mann +offensive +shell +shouldn +waist +##tus +plain +ross +organ +resolution +manufacturing +adding +relative +kennedy +98 +whilst +moth +marketing +gardens +crash +72 +heading +partners +credited +carlos +moves +cable +##zi +marshall +##out +depending +bottle +represents +rejected +responded +existed +04 +jobs +denmark +lock +##ating +treated +graham +routes +talent +commissioner +drugs +secure +tests +reign +restored +photography +##gi +contributions +oklahoma +designer +disc +grin +seattle +robin +paused +atlanta +unusual +##gate +praised +las +laughing +satellite +hungary +visiting +##sky +interesting +factors +deck +poems +norman +##water +stuck +speaker +rifle +domain +premiered +##her +dc +comics +actors +01 +reputation +eliminated +8th +ceiling +prisoners +script +##nce +leather +austin +mississippi +rapidly +admiral +parallel +charlotte +guilty +tools +gender +divisions +fruit +##bs +laboratory +nelson +fantasy +marry +rapid +aunt +tribe +requirements +aspects +suicide +amongst +adams +bone +ukraine +abc +kick +sees +edinburgh +clothing +column +rough +gods +hunting +broadway +gathered +concerns +##ek +spending +ty +12th +snapped +requires +solar +bones +cavalry +##tta +iowa +drinking +waste +index +franklin +charity +thompson +stewart +tip +flash +landscape +friday +enjoy +singh +poem +listening +##back +eighth +fred +differences +adapted +bomb +ukrainian +surgery +corporate +masters +anywhere +##more +waves +odd +sean +portugal +orleans +dick +debate +kent +eating +puerto +cleared +96 +expect +cinema +97 +guitarist +blocks +electrical +agree +involving +depth +dying +panel +struggle +##ged +peninsula +adults +novels +emerged +vienna +metro +debuted +shoes +tamil +songwriter +meets +prove +beating +instance +heaven +scared +sending +marks +artistic +passage +superior +03 +significantly +shopping +##tive +retained +##izing +malaysia +technique +cheeks +##ola +warren +maintenance +destroy +extreme +allied +120 +appearing +##yn +fill +advice +alabama +qualifying +policies +cleveland +hat +battery +smart +authors +10th +soundtrack +acted +dated +lb +glance +equipped +coalition +funny +outer +ambassador +roy +possibility +couples +campbell +dna +loose +ethan +supplies +1898 +gonna +88 +monster +##res +shake +agents +frequency +springs +dogs +practices +61 +gang +plastic +easier +suggests +gulf +blade +exposed +colors +industries +markets +pan +nervous +electoral +charts +legislation +ownership +##idae +mac +appointment +shield +copy +assault +socialist +abbey +monument +license +throne +employment +jay +93 +replacement +charter +cloud +powered +suffering +accounts +oak +connecticut +strongly +wright +colour +crystal +13th +context +welsh +networks +voiced +gabriel +jerry +##cing +forehead +mp +##ens +manage +schedule +totally +remix +##ii +forests +occupation +print +nicholas +brazilian +strategic +vampires +engineers +76 +roots +seek +correct +instrumental +und +alfred +backed +hop +##des +stanley +robinson +traveled +wayne +welcome +austrian +achieve +67 +exit +rates +1899 +strip +whereas +##cs +sing +deeply +adventure +bobby +rick +jamie +careful +components +cap +useful +personality +knee +##shi +pushing +hosts +02 +protest +ca +ottoman +symphony +##sis +63 +boundary +1890 +processes +considering +considerable +tons +##work +##ft +##nia +cooper +trading +dear +conduct +91 +illegal +apple +revolutionary +holiday +definition +harder +##van +jacob +circumstances +destruction +##lle +popularity +grip +classified +liverpool +donald +baltimore +flows +seeking +honour +approval +92 +mechanical +till +happening +statue +critic +increasingly +immediate +describe +commerce +stare +##ster +indonesia +meat +rounds +boats +baker +orthodox +depression +formally +worn +naked +claire +muttered +sentence +11th +emily +document +77 +criticism +wished +vessel +spiritual +bent +virgin +parker +minimum +murray +lunch +danny +printed +compilation +keyboards +false +blow +belonged +68 +raising +78 +cutting +##board +pittsburgh +##up +9th +shadows +81 +hated +indigenous +jon +15th +barry +scholar +ah +##zer +oliver +##gy +stick +susan +meetings +attracted +spell +romantic +##ver +ye +1895 +photo +demanded +customers +##ac +1896 +logan +revival +keys +modified +commanded +jeans +##ious +upset +raw +phil +detective +hiding +resident +vincent +##bly +experiences +diamond +defeating +coverage +lucas +external +parks +franchise +helen +bible +successor +percussion +celebrated +il +lift +profile +clan +romania +##ied +mills +##su +nobody +achievement +shrugged +fault +1897 +rhythm +initiative +breakfast +carbon +700 +69 +lasted +violent +74 +wound +ken +killer +gradually +filmed +°c +dollars +processing +94 +remove +criticized +guests +sang +chemistry +##vin +legislature +disney +##bridge +uniform +escaped +integrated +proposal +purple +denied +liquid +karl +influential +morris +nights +stones +intense +experimental +twisted +71 +84 +##ld +pace +nazi +mitchell +ny +blind +reporter +newspapers +14th +centers +burn +basin +forgotten +surviving +filed +collections +monastery +losses +manual +couch +description +appropriate +merely +tag +missions +sebastian +restoration +replacing +triple +73 +elder +julia +warriors +benjamin +julian +convinced +stronger +amazing +declined +versus +merchant +happens +output +finland +bare +barbara +absence +ignored +dawn +injuries +##port +producers +##ram +82 +luis +##ities +kw +admit +expensive +electricity +nba +exception +symbol +##ving +ladies +shower +sheriff +characteristics +##je +aimed +button +ratio +effectively +summit +angle +jury +bears +foster +vessels +pants +executed +evans +dozen +advertising +kicked +patrol +1889 +competitions +lifetime +principles +athletics +##logy +birmingham +sponsored +89 +rob +nomination +1893 +acoustic +##sm +creature +longest +##tra +credits +harbor +dust +josh +##so +territories +milk +infrastructure +completion +thailand +indians +leon +archbishop +##sy +assist +pitch +blake +arrangement +girlfriend +serbian +operational +hence +sad +scent +fur +dj +sessions +hp +refer +rarely +##ora +exists +1892 +##ten +scientists +dirty +penalty +burst +portrait +seed +79 +pole +limits +rival +1894 +stable +alpha +grave +constitutional +alcohol +arrest +flower +mystery +devil +architectural +relationships +greatly +habitat +##istic +larry +progressive +remote +cotton +##ics +##ok +preserved +reaches +##ming +cited +86 +vast +scholarship +decisions +cbs +joy +teach +1885 +editions +knocked +eve +searching +partly +participation +gap +animated +fate +excellent +##ett +na +87 +alternate +saints +youngest +##ily +climbed +##ita +##tors +suggest +##ct +discussion +staying +choir +lakes +jacket +revenue +nevertheless +peaked +instrument +wondering +annually +managing +neil +1891 +signing +terry +##ice +apply +clinical +brooklyn +aim +catherine +fuck +farmers +figured +ninth +pride +hugh +evolution +ordinary +involvement +comfortable +shouted +tech +encouraged +taiwan +representation +sharing +##lia +##em +panic +exact +cargo +competing +fat +cried +83 +1920s +occasions +pa +cabin +borders +utah +marcus +##isation +badly +muscles +##ance +victorian +transition +warner +bet +permission +##rin +slave +terrible +similarly +shares +seth +uefa +possession +medals +benefits +colleges +lowered +perfectly +mall +transit +##ye +##kar +publisher +##ened +harrison +deaths +elevation +##ae +asleep +machines +sigh +ash +hardly +argument +occasion +parent +leo +decline +1888 +contribution +##ua +concentration +1000 +opportunities +hispanic +guardian +extent +emotions +hips +mason +volumes +bloody +controversy +diameter +steady +mistake +phoenix +identify +violin +##sk +departure +richmond +spin +funeral +enemies +1864 +gear +literally +connor +random +sergeant +grab +confusion +1865 +transmission +informed +op +leaning +sacred +suspended +thinks +gates +portland +luck +agencies +yours +hull +expert +muscle +layer +practical +sculpture +jerusalem +latest +lloyd +statistics +deeper +recommended +warrior +arkansas +mess +supports +greg +eagle +1880 +recovered +rated +concerts +rushed +##ano +stops +eggs +files +premiere +keith +##vo +delhi +turner +pit +affair +belief +paint +##zing +mate +##ach +##ev +victim +##ology +withdrew +bonus +styles +fled +##ud +glasgow +technologies +funded +nbc +adaptation +##ata +portrayed +cooperation +supporters +judges +bernard +justin +hallway +ralph +##ick +graduating +controversial +distant +continental +spider +bite +##ho +recognize +intention +mixing +##ese +egyptian +bow +tourism +suppose +claiming +tiger +dominated +participants +vi +##ru +nurse +partially +tape +##rum +psychology +##rn +essential +touring +duo +voting +civilian +emotional +channels +##king +apparent +hebrew +1887 +tommy +carrier +intersection +beast +hudson +##gar +##zo +lab +nova +bench +discuss +costa +##ered +detailed +behalf +drivers +unfortunately +obtain +##lis +rocky +##dae +siege +friendship +honey +##rian +1861 +amy +hang +posted +governments +collins +respond +wildlife +preferred +operator +##po +laura +pregnant +videos +dennis +suspected +boots +instantly +weird +automatic +businessman +alleged +placing +throwing +ph +mood +1862 +perry +venue +jet +remainder +##lli +##ci +passion +biological +boyfriend +1863 +dirt +buffalo +ron +segment +fa +abuse +##era +genre +thrown +stroke +colored +stress +exercise +displayed +##gen +struggled +##tti +abroad +dramatic +wonderful +thereafter +madrid +component +widespread +##sed +tale +citizen +todd +monday +1886 +vancouver +overseas +forcing +crying +descent +##ris +discussed +substantial +ranks +regime +1870 +provinces +switch +drum +zane +ted +tribes +proof +lp +cream +researchers +volunteer +manor +silk +milan +donated +allies +venture +principle +delivery +enterprise +##ves +##ans +bars +traditionally +witch +reminded +copper +##uk +pete +inter +links +colin +grinned +elsewhere +competitive +frequent +##oy +scream +##hu +tension +texts +submarine +finnish +defending +defend +pat +detail +1884 +affiliated +stuart +themes +villa +periods +tool +belgian +ruling +crimes +answers +folded +licensed +resort +demolished +hans +lucy +1881 +lion +traded +photographs +writes +craig +##fa +trials +generated +beth +noble +debt +percentage +yorkshire +erected +ss +viewed +grades +confidence +ceased +islam +telephone +retail +##ible +chile +m² +roberts +sixteen +##ich +commented +hampshire +innocent +dual +pounds +checked +regulations +afghanistan +sung +rico +liberty +assets +bigger +options +angels +relegated +tribute +wells +attending +leaf +##yan +butler +romanian +forum +monthly +lisa +patterns +gmina +##tory +madison +hurricane +rev +##ians +bristol +##ula +elite +valuable +disaster +democracy +awareness +germans +freyja +##ins +loop +absolutely +paying +populations +maine +sole +prayer +spencer +releases +doorway +bull +##ani +lover +midnight +conclusion +##sson +thirteen +lily +mediterranean +##lt +nhl +proud +sample +##hill +drummer +guinea +##ova +murphy +climb +##ston +instant +attributed +horn +ain +railways +steven +##ao +autumn +ferry +opponent +root +traveling +secured +corridor +stretched +tales +sheet +trinity +cattle +helps +indicates +manhattan +murdered +fitted +1882 +gentle +grandmother +mines +shocked +vegas +produces +##light +caribbean +##ou +belong +continuous +desperate +drunk +historically +trio +waved +raf +dealing +nathan +bat +murmured +interrupted +residing +scientist +pioneer +harold +aaron +##net +delta +attempting +minority +mini +believes +chorus +tend +lots +eyed +indoor +load +shots +updated +jail +##llo +concerning +connecting +wealth +##ved +slaves +arrive +rangers +sufficient +rebuilt +##wick +cardinal +flood +muhammad +whenever +relation +runners +moral +repair +viewers +arriving +revenge +punk +assisted +bath +fairly +breathe +lists +innings +illustrated +whisper +nearest +voters +clinton +ties +ultimate +screamed +beijing +lions +andre +fictional +gathering +comfort +radar +suitable +dismissed +hms +ban +pine +wrist +atmosphere +voivodeship +bid +timber +##ned +##nan +giants +##ane +cameron +recovery +uss +identical +categories +switched +serbia +laughter +noah +ensemble +therapy +peoples +touching +##off +locally +pearl +platforms +everywhere +ballet +tables +lanka +herbert +outdoor +toured +derek +1883 +spaces +contested +swept +1878 +exclusive +slight +connections +##dra +winds +prisoner +collective +bangladesh +tube +publicly +wealthy +thai +##ys +isolated +select +##ric +insisted +pen +fortune +ticket +spotted +reportedly +animation +enforcement +tanks +110 +decides +wider +lowest +owen +##time +nod +hitting +##hn +gregory +furthermore +magazines +fighters +solutions +##ery +pointing +requested +peru +reed +chancellor +knights +mask +worker +eldest +flames +reduction +1860 +volunteers +##tis +reporting +##hl +wire +advisory +endemic +origins +settlers +pursue +knock +consumer +1876 +eu +compound +creatures +mansion +sentenced +ivan +deployed +guitars +frowned +involves +mechanism +kilometers +perspective +shops +maps +terminus +duncan +alien +fist +bridges +##pers +heroes +fed +derby +swallowed +##ros +patent +sara +illness +characterized +adventures +slide +hawaii +jurisdiction +##op +organised +##side +adelaide +walks +biology +se +##ties +rogers +swing +tightly +boundaries +##rie +prepare +implementation +stolen +##sha +certified +colombia +edwards +garage +##mm +recalled +##ball +rage +harm +nigeria +breast +##ren +furniture +pupils +settle +##lus +cuba +balls +client +alaska +21st +linear +thrust +celebration +latino +genetic +terror +##cia +##ening +lightning +fee +witness +lodge +establishing +skull +##ique +earning +hood +##ei +rebellion +wang +sporting +warned +missile +devoted +activist +porch +worship +fourteen +package +1871 +decorated +##shire +housed +##ock +chess +sailed +doctors +oscar +joan +treat +garcia +harbour +jeremy +##ire +traditions +dominant +jacques +##gon +##wan +relocated +1879 +amendment +sized +companion +simultaneously +volleyball +spun +acre +increases +stopping +loves +belongs +affect +drafted +tossed +scout +battles +1875 +filming +shoved +munich +tenure +vertical +romance +pc +##cher +argue +##ical +craft +ranging +www +opens +honest +tyler +yesterday +virtual +##let +muslims +reveal +snake +immigrants +radical +screaming +speakers +firing +saving +belonging +ease +lighting +prefecture +blame +farmer +hungry +grows +rubbed +beam +sur +subsidiary +##cha +armenian +sao +dropping +conventional +##fer +microsoft +reply +qualify +spots +1867 +sweat +festivals +##ken +immigration +physician +discover +exposure +sandy +explanation +isaac +implemented +##fish +hart +initiated +connect +stakes +presents +heights +householder +pleased +tourist +regardless +slip +closest +##ction +surely +sultan +brings +riley +preparation +aboard +slammed +baptist +experiment +ongoing +interstate +organic +playoffs +##ika +1877 +130 +##tar +hindu +error +tours +tier +plenty +arrangements +talks +trapped +excited +sank +ho +athens +1872 +denver +welfare +suburb +athletes +trick +diverse +belly +exclusively +yelled +1868 +##med +conversion +##ette +1874 +internationally +computers +conductor +abilities +sensitive +hello +dispute +measured +globe +rocket +prices +amsterdam +flights +tigers +inn +municipalities +emotion +references +3d +##mus +explains +airlines +manufactured +pm +archaeological +1873 +interpretation +devon +comment +##ites +settlements +kissing +absolute +improvement +suite +impressed +barcelona +sullivan +jefferson +towers +jesse +julie +##tin +##lu +grandson +hi +gauge +regard +rings +interviews +trace +raymond +thumb +departments +burns +serial +bulgarian +scores +demonstrated +##ix +1866 +kyle +alberta +underneath +romanized +##ward +relieved +acquisition +phrase +cliff +reveals +han +cuts +merger +custom +##dar +nee +gilbert +graduation +##nts +assessment +cafe +difficulty +demands +swung +democrat +jennifer +commons +1940s +grove +##yo +completing +focuses +sum +substitute +bearing +stretch +reception +##py +reflected +essentially +destination +pairs +##ched +survival +resource +##bach +promoting +doubles +messages +tear +##down +##fully +parade +florence +harvey +incumbent +partial +framework +900 +pedro +frozen +procedure +olivia +controls +##mic +shelter +personally +temperatures +##od +brisbane +tested +sits +marble +comprehensive +oxygen +leonard +##kov +inaugural +iranian +referring +quarters +attitude +##ivity +mainstream +lined +mars +dakota +norfolk +unsuccessful +##° +explosion +helicopter +congressional +##sing +inspector +bitch +seal +departed +divine +##ters +coaching +examination +punishment +manufacturer +sink +columns +unincorporated +signals +nevada +squeezed +dylan +dining +photos +martial +manuel +eighteen +elevator +brushed +plates +ministers +ivy +congregation +##len +slept +specialized +taxes +curve +restricted +negotiations +likes +statistical +arnold +inspiration +execution +bold +intermediate +significance +margin +ruler +wheels +gothic +intellectual +dependent +listened +eligible +buses +widow +syria +earn +cincinnati +collapsed +recipient +secrets +accessible +philippine +maritime +goddess +clerk +surrender +breaks +playoff +database +##ified +##lon +ideal +beetle +aspect +soap +regulation +strings +expand +anglo +shorter +crosses +retreat +tough +coins +wallace +directions +pressing +##oon +shipping +locomotives +comparison +topics +nephew +##mes +distinction +honors +travelled +sierra +ibn +##over +fortress +sa +recognised +carved +1869 +clients +##dan +intent +##mar +coaches +describing +bread +##ington +beaten +northwestern +##ona +merit +youtube +collapse +challenges +em +historians +objective +submitted +virus +attacking +drake +assume +##ere +diseases +marc +stem +leeds +##cus +##ab +farming +glasses +##lock +visits +nowhere +fellowship +relevant +carries +restaurants +experiments +101 +constantly +bases +targets +shah +tenth +opponents +verse +territorial +##ira +writings +corruption +##hs +instruction +inherited +reverse +emphasis +##vic +employee +arch +keeps +rabbi +watson +payment +uh +##ala +nancy +##tre +venice +fastest +sexy +banned +adrian +properly +ruth +touchdown +dollar +boards +metre +circles +edges +favour +comments +ok +travels +liberation +scattered +firmly +##ular +holland +permitted +diesel +kenya +den +originated +##ral +demons +resumed +dragged +rider +##rus +servant +blinked +extend +torn +##ias +##sey +input +meal +everybody +cylinder +kinds +camps +##fe +bullet +logic +##wn +croatian +evolved +healthy +fool +chocolate +wise +preserve +pradesh +##ess +respective +1850 +##ew +chicken +artificial +gross +corresponding +convicted +cage +caroline +dialogue +##dor +narrative +stranger +mario +br +christianity +failing +trent +commanding +buddhist +1848 +maurice +focusing +yale +bike +altitude +##ering +mouse +revised +##sley +veteran +##ig +pulls +theology +crashed +campaigns +legion +##ability +drag +excellence +customer +cancelled +intensity +excuse +##lar +liga +participating +contributing +printing +##burn +variable +##rk +curious +bin +legacy +renaissance +##my +symptoms +binding +vocalist +dancer +##nie +grammar +gospel +democrats +ya +enters +sc +diplomatic +hitler +##ser +clouds +mathematical +quit +defended +oriented +##heim +fundamental +hardware +impressive +equally +convince +confederate +guilt +chuck +sliding +##ware +magnetic +narrowed +petersburg +bulgaria +otto +phd +skill +##ama +reader +hopes +pitcher +reservoir +hearts +automatically +expecting +mysterious +bennett +extensively +imagined +seeds +monitor +fix +##ative +journalism +struggling +signature +ranch +encounter +photographer +observation +protests +##pin +influences +##hr +calendar +##all +cruz +croatia +locomotive +hughes +naturally +shakespeare +basement +hook +uncredited +faded +theories +approaches +dare +phillips +filling +fury +obama +##ain +efficient +arc +deliver +min +raid +breeding +inducted +leagues +efficiency +axis +montana +eagles +##ked +supplied +instructions +karen +picking +indicating +trap +anchor +practically +christians +tomb +vary +occasional +electronics +lords +readers +newcastle +faint +innovation +collect +situations +engagement +160 +claude +mixture +##feld +peer +tissue +logo +lean +##ration +°f +floors +##ven +architects +reducing +##our +##ments +rope +1859 +ottawa +##har +samples +banking +declaration +proteins +resignation +francois +saudi +advocate +exhibited +armor +twins +divorce +##ras +abraham +reviewed +jo +temporarily +matrix +physically +pulse +curled +##ena +difficulties +bengal +usage +##ban +annie +riders +certificate +##pi +holes +warsaw +distinctive +jessica +##mon +mutual +1857 +customs +circular +eugene +removal +loaded +mere +vulnerable +depicted +generations +dame +heir +enormous +lightly +climbing +pitched +lessons +pilots +nepal +ram +google +preparing +brad +louise +renowned +##₂ +liam +##ably +plaza +shaw +sophie +brilliant +bills +##bar +##nik +fucking +mainland +server +pleasant +seized +veterans +jerked +fail +beta +brush +radiation +stored +warmth +southeastern +nate +sin +raced +berkeley +joke +athlete +designation +trunk +##low +roland +qualification +archives +heels +artwork +receives +judicial +reserves +##bed +woke +installation +abu +floating +fake +lesser +excitement +interface +concentrated +addressed +characteristic +amanda +saxophone +monk +auto +##bus +releasing +egg +dies +interaction +defender +ce +outbreak +glory +loving +##bert +sequel +consciousness +http +awake +ski +enrolled +##ress +handling +rookie +brow +somebody +biography +warfare +amounts +contracts +presentation +fabric +dissolved +challenged +meter +psychological +lt +elevated +rally +accurate +##tha +hospitals +undergraduate +specialist +venezuela +exhibit +shed +nursing +protestant +fluid +structural +footage +jared +consistent +prey +##ska +succession +reflect +exile +lebanon +wiped +suspect +shanghai +resting +integration +preservation +marvel +variant +pirates +sheep +rounded +capita +sailing +colonies +manuscript +deemed +variations +clarke +functional +emerging +boxing +relaxed +curse +azerbaijan +heavyweight +nickname +editorial +rang +grid +tightened +earthquake +flashed +miguel +rushing +##ches +improvements +boxes +brooks +180 +consumption +molecular +felix +societies +repeatedly +variation +aids +civic +graphics +professionals +realm +autonomous +receiver +delayed +workshop +militia +chairs +trump +canyon +##point +harsh +extending +lovely +happiness +##jan +stake +eyebrows +embassy +wellington +hannah +##ella +sony +corners +bishops +swear +cloth +contents +xi +namely +commenced +1854 +stanford +nashville +courage +graphic +commitment +garrison +##bin +hamlet +clearing +rebels +attraction +literacy +cooking +ruins +temples +jenny +humanity +celebrate +hasn +freight +sixty +rebel +bastard +##art +newton +##ada +deer +##ges +##ching +smiles +delaware +singers +##ets +approaching +assists +flame +##ph +boulevard +barrel +planted +##ome +pursuit +##sia +consequences +posts +shallow +invitation +rode +depot +ernest +kane +rod +concepts +preston +topic +chambers +striking +blast +arrives +descendants +montgomery +ranges +worlds +##lay +##ari +span +chaos +praise +##ag +fewer +1855 +sanctuary +mud +fbi +##ions +programmes +maintaining +unity +harper +bore +handsome +closure +tournaments +thunder +nebraska +linda +facade +puts +satisfied +argentine +dale +cork +dome +panama +##yl +1858 +tasks +experts +##ates +feeding +equation +##las +##ida +##tu +engage +bryan +##ax +um +quartet +melody +disbanded +sheffield +blocked +gasped +delay +kisses +maggie +connects +##non +sts +poured +creator +publishers +##we +guided +ellis +extinct +hug +gaining +##ord +complicated +##bility +poll +clenched +investigate +##use +thereby +quantum +spine +cdp +humor +kills +administered +semifinals +##du +encountered +ignore +##bu +commentary +##maker +bother +roosevelt +140 +plains +halfway +flowing +cultures +crack +imprisoned +neighboring +airline +##ses +##view +##mate +##ec +gather +wolves +marathon +transformed +##ill +cruise +organisations +carol +punch +exhibitions +numbered +alarm +ratings +daddy +silently +##stein +queens +colours +impression +guidance +liu +tactical +##rat +marshal +della +arrow +##ings +rested +feared +tender +owns +bitter +advisor +escort +##ides +spare +farms +grants +##ene +dragons +encourage +colleagues +cameras +##und +sucked +pile +spirits +prague +statements +suspension +landmark +fence +torture +recreation +bags +permanently +survivors +pond +spy +predecessor +bombing +coup +##og +protecting +transformation +glow +##lands +##book +dug +priests +andrea +feat +barn +jumping +##chen +##ologist +##con +casualties +stern +auckland +pipe +serie +revealing +ba +##bel +trevor +mercy +spectrum +yang +consist +governing +collaborated +possessed +epic +comprises +blew +shane +##ack +lopez +honored +magical +sacrifice +judgment +perceived +hammer +mtv +baronet +tune +das +missionary +sheets +350 +neutral +oral +threatening +attractive +shade +aims +seminary +##master +estates +1856 +michel +wounds +refugees +manufacturers +##nic +mercury +syndrome +porter +##iya +##din +hamburg +identification +upstairs +purse +widened +pause +cared +breathed +affiliate +santiago +prevented +celtic +fisher +125 +recruited +byzantine +reconstruction +farther +##mp +diet +sake +au +spite +sensation +##ert +blank +separation +105 +##hon +vladimir +armies +anime +##lie +accommodate +orbit +cult +sofia +archive +##ify +##box +founders +sustained +disorder +honours +northeastern +mia +crops +violet +threats +blanket +fires +canton +followers +southwestern +prototype +voyage +assignment +altered +moderate +protocol +pistol +##eo +questioned +brass +lifting +1852 +math +authored +##ual +doug +dimensional +dynamic +##san +1851 +pronounced +grateful +quest +uncomfortable +boom +presidency +stevens +relating +politicians +chen +barrier +quinn +diana +mosque +tribal +cheese +palmer +portions +sometime +chester +treasure +wu +bend +download +millions +reforms +registration +##osa +consequently +monitoring +ate +preliminary +brandon +invented +ps +eaten +exterior +intervention +ports +documented +log +displays +lecture +sally +favourite +##itz +vermont +lo +invisible +isle +breed +##ator +journalists +relay +speaks +backward +explore +midfielder +actively +stefan +procedures +cannon +blond +kenneth +centered +servants +chains +libraries +malcolm +essex +henri +slavery +##hal +facts +fairy +coached +cassie +cats +washed +cop +##fi +announcement +item +2000s +vinyl +activated +marco +frontier +growled +curriculum +##das +loyal +accomplished +leslie +ritual +kenny +##00 +vii +napoleon +hollow +hybrid +jungle +stationed +friedrich +counted +##ulated +platinum +theatrical +seated +col +rubber +glen +1840 +diversity +healing +extends +id +provisions +administrator +columbus +##oe +tributary +te +assured +org +##uous +prestigious +examined +lectures +grammy +ronald +associations +bailey +allan +essays +flute +believing +consultant +proceedings +travelling +1853 +kit +kerala +yugoslavia +buddy +methodist +##ith +burial +centres +batman +##nda +discontinued +bo +dock +stockholm +lungs +severely +##nk +citing +manga +##ugh +steal +mumbai +iraqi +robot +celebrity +bride +broadcasts +abolished +pot +joel +overhead +franz +packed +reconnaissance +johann +acknowledged +introduce +handled +doctorate +developments +drinks +alley +palestine +##nis +##aki +proceeded +recover +bradley +grain +patch +afford +infection +nationalist +legendary +##ath +interchange +virtually +gen +gravity +exploration +amber +vital +wishes +powell +doctrine +elbow +screenplay +##bird +contribute +indonesian +pet +creates +##com +enzyme +kylie +discipline +drops +manila +hunger +##ien +layers +suffer +fever +bits +monica +keyboard +manages +##hood +searched +appeals +##bad +testament +grande +reid +##war +beliefs +congo +##ification +##dia +si +requiring +##via +casey +1849 +regret +streak +rape +depends +syrian +sprint +pound +tourists +upcoming +pub +##xi +tense +##els +practiced +echo +nationwide +guild +motorcycle +liz +##zar +chiefs +desired +elena +bye +precious +absorbed +relatives +booth +pianist +##mal +citizenship +exhausted +wilhelm +##ceae +##hed +noting +quarterback +urge +hectares +##gue +ace +holly +##tal +blonde +davies +parked +sustainable +stepping +twentieth +airfield +galaxy +nest +chip +##nell +tan +shaft +paulo +requirement +##zy +paradise +tobacco +trans +renewed +vietnamese +##cker +##ju +suggesting +catching +holmes +enjoying +md +trips +colt +holder +butterfly +nerve +reformed +cherry +bowling +trailer +carriage +goodbye +appreciate +toy +joshua +interactive +enabled +involve +##kan +collar +determination +bunch +facebook +recall +shorts +superintendent +episcopal +frustration +giovanni +nineteenth +laser +privately +array +circulation +##ovic +armstrong +deals +painful +permit +discrimination +##wi +aires +retiring +cottage +ni +##sta +horizon +ellen +jamaica +ripped +fernando +chapters +playstation +patron +lecturer +navigation +behaviour +genes +georgian +export +solomon +rivals +swift +seventeen +rodriguez +princeton +independently +sox +1847 +arguing +entity +casting +hank +criteria +oakland +geographic +milwaukee +reflection +expanding +conquest +dubbed +##tv +halt +brave +brunswick +doi +arched +curtis +divorced +predominantly +somerset +streams +ugly +zoo +horrible +curved +buenos +fierce +dictionary +vector +theological +unions +handful +stability +chan +punjab +segments +##lly +altar +ignoring +gesture +monsters +pastor +##stone +thighs +unexpected +operators +abruptly +coin +compiled +associates +improving +migration +pin +##ose +compact +collegiate +reserved +##urs +quarterfinals +roster +restore +assembled +hurry +oval +##cies +1846 +flags +martha +##del +victories +sharply +##rated +argues +deadly +neo +drawings +symbols +performer +##iel +griffin +restrictions +editing +andrews +java +journals +arabia +compositions +dee +pierce +removing +hindi +casino +runway +civilians +minds +nasa +hotels +##zation +refuge +rent +retain +potentially +conferences +suburban +conducting +##tto +##tions +##tle +descended +massacre +##cal +ammunition +terrain +fork +souls +counts +chelsea +durham +drives +cab +##bank +perth +realizing +palestinian +finn +simpson +##dal +betty +##ule +moreover +particles +cardinals +tent +evaluation +extraordinary +##oid +inscription +##works +wednesday +chloe +maintains +panels +ashley +trucks +##nation +cluster +sunlight +strikes +zhang +##wing +dialect +canon +##ap +tucked +##ws +collecting +##mas +##can +##sville +maker +quoted +evan +franco +aria +buying +cleaning +eva +closet +provision +apollo +clinic +rat +##ez +necessarily +ac +##gle +##ising +venues +flipped +cent +spreading +trustees +checking +authorized +##sco +disappointed +##ado +notion +duration +trumpet +hesitated +topped +brussels +rolls +theoretical +hint +define +aggressive +repeat +wash +peaceful +optical +width +allegedly +mcdonald +strict +copyright +##illa +investors +mar +jam +witnesses +sounding +miranda +michelle +privacy +hugo +harmony +##pp +valid +lynn +glared +nina +102 +headquartered +diving +boarding +gibson +##ncy +albanian +marsh +routine +dealt +enhanced +er +intelligent +substance +targeted +enlisted +discovers +spinning +observations +pissed +smoking +rebecca +capitol +visa +varied +costume +seemingly +indies +compensation +surgeon +thursday +arsenal +westminster +suburbs +rid +anglican +##ridge +knots +foods +alumni +lighter +fraser +whoever +portal +scandal +##ray +gavin +advised +instructor +flooding +terrorist +##ale +teenage +interim +senses +duck +teen +thesis +abby +eager +overcome +##ile +newport +glenn +rises +shame +##cc +prompted +priority +forgot +bomber +nicolas +protective +360 +cartoon +katherine +breeze +lonely +trusted +henderson +richardson +relax +banner +candy +palms +remarkable +##rio +legends +cricketer +essay +ordained +edmund +rifles +trigger +##uri +##away +sail +alert +1830 +audiences +penn +sussex +siblings +pursued +indianapolis +resist +rosa +consequence +succeed +avoided +1845 +##ulation +inland +##tie +##nna +counsel +profession +chronicle +hurried +##una +eyebrow +eventual +bleeding +innovative +cure +##dom +committees +accounting +con +scope +hardy +heather +tenor +gut +herald +codes +tore +scales +wagon +##oo +luxury +tin +prefer +fountain +triangle +bonds +darling +convoy +dried +traced +beings +troy +accidentally +slam +findings +smelled +joey +lawyers +outcome +steep +bosnia +configuration +shifting +toll +brook +performers +lobby +philosophical +construct +shrine +aggregate +boot +cox +phenomenon +savage +insane +solely +reynolds +lifestyle +##ima +nationally +holdings +consideration +enable +edgar +mo +mama +##tein +fights +relegation +chances +atomic +hub +conjunction +awkward +reactions +currency +finale +kumar +underwent +steering +elaborate +gifts +comprising +melissa +veins +reasonable +sunshine +chi +solve +trails +inhabited +elimination +ethics +huh +ana +molly +consent +apartments +layout +marines +##ces +hunters +bulk +##oma +hometown +##wall +##mont +cracked +reads +neighbouring +withdrawn +admission +wingspan +damned +anthology +lancashire +brands +batting +forgive +cuban +awful +##lyn +104 +dimensions +imagination +##ade +dante +##ship +tracking +desperately +goalkeeper +##yne +groaned +workshops +confident +burton +gerald +milton +circus +uncertain +slope +copenhagen +sophia +fog +philosopher +portraits +accent +cycling +varying +gripped +larvae +garrett +specified +scotia +mature +luther +kurt +rap +##kes +aerial +750 +ferdinand +heated +es +transported +##shan +safely +nonetheless +##orn +##gal +motors +demanding +##sburg +startled +##brook +ally +generate +caps +ghana +stained +demo +mentions +beds +ap +afterward +diary +##bling +utility +##iro +richards +1837 +conspiracy +conscious +shining +footsteps +observer +cyprus +urged +loyalty +developer +probability +olive +upgraded +gym +miracle +insects +graves +1844 +ourselves +hydrogen +amazon +katie +tickets +poets +##pm +planes +##pan +prevention +witnessed +dense +jin +randy +tang +warehouse +monroe +bang +archived +elderly +investigations +alec +granite +mineral +conflicts +controlling +aboriginal +carlo +##zu +mechanics +stan +stark +rhode +skirt +est +##berry +bombs +respected +##horn +imposed +limestone +deny +nominee +memphis +grabbing +disabled +##als +amusement +aa +frankfurt +corn +referendum +varies +slowed +disk +firms +unconscious +incredible +clue +sue +##zhou +twist +##cio +joins +idaho +chad +developers +computing +destroyer +103 +mortal +tucker +kingston +choices +yu +carson +1800 +os +whitney +geneva +pretend +dimension +staged +plateau +maya +##une +freestyle +##bc +rovers +hiv +##ids +tristan +classroom +prospect +##hus +honestly +diploma +lied +thermal +auxiliary +feast +unlikely +iata +##tel +morocco +pounding +treasury +lithuania +considerably +1841 +dish +1812 +geological +matching +stumbled +destroying +marched +brien +advances +cake +nicole +belle +settling +measuring +directing +##mie +tuesday +bassist +capabilities +stunned +fraud +torpedo +##list +##phone +anton +wisdom +surveillance +ruined +##ulate +lawsuit +healthcare +theorem +halls +trend +aka +horizontal +dozens +acquire +lasting +swim +hawk +gorgeous +fees +vicinity +decrease +adoption +tactics +##ography +pakistani +##ole +draws +##hall +willie +burke +heath +algorithm +integral +powder +elliott +brigadier +jackie +tate +varieties +darker +##cho +lately +cigarette +specimens +adds +##ree +##ensis +##inger +exploded +finalist +cia +murders +wilderness +arguments +nicknamed +acceptance +onwards +manufacture +robertson +jets +tampa +enterprises +blog +loudly +composers +nominations +1838 +ai +malta +inquiry +automobile +hosting +viii +rays +tilted +grief +museums +strategies +furious +euro +equality +cohen +poison +surrey +wireless +governed +ridiculous +moses +##esh +##room +vanished +##ito +barnes +attract +morrison +istanbul +##iness +absent +rotation +petition +janet +##logical +satisfaction +custody +deliberately +observatory +comedian +surfaces +pinyin +novelist +strictly +canterbury +oslo +monks +embrace +ibm +jealous +photograph +continent +dorothy +marina +doc +excess +holden +allegations +explaining +stack +avoiding +lance +storyline +majesty +poorly +spike +dos +bradford +raven +travis +classics +proven +voltage +pillow +fists +butt +1842 +interpreted +##car +1839 +gage +telegraph +lens +promising +expelled +casual +collector +zones +##min +silly +nintendo +##kh +##bra +downstairs +chef +suspicious +afl +flies +vacant +uganda +pregnancy +condemned +lutheran +estimates +cheap +decree +saxon +proximity +stripped +idiot +deposits +contrary +presenter +magnus +glacier +im +offense +edwin +##ori +upright +##long +bolt +##ois +toss +geographical +##izes +environments +delicate +marking +abstract +xavier +nails +windsor +plantation +occurring +equity +saskatchewan +fears +drifted +sequences +vegetation +revolt +##stic +1843 +sooner +fusion +opposing +nato +skating +1836 +secretly +ruin +lease +##oc +edit +##nne +flora +anxiety +ruby +##ological +##mia +tel +bout +taxi +emmy +frost +rainbow +compounds +foundations +rainfall +assassination +nightmare +dominican +##win +achievements +deserve +orlando +intact +armenia +##nte +calgary +valentine +106 +marion +proclaimed +theodore +bells +courtyard +thigh +gonzalez +console +troop +minimal +monte +everyday +##ence +##if +supporter +terrorism +buck +openly +presbyterian +activists +carpet +##iers +rubbing +uprising +##yi +cute +conceived +legally +##cht +millennium +cello +velocity +ji +rescued +cardiff +1835 +rex +concentrate +senators +beard +rendered +glowing +battalions +scouts +competitors +sculptor +catalogue +arctic +ion +raja +bicycle +wow +glancing +lawn +##woman +gentleman +lighthouse +publish +predicted +calculated +##val +variants +##gne +strain +##ui +winston +deceased +##nus +touchdowns +brady +caleb +sinking +echoed +crush +hon +blessed +protagonist +hayes +endangered +magnitude +editors +##tine +estimate +responsibilities +##mel +backup +laying +consumed +sealed +zurich +lovers +frustrated +##eau +ahmed +kicking +mit +treasurer +1832 +biblical +refuse +terrified +pump +agrees +genuine +imprisonment +refuses +plymouth +##hen +lou +##nen +tara +trembling +antarctic +ton +learns +##tas +crap +crucial +faction +atop +##borough +wrap +lancaster +odds +hopkins +erik +lyon +##eon +bros +##ode +snap +locality +tips +empress +crowned +cal +acclaimed +chuckled +##ory +clara +sends +mild +towel +##fl +##day +##а +wishing +assuming +interviewed +##bal +##die +interactions +eden +cups +helena +##lf +indie +beck +##fire +batteries +filipino +wizard +parted +##lam +traces +##born +rows +idol +albany +delegates +##ees +##sar +discussions +##ex +notre +instructed +belgrade +highways +suggestion +lauren +possess +orientation +alexandria +abdul +beats +salary +reunion +ludwig +alright +wagner +intimate +pockets +slovenia +hugged +brighton +merchants +cruel +stole +trek +slopes +repairs +enrollment +politically +underlying +promotional +counting +boeing +##bb +isabella +naming +##и +keen +bacteria +listing +separately +belfast +ussr +450 +lithuanian +anybody +ribs +sphere +martinez +cock +embarrassed +proposals +fragments +nationals +##fs +##wski +premises +fin +1500 +alpine +matched +freely +bounded +jace +sleeve +##af +gaming +pier +populated +evident +##like +frances +flooded +##dle +frightened +pour +trainer +framed +visitor +challenging +pig +wickets +##fold +infected +email +##pes +arose +##aw +reward +ecuador +oblast +vale +ch +shuttle +##usa +bach +rankings +forbidden +cornwall +accordance +salem +consumers +bruno +fantastic +toes +machinery +resolved +julius +remembering +propaganda +iceland +bombardment +tide +contacts +wives +##rah +concerto +macdonald +albania +implement +daisy +tapped +sudan +helmet +angela +mistress +##lic +crop +sunk +finest +##craft +hostile +##ute +##tsu +boxer +fr +paths +adjusted +habit +ballot +supervision +soprano +##zen +bullets +wicked +sunset +regiments +disappear +lamp +performs +app +##gia +##oa +rabbit +digging +incidents +entries +##cion +dishes +##oi +introducing +##ati +##fied +freshman +slot +jill +tackles +baroque +backs +##iest +lone +sponsor +destiny +altogether +convert +##aro +consensus +shapes +demonstration +basically +feminist +auction +artifacts +##bing +strongest +twitter +halifax +2019 +allmusic +mighty +smallest +precise +alexandra +viola +##los +##ille +manuscripts +##illo +dancers +ari +managers +monuments +blades +barracks +springfield +maiden +consolidated +electron +##end +berry +airing +wheat +nobel +inclusion +blair +payments +geography +bee +cc +eleanor +react +##hurst +afc +manitoba +##yu +su +lineup +fitness +recreational +investments +airborne +disappointment +##dis +edmonton +viewing +##row +renovation +##cast +infant +bankruptcy +roses +aftermath +pavilion +##yer +carpenter +withdrawal +ladder +##hy +discussing +popped +reliable +agreements +rochester +##abad +curves +bombers +220 +rao +reverend +decreased +choosing +107 +stiff +consulting +naples +crawford +tracy +ka +ribbon +cops +##lee +crushed +deciding +unified +teenager +accepting +flagship +explorer +poles +sanchez +inspection +revived +skilled +induced +exchanged +flee +locals +tragedy +swallow +loading +hanna +demonstrate +##ela +salvador +flown +contestants +civilization +##ines +wanna +rhodes +fletcher +hector +knocking +considers +##ough +nash +mechanisms +sensed +mentally +walt +unclear +##eus +renovated +madame +##cks +crews +governmental +##hin +undertaken +monkey +##ben +##ato +fatal +armored +copa +caves +governance +grasp +perception +certification +froze +damp +tugged +wyoming +##rg +##ero +newman +##lor +nerves +curiosity +graph +115 +##ami +withdraw +tunnels +dull +meredith +moss +exhibits +neighbors +communicate +accuracy +explored +raiders +republicans +secular +kat +superman +penny +criticised +##tch +freed +update +conviction +wade +ham +likewise +delegation +gotta +doll +promises +technological +myth +nationality +resolve +convent +##mark +sharon +dig +sip +coordinator +entrepreneur +fold +##dine +capability +councillor +synonym +blown +swan +cursed +1815 +jonas +haired +sofa +canvas +keeper +rivalry +##hart +rapper +speedway +swords +postal +maxwell +estonia +potter +recurring +##nn +##ave +errors +##oni +cognitive +1834 +##² +claws +nadu +roberto +bce +wrestler +ellie +##ations +infinite +ink +##tia +presumably +finite +staircase +108 +noel +patricia +nacional +##cation +chill +eternal +tu +preventing +prussia +fossil +limbs +##logist +ernst +frog +perez +rene +##ace +pizza +prussian +##ios +##vy +molecules +regulatory +answering +opinions +sworn +lengths +supposedly +hypothesis +upward +habitats +seating +ancestors +drank +yield +hd +synthesis +researcher +modest +##var +mothers +peered +voluntary +homeland +##the +acclaim +##igan +static +valve +luxembourg +alto +carroll +fe +receptor +norton +ambulance +##tian +johnston +catholics +depicting +jointly +elephant +gloria +mentor +badge +ahmad +distinguish +remarked +councils +precisely +allison +advancing +detection +crowded +##10 +cooperative +ankle +mercedes +dagger +surrendered +pollution +commit +subway +jeffrey +lesson +sculptures +provider +##fication +membrane +timothy +rectangular +fiscal +heating +teammate +basket +particle +anonymous +deployment +##ple +missiles +courthouse +proportion +shoe +sec +##ller +complaints +forbes +blacks +abandon +remind +sizes +overwhelming +autobiography +natalie +##awa +risks +contestant +countryside +babies +scorer +invaded +enclosed +proceed +hurling +disorders +##cu +reflecting +continuously +cruiser +graduates +freeway +investigated +ore +deserved +maid +blocking +phillip +jorge +shakes +dove +mann +variables +lacked +burden +accompanying +que +consistently +organizing +provisional +complained +endless +##rm +tubes +juice +georges +krishna +mick +labels +thriller +##uch +laps +arcade +sage +snail +##table +shannon +fi +laurence +seoul +vacation +presenting +hire +churchill +surprisingly +prohibited +savannah +technically +##oli +170 +##lessly +testimony +suited +speeds +toys +romans +mlb +flowering +measurement +talented +kay +settings +charleston +expectations +shattered +achieving +triumph +ceremonies +portsmouth +lanes +mandatory +loser +stretching +cologne +realizes +seventy +cornell +careers +webb +##ulating +americas +budapest +ava +suspicion +##ison +yo +conrad +##hai +sterling +jessie +rector +##az +1831 +transform +organize +loans +christine +volcanic +warrant +slender +summers +subfamily +newer +danced +dynamics +rhine +proceeds +heinrich +gastropod +commands +sings +facilitate +easter +ra +positioned +responses +expense +fruits +yanked +imported +25th +velvet +vic +primitive +tribune +baldwin +neighbourhood +donna +rip +hay +pr +##uro +1814 +espn +welcomed +##aria +qualifier +glare +highland +timing +##cted +shells +eased +geometry +louder +exciting +slovakia +##sion +##iz +##lot +savings +prairie +##ques +marching +rafael +tonnes +##lled +curtain +preceding +shy +heal +greene +worthy +##pot +detachment +bury +sherman +##eck +reinforced +seeks +bottles +contracted +duchess +outfit +walsh +##sc +mickey +##ase +geoffrey +archer +squeeze +dawson +eliminate +invention +##enberg +neal +##eth +stance +dealer +coral +maple +retire +polo +simplified +##ht +1833 +hid +watts +backwards +jules +##oke +genesis +mt +frames +rebounds +burma +woodland +moist +santos +whispers +drained +subspecies +##aa +streaming +ulster +burnt +correspondence +maternal +gerard +denis +stealing +##load +genius +duchy +##oria +inaugurated +momentum +suits +placement +sovereign +clause +thames +##hara +confederation +reservation +sketch +yankees +lets +rotten +charm +hal +verses +ultra +commercially +dot +salon +citation +adopt +winnipeg +mist +allocated +cairo +##boy +jenkins +interference +objectives +##wind +1820 +portfolio +armoured +sectors +##eh +initiatives +##world +integrity +exercises +robe +tap +ab +gazed +##tones +distracted +rulers +111 +favorable +jerome +tended +cart +factories +##eri +diplomat +valued +gravel +charitable +##try +calvin +exploring +chang +shepherd +terrace +pdf +pupil +##ural +reflects +ups +##rch +governors +shelf +depths +##nberg +trailed +crest +tackle +##nian +##ats +hatred +##kai +clare +makers +ethiopia +longtime +detected +embedded +lacking +slapped +rely +thomson +anticipation +iso +morton +successive +agnes +screenwriter +straightened +philippe +playwright +haunted +licence +iris +intentions +sutton +112 +logical +correctly +##weight +branded +licked +tipped +silva +ricky +narrator +requests +##ents +greeted +supernatural +cow +##wald +lung +refusing +employer +strait +gaelic +liner +##piece +zoe +sabha +##mba +driveway +harvest +prints +bates +reluctantly +threshold +algebra +ira +wherever +coupled +240 +assumption +picks +##air +designers +raids +gentlemen +##ean +roller +blowing +leipzig +locks +screw +dressing +strand +##lings +scar +dwarf +depicts +##nu +nods +##mine +differ +boris +##eur +yuan +flip +##gie +mob +invested +questioning +applying +##ture +shout +##sel +gameplay +blamed +illustrations +bothered +weakness +rehabilitation +##of +##zes +envelope +rumors +miners +leicester +subtle +kerry +##ico +ferguson +##fu +premiership +ne +##cat +bengali +prof +catches +remnants +dana +##rily +shouting +presidents +baltic +ought +ghosts +dances +sailors +shirley +fancy +dominic +##bie +madonna +##rick +bark +buttons +gymnasium +ashes +liver +toby +oath +providence +doyle +evangelical +nixon +cement +carnegie +embarked +hatch +surroundings +guarantee +needing +pirate +essence +##bee +filter +crane +hammond +projected +immune +percy +twelfth +##ult +regent +doctoral +damon +mikhail +##ichi +lu +critically +elect +realised +abortion +acute +screening +mythology +steadily +##fc +frown +nottingham +kirk +wa +minneapolis +##rra +module +algeria +mc +nautical +encounters +surprising +statues +availability +shirts +pie +alma +brows +munster +mack +soup +crater +tornado +sanskrit +cedar +explosive +bordered +dixon +planets +stamp +exam +happily +##bble +carriers +kidnapped +##vis +accommodation +emigrated +##met +knockout +correspondent +violation +profits +peaks +lang +specimen +agenda +ancestry +pottery +spelling +equations +obtaining +ki +linking +1825 +debris +asylum +##20 +buddhism +teddy +##ants +gazette +##nger +##sse +dental +eligibility +utc +fathers +averaged +zimbabwe +francesco +coloured +hissed +translator +lynch +mandate +humanities +mackenzie +uniforms +lin +##iana +##gio +asset +mhz +fitting +samantha +genera +wei +rim +beloved +shark +riot +entities +expressions +indo +carmen +slipping +owing +abbot +neighbor +sidney +##av +rats +recommendations +encouraging +squadrons +anticipated +commanders +conquered +##oto +donations +diagnosed +##mond +divide +##iva +guessed +decoration +vernon +auditorium +revelation +conversations +##kers +##power +herzegovina +dash +alike +protested +lateral +herman +accredited +mg +##gent +freeman +mel +fiji +crow +crimson +##rine +livestock +##pped +humanitarian +bored +oz +whip +##lene +##ali +legitimate +alter +grinning +spelled +anxious +oriental +wesley +##nin +##hole +carnival +controller +detect +##ssa +bowed +educator +kosovo +macedonia +##sin +occupy +mastering +stephanie +janeiro +para +unaware +nurses +noon +135 +cam +hopefully +ranger +combine +sociology +polar +rica +##eer +neill +##sman +holocaust +##ip +doubled +lust +1828 +109 +decent +cooling +unveiled +##card +1829 +nsw +homer +chapman +meyer +##gin +dive +mae +reagan +expertise +##gled +darwin +brooke +sided +prosecution +investigating +comprised +petroleum +genres +reluctant +differently +trilogy +johns +vegetables +corpse +highlighted +lounge +pension +unsuccessfully +elegant +aided +ivory +beatles +amelia +cain +dubai +sunny +immigrant +babe +click +##nder +underwater +pepper +combining +mumbled +atlas +horns +accessed +ballad +physicians +homeless +gestured +rpm +freak +louisville +corporations +patriots +prizes +rational +warn +modes +decorative +overnight +din +troubled +phantom +##ort +monarch +sheer +##dorf +generals +guidelines +organs +addresses +##zon +enhance +curling +parishes +cord +##kie +linux +caesar +deutsche +bavaria +##bia +coleman +cyclone +##eria +bacon +petty +##yama +##old +hampton +diagnosis +1824 +throws +complexity +rita +disputed +##₃ +pablo +##sch +marketed +trafficking +##ulus +examine +plague +formats +##oh +vault +faithful +##bourne +webster +##ox +highlights +##ient +##ann +phones +vacuum +sandwich +modeling +##gated +bolivia +clergy +qualities +isabel +##nas +##ars +wears +screams +reunited +annoyed +bra +##ancy +##rate +differential +transmitter +tattoo +container +poker +##och +excessive +resides +cowboys +##tum +augustus +trash +providers +statute +retreated +balcony +reversed +void +storey +preceded +masses +leap +laughs +neighborhoods +wards +schemes +falcon +santo +battlefield +pad +ronnie +thread +lesbian +venus +##dian +beg +sandstone +daylight +punched +gwen +analog +stroked +wwe +acceptable +measurements +dec +toxic +##kel +adequate +surgical +economist +parameters +varsity +##sberg +quantity +ella +##chy +##rton +countess +generating +precision +diamonds +expressway +ga +##ı +1821 +uruguay +talents +galleries +expenses +scanned +colleague +outlets +ryder +lucien +##ila +paramount +##bon +syracuse +dim +fangs +gown +sweep +##sie +toyota +missionaries +websites +##nsis +sentences +adviser +val +trademark +spells +##plane +patience +starter +slim +##borg +toe +incredibly +shoots +elliot +nobility +##wyn +cowboy +endorsed +gardner +tendency +persuaded +organisms +emissions +kazakhstan +amused +boring +chips +themed +##hand +llc +constantinople +chasing +systematic +guatemala +borrowed +erin +carey +##hard +highlands +struggles +1810 +##ifying +##ced +wong +exceptions +develops +enlarged +kindergarten +castro +##ern +##rina +leigh +zombie +juvenile +##most +consul +##nar +sailor +hyde +clarence +intensive +pinned +nasty +useless +jung +clayton +stuffed +exceptional +ix +apostolic +230 +transactions +##dge +exempt +swinging +cove +religions +##ash +shields +dairy +bypass +190 +pursuing +bug +joyce +bombay +chassis +southampton +chat +interact +redesignated +##pen +nascar +pray +salmon +rigid +regained +malaysian +grim +publicity +constituted +capturing +toilet +delegate +purely +tray +drift +loosely +striker +weakened +trinidad +mitch +itv +defines +transmitted +ming +scarlet +nodding +fitzgerald +fu +narrowly +sp +tooth +standings +virtue +##₁ +##wara +##cting +chateau +gloves +lid +##nel +hurting +conservatory +##pel +sinclair +reopened +sympathy +nigerian +strode +advocated +optional +chronic +discharge +##rc +suck +compatible +laurel +stella +shi +fails +wage +dodge +128 +informal +sorts +levi +buddha +villagers +##aka +chronicles +heavier +summoned +gateway +3000 +eleventh +jewelry +translations +accordingly +seas +##ency +fiber +pyramid +cubic +dragging +##ista +caring +##ops +android +contacted +lunar +##dt +kai +lisbon +patted +1826 +sacramento +theft +madagascar +subtropical +disputes +ta +holidays +piper +willow +mare +cane +itunes +newfoundland +benny +companions +dong +raj +observe +roar +charming +plaque +tibetan +fossils +enacted +manning +bubble +tina +tanzania +##eda +##hir +funk +swamp +deputies +cloak +ufc +scenario +par +scratch +metals +anthem +guru +engaging +specially +##boat +dialects +nineteen +cecil +duet +disability +messenger +unofficial +##lies +defunct +eds +moonlight +drainage +surname +puzzle +honda +switching +conservatives +mammals +knox +broadcaster +sidewalk +cope +##ried +benson +princes +peterson +##sal +bedford +sharks +eli +wreck +alberto +gasp +archaeology +lgbt +teaches +securities +madness +compromise +waving +coordination +davidson +visions +leased +possibilities +eighty +jun +fernandez +enthusiasm +assassin +sponsorship +reviewer +kingdoms +estonian +laboratories +##fy +##nal +applies +verb +celebrations +##zzo +rowing +lightweight +sadness +submit +mvp +balanced +dude +##vas +explicitly +metric +magnificent +mound +brett +mohammad +mistakes +irregular +##hing +##ass +sanders +betrayed +shipped +surge +##enburg +reporters +termed +georg +pity +verbal +bulls +abbreviated +enabling +appealed +##are +##atic +sicily +sting +heel +sweetheart +bart +spacecraft +brutal +monarchy +##tter +aberdeen +cameo +diane +##ub +survivor +clyde +##aries +complaint +##makers +clarinet +delicious +chilean +karnataka +coordinates +1818 +panties +##rst +pretending +ar +dramatically +kiev +bella +tends +distances +113 +catalog +launching +instances +telecommunications +portable +lindsay +vatican +##eim +angles +aliens +marker +stint +screens +bolton +##rne +judy +wool +benedict +plasma +europa +spark +imaging +filmmaker +swiftly +##een +contributor +##nor +opted +stamps +apologize +financing +butter +gideon +sophisticated +alignment +avery +chemicals +yearly +speculation +prominence +professionally +##ils +immortal +institutional +inception +wrists +identifying +tribunal +derives +gains +##wo +papal +preference +linguistic +vince +operative +brewery +##ont +unemployment +boyd +##ured +##outs +albeit +prophet +1813 +bi +##rr +##face +##rad +quarterly +asteroid +cleaned +radius +temper +##llen +telugu +jerk +viscount +menu +##ote +glimpse +##aya +yacht +hawaiian +baden +##rl +laptop +readily +##gu +monetary +offshore +scots +watches +##yang +##arian +upgrade +needle +xbox +lea +encyclopedia +flank +fingertips +##pus +delight +teachings +confirm +roth +beaches +midway +winters +##iah +teasing +daytime +beverly +gambling +bonnie +##backs +regulated +clement +hermann +tricks +knot +##shing +##uring +##vre +detached +ecological +owed +specialty +byron +inventor +bats +stays +screened +unesco +midland +trim +affection +##ander +##rry +jess +thoroughly +feedback +##uma +chennai +strained +heartbeat +wrapping +overtime +pleaded +##sworth +mon +leisure +oclc +##tate +##ele +feathers +angelo +thirds +nuts +surveys +clever +gill +commentator +##dos +darren +rides +gibraltar +##nc +##mu +dissolution +dedication +shin +meals +saddle +elvis +reds +chaired +taller +appreciation +functioning +niece +favored +advocacy +robbie +criminals +suffolk +yugoslav +passport +constable +congressman +hastings +vera +##rov +consecrated +sparks +ecclesiastical +confined +##ovich +muller +floyd +nora +1822 +paved +1827 +cumberland +ned +saga +spiral +##flow +appreciated +yi +collaborative +treating +similarities +feminine +finishes +##ib +jade +import +##nse +##hot +champagne +mice +securing +celebrities +helsinki +attributes +##gos +cousins +phases +ache +lucia +gandhi +submission +vicar +spear +shine +tasmania +biting +detention +constitute +tighter +seasonal +##gus +terrestrial +matthews +##oka +effectiveness +parody +philharmonic +##onic +1816 +strangers +encoded +consortium +guaranteed +regards +shifts +tortured +collision +supervisor +inform +broader +insight +theaters +armour +emeritus +blink +incorporates +mapping +##50 +##ein +handball +flexible +##nta +substantially +generous +thief +##own +carr +loses +1793 +prose +ucla +romeo +generic +metallic +realization +damages +mk +commissioners +zach +default +##ther +helicopters +lengthy +stems +spa +partnered +spectators +rogue +indication +penalties +teresa +1801 +sen +##tric +dalton +##wich +irving +photographic +##vey +dell +deaf +peters +excluded +unsure +##vable +patterson +crawled +##zio +resided +whipped +latvia +slower +ecole +pipes +employers +maharashtra +comparable +va +textile +pageant +##gel +alphabet +binary +irrigation +chartered +choked +antoine +offs +waking +supplement +##wen +quantities +demolition +regain +locate +urdu +folks +alt +114 +##mc +scary +andreas +whites +##ava +classrooms +mw +aesthetic +publishes +valleys +guides +cubs +johannes +bryant +conventions +affecting +##itt +drain +awesome +isolation +prosecutor +ambitious +apology +captive +downs +atmospheric +lorenzo +aisle +beef +foul +##onia +kidding +composite +disturbed +illusion +natives +##ffer +emi +rockets +riverside +wartime +painters +adolf +melted +##ail +uncertainty +simulation +hawks +progressed +meantime +builder +spray +breach +unhappy +regina +russians +##urg +determining +##tation +tram +1806 +##quin +aging +##12 +1823 +garion +rented +mister +diaz +terminated +clip +1817 +depend +nervously +disco +owe +defenders +shiva +notorious +disbelief +shiny +worcester +##gation +##yr +trailing +undertook +islander +belarus +limitations +watershed +fuller +overlooking +utilized +raphael +1819 +synthetic +breakdown +klein +##nate +moaned +memoir +lamb +practicing +##erly +cellular +arrows +exotic +##graphy +witches +117 +charted +rey +hut +hierarchy +subdivision +freshwater +giuseppe +aloud +reyes +qatar +marty +sideways +utterly +sexually +jude +prayers +mccarthy +softball +blend +damien +##gging +##metric +wholly +erupted +lebanese +negro +revenues +tasted +comparative +teamed +transaction +labeled +maori +sovereignty +parkway +trauma +gran +malay +121 +advancement +descendant +2020 +buzz +salvation +inventory +symbolic +##making +antarctica +mps +##gas +##bro +mohammed +myanmar +holt +submarines +tones +##lman +locker +patriarch +bangkok +emerson +remarks +predators +kin +afghan +confession +norwich +rental +emerge +advantages +##zel +rca +##hold +shortened +storms +aidan +##matic +autonomy +compliance +##quet +dudley +atp +##osis +1803 +motto +documentation +summary +professors +spectacular +christina +archdiocese +flashing +innocence +remake +##dell +psychic +reef +scare +employ +rs +sticks +meg +gus +leans +##ude +accompany +bergen +tomas +##iko +doom +wages +pools +##nch +##bes +breasts +scholarly +alison +outline +brittany +breakthrough +willis +realistic +##cut +##boro +competitor +##stan +pike +picnic +icon +designing +commercials +washing +villain +skiing +micro +costumes +auburn +halted +executives +##hat +logistics +cycles +vowel +applicable +barrett +exclaimed +eurovision +eternity +ramon +##umi +##lls +modifications +sweeping +disgust +##uck +torch +aviv +ensuring +rude +dusty +sonic +donovan +outskirts +cu +pathway +##band +##gun +##lines +disciplines +acids +cadet +paired +##40 +sketches +##sive +marriages +##⁺ +folding +peers +slovak +implies +admired +##beck +1880s +leopold +instinct +attained +weston +megan +horace +##ination +dorsal +ingredients +evolutionary +##its +complications +deity +lethal +brushing +levy +deserted +institutes +posthumously +delivering +telescope +coronation +motivated +rapids +luc +flicked +pays +volcano +tanner +weighed +##nica +crowds +frankie +gifted +addressing +granddaughter +winding +##rna +constantine +gomez +##front +landscapes +rudolf +anthropology +slate +werewolf +##lio +astronomy +circa +rouge +dreaming +sack +knelt +drowned +naomi +prolific +tracked +freezing +herb +##dium +agony +randall +twisting +wendy +deposit +touches +vein +wheeler +##bbled +##bor +batted +retaining +tire +presently +compare +specification +daemon +nigel +##grave +merry +recommendation +czechoslovakia +sandra +ng +roma +##sts +lambert +inheritance +sheikh +winchester +cries +examining +##yle +comeback +cuisine +nave +##iv +ko +retrieve +tomatoes +barker +polished +defining +irene +lantern +personalities +begging +tract +swore +1809 +175 +##gic +omaha +brotherhood +##rley +haiti +##ots +exeter +##ete +##zia +steele +dumb +pearson +210 +surveyed +elisabeth +trends +##ef +fritz +##rf +premium +bugs +fraction +calmly +viking +##birds +tug +inserted +unusually +##ield +confronted +distress +crashing +brent +turks +resign +##olo +cambodia +gabe +sauce +##kal +evelyn +116 +extant +clusters +quarry +teenagers +luna +##lers +##ister +affiliation +drill +##ashi +panthers +scenic +libya +anita +strengthen +inscriptions +##cated +lace +sued +judith +riots +##uted +mint +##eta +preparations +midst +dub +challenger +##vich +mock +cf +displaced +wicket +breaths +enables +schmidt +analyst +##lum +ag +highlight +automotive +axe +josef +newark +sufficiently +resembles +50th +##pal +flushed +mum +traits +##ante +commodore +incomplete +warming +titular +ceremonial +ethical +118 +celebrating +eighteenth +cao +lima +medalist +mobility +strips +snakes +##city +miniature +zagreb +barton +escapes +umbrella +automated +doubted +differs +cooled +georgetown +dresden +cooked +fade +wyatt +rna +jacobs +carlton +abundant +stereo +boost +madras +inning +##hia +spur +ip +malayalam +begged +osaka +groan +escaping +charging +dose +vista +##aj +bud +papa +communists +advocates +edged +tri +##cent +resemble +peaking +necklace +fried +montenegro +saxony +goose +glances +stuttgart +curator +recruit +grocery +sympathetic +##tting +##fort +127 +lotus +randolph +ancestor +##rand +succeeding +jupiter +1798 +macedonian +##heads +hiking +1808 +handing +fischer +##itive +garbage +node +##pies +prone +singular +papua +inclined +attractions +italia +pouring +motioned +grandma +garnered +jacksonville +corp +ego +ringing +aluminum +##hausen +ordering +##foot +drawer +traders +synagogue +##play +##kawa +resistant +wandering +fragile +fiona +teased +var +hardcore +soaked +jubilee +decisive +exposition +mercer +poster +valencia +hale +kuwait +1811 +##ises +##wr +##eed +tavern +gamma +122 +johan +##uer +airways +amino +gil +##ury +vocational +domains +torres +##sp +generator +folklore +outcomes +##keeper +canberra +shooter +fl +beams +confrontation +##lling +##gram +feb +aligned +forestry +pipeline +jax +motorway +conception +decay +##tos +coffin +##cott +stalin +1805 +escorted +minded +##nam +sitcom +purchasing +twilight +veronica +additions +passive +tensions +straw +123 +frequencies +1804 +refugee +cultivation +##iate +christie +clary +bulletin +crept +disposal +##rich +##zong +processor +crescent +##rol +bmw +emphasized +whale +nazis +aurora +##eng +dwelling +hauled +sponsors +toledo +mega +ideology +theatres +tessa +cerambycidae +saves +turtle +cone +suspects +kara +rusty +yelling +greeks +mozart +shades +cocked +participant +##tro +shire +spit +freeze +necessity +##cos +inmates +nielsen +councillors +loaned +uncommon +omar +peasants +botanical +offspring +daniels +formations +jokes +1794 +pioneers +sigma +licensing +##sus +wheelchair +polite +1807 +liquor +pratt +trustee +##uta +forewings +balloon +##zz +kilometre +camping +explicit +casually +shawn +foolish +teammates +nm +hassan +carrie +judged +satisfy +vanessa +knives +selective +cnn +flowed +##lice +eclipse +stressed +eliza +mathematician +cease +cultivated +##roy +commissions +browns +##ania +destroyers +sheridan +meadow +##rius +minerals +##cial +downstream +clash +gram +memoirs +ventures +baha +seymour +archie +midlands +edith +fare +flynn +invite +canceled +tiles +stabbed +boulder +incorporate +amended +camden +facial +mollusk +unreleased +descriptions +yoga +grabs +550 +raises +ramp +shiver +##rose +coined +pioneering +tunes +qing +warwick +tops +119 +melanie +giles +##rous +wandered +##inal +annexed +nov +30th +unnamed +##ished +organizational +airplane +normandy +stoke +whistle +blessing +violations +chased +holders +shotgun +##ctic +outlet +reactor +##vik +tires +tearing +shores +fortified +mascot +constituencies +nc +columnist +productive +tibet +##rta +lineage +hooked +oct +tapes +judging +cody +##gger +hansen +kashmir +triggered +##eva +solved +cliffs +##tree +resisted +anatomy +protesters +transparent +implied +##iga +injection +mattress +excluding +##mbo +defenses +helpless +devotion +##elli +growl +liberals +weber +phenomena +atoms +plug +##iff +mortality +apprentice +howe +convincing +aaa +swimmer +barber +leone +promptly +sodium +def +nowadays +arise +##oning +gloucester +corrected +dignity +norm +erie +##ders +elders +evacuated +sylvia +compression +##yar +hartford +pose +backpack +reasoning +accepts +24th +wipe +millimetres +marcel +##oda +dodgers +albion +1790 +overwhelmed +aerospace +oaks +1795 +showcase +acknowledge +recovering +nolan +ashe +hurts +geology +fashioned +disappearance +farewell +swollen +shrug +marquis +wimbledon +124 +rue +1792 +commemorate +reduces +experiencing +inevitable +calcutta +intel +##court +murderer +sticking +fisheries +imagery +bloom +280 +brake +##inus +gustav +hesitation +memorable +po +viral +beans +accidents +tunisia +antenna +spilled +consort +treatments +aye +perimeter +##gard +donation +hostage +migrated +banker +addiction +apex +lil +trout +##ously +conscience +##nova +rams +sands +genome +passionate +troubles +##lets +##set +amid +##ibility +##ret +higgins +exceed +vikings +##vie +payne +##zan +muscular +##ste +defendant +sucking +##wal +ibrahim +fuselage +claudia +vfl +europeans +snails +interval +##garh +preparatory +statewide +tasked +lacrosse +viktor +##lation +angola +##hra +flint +implications +employs +teens +patrons +stall +weekends +barriers +scrambled +nucleus +tehran +jenna +parsons +lifelong +robots +displacement +5000 +##bles +precipitation +##gt +knuckles +clutched +1802 +marrying +ecology +marx +accusations +declare +scars +kolkata +mat +meadows +bermuda +skeleton +finalists +vintage +crawl +coordinate +affects +subjected +orchestral +mistaken +##tc +mirrors +dipped +relied +260 +arches +candle +##nick +incorporating +wildly +fond +basilica +owl +fringe +rituals +whispering +stirred +feud +tertiary +slick +goat +honorable +whereby +skip +ricardo +stripes +parachute +adjoining +submerged +synthesizer +##gren +intend +positively +ninety +phi +beaver +partition +fellows +alexis +prohibition +carlisle +bizarre +fraternity +##bre +doubts +icy +cbc +aquatic +sneak +sonny +combines +airports +crude +supervised +spatial +merge +alfonso +##bic +corrupt +scan +undergo +##ams +disabilities +colombian +comparing +dolphins +perkins +##lish +reprinted +unanimous +bounced +hairs +underworld +midwest +semester +bucket +paperback +miniseries +coventry +demise +##leigh +demonstrations +sensor +rotating +yan +##hler +arrange +soils +##idge +hyderabad +labs +##dr +brakes +grandchildren +##nde +negotiated +rover +ferrari +continuation +directorate +augusta +stevenson +counterpart +gore +##rda +nursery +rican +ave +collectively +broadly +pastoral +repertoire +asserted +discovering +nordic +styled +fiba +cunningham +harley +middlesex +survives +tumor +tempo +zack +aiming +lok +urgent +##rade +##nto +devils +##ement +contractor +turin +##wl +##ool +bliss +repaired +simmons +moan +astronomical +cr +negotiate +lyric +1890s +lara +bred +clad +angus +pbs +##ience +engineered +posed +##lk +hernandez +possessions +elbows +psychiatric +strokes +confluence +electorate +lifts +campuses +lava +alps +##ep +##ution +##date +physicist +woody +##page +##ographic +##itis +juliet +reformation +sparhawk +320 +complement +suppressed +jewel +##½ +floated +##kas +continuity +sadly +##ische +inability +melting +scanning +paula +flour +judaism +safer +vague +##lm +solving +curb +##stown +financially +gable +bees +expired +miserable +cassidy +dominion +1789 +cupped +145 +robbery +facto +amos +warden +resume +tallest +marvin +ing +pounded +usd +declaring +gasoline +##aux +darkened +270 +650 +sophomore +##mere +erection +gossip +televised +risen +dial +##eu +pillars +##link +passages +profound +##tina +arabian +ashton +silicon +nail +##ead +##lated +##wer +##hardt +fleming +firearms +ducked +circuits +blows +waterloo +titans +##lina +atom +fireplace +cheshire +financed +activation +algorithms +##zzi +constituent +catcher +cherokee +partnerships +sexuality +platoon +tragic +vivian +guarded +whiskey +meditation +poetic +##late +##nga +##ake +porto +listeners +dominance +kendra +mona +chandler +factions +22nd +salisbury +attitudes +derivative +##ido +##haus +intake +paced +javier +illustrator +barrels +bias +cockpit +burnett +dreamed +ensuing +##anda +receptors +someday +hawkins +mattered +##lal +slavic +1799 +jesuit +cameroon +wasted +tai +wax +lowering +victorious +freaking +outright +hancock +librarian +sensing +bald +calcium +myers +tablet +announcing +barack +shipyard +pharmaceutical +##uan +greenwich +flush +medley +patches +wolfgang +pt +speeches +acquiring +exams +nikolai +##gg +hayden +kannada +##type +reilly +##pt +waitress +abdomen +devastated +capped +pseudonym +pharmacy +fulfill +paraguay +1796 +clicked +##trom +archipelago +syndicated +##hman +lumber +orgasm +rejection +clifford +lorraine +advent +mafia +rodney +brock +##ght +##used +##elia +cassette +chamberlain +despair +mongolia +sensors +developmental +upstream +##eg +##alis +spanning +165 +trombone +basque +seeded +interred +renewable +rhys +leapt +revision +molecule +##ages +chord +vicious +nord +shivered +23rd +arlington +debts +corpus +sunrise +bays +blackburn +centimetres +##uded +shuddered +gm +strangely +gripping +cartoons +isabelle +orbital +##ppa +seals +proving +##lton +refusal +strengthened +bust +assisting +baghdad +batsman +portrayal +mara +pushes +spears +og +##cock +reside +nathaniel +brennan +1776 +confirmation +caucus +##worthy +markings +yemen +nobles +ku +lazy +viewer +catalan +encompasses +sawyer +##fall +sparked +substances +patents +braves +arranger +evacuation +sergio +persuade +dover +tolerance +penguin +cum +jockey +insufficient +townships +occupying +declining +plural +processed +projection +puppet +flanders +introduces +liability +##yon +gymnastics +antwerp +taipei +hobart +candles +jeep +wes +observers +126 +chaplain +bundle +glorious +##hine +hazel +flung +sol +excavations +dumped +stares +sh +bangalore +triangular +icelandic +intervals +expressing +turbine +##vers +songwriting +crafts +##igo +jasmine +ditch +rite +##ways +entertaining +comply +sorrow +wrestlers +basel +emirates +marian +rivera +helpful +##some +caution +downward +networking +##atory +##tered +darted +genocide +emergence +replies +specializing +spokesman +convenient +unlocked +fading +augustine +concentrations +resemblance +elijah +investigator +andhra +##uda +promotes +bean +##rrell +fleeing +wan +simone +announcer +##ame +##bby +lydia +weaver +132 +residency +modification +##fest +stretches +##ast +alternatively +nat +lowe +lacks +##ented +pam +tile +concealed +inferior +abdullah +residences +tissues +vengeance +##ided +moisture +peculiar +groove +zip +bologna +jennings +ninja +oversaw +zombies +pumping +batch +livingston +emerald +installations +1797 +peel +nitrogen +rama +##fying +##star +schooling +strands +responding +werner +##ost +lime +casa +accurately +targeting +##rod +underway +##uru +hemisphere +lester +##yard +occupies +2d +griffith +angrily +reorganized +##owing +courtney +deposited +##dd +##30 +estadio +##ifies +dunn +exiled +##ying +checks +##combe +##о +##fly +successes +unexpectedly +blu +assessed +##flower +##ه +observing +sacked +spiders +kn +##tail +mu +nodes +prosperity +audrey +divisional +155 +broncos +tangled +adjust +feeds +erosion +paolo +surf +directory +snatched +humid +admiralty +screwed +gt +reddish +##nese +modules +trench +lamps +bind +leah +bucks +competes +##nz +##form +transcription +##uc +isles +violently +clutching +pga +cyclist +inflation +flats +ragged +unnecessary +##hian +stubborn +coordinated +harriet +baba +disqualified +330 +insect +wolfe +##fies +reinforcements +rocked +duel +winked +embraced +bricks +##raj +hiatus +defeats +pending +brightly +jealousy +##xton +##hm +##uki +lena +gdp +colorful +##dley +stein +kidney +##shu +underwear +wanderers +##haw +##icus +guardians +m³ +roared +habits +##wise +permits +gp +uranium +punished +disguise +bundesliga +elise +dundee +erotic +partisan +pi +collectors +float +individually +rendering +behavioral +bucharest +ser +hare +valerie +corporal +nutrition +proportional +##isa +immense +##kis +pavement +##zie +##eld +sutherland +crouched +1775 +##lp +suzuki +trades +endurance +operas +crosby +prayed +priory +rory +socially +##urn +gujarat +##pu +walton +cube +pasha +privilege +lennon +floods +thorne +waterfall +nipple +scouting +approve +##lov +minorities +voter +dwight +extensions +assure +ballroom +slap +dripping +privileges +rejoined +confessed +demonstrating +patriotic +yell +investor +##uth +pagan +slumped +squares +##cle +##kins +confront +bert +embarrassment +##aid +aston +urging +sweater +starr +yuri +brains +williamson +commuter +mortar +structured +selfish +exports +##jon +cds +##him +unfinished +##rre +mortgage +destinations +##nagar +canoe +solitary +buchanan +delays +magistrate +fk +##pling +motivation +##lier +##vier +recruiting +assess +##mouth +malik +antique +1791 +pius +rahman +reich +tub +zhou +smashed +airs +galway +xii +conditioning +honduras +discharged +dexter +##pf +lionel +129 +debates +lemon +tiffany +volunteered +dom +dioxide +procession +devi +sic +tremendous +advertisements +colts +transferring +verdict +hanover +decommissioned +utter +relate +pac +racism +##top +beacon +limp +similarity +terra +occurrence +ant +##how +becky +capt +updates +armament +richie +pal +##graph +halloween +mayo +##ssen +##bone +cara +serena +fcc +dolls +obligations +##dling +violated +lafayette +jakarta +exploitation +##ime +infamous +iconic +##lah +##park +kitty +moody +reginald +dread +spill +crystals +olivier +modeled +bluff +equilibrium +separating +notices +ordnance +extinction +onset +cosmic +attachment +sammy +expose +privy +anchored +##bil +abbott +admits +bending +baritone +emmanuel +policeman +vaughan +winged +climax +dresses +denny +polytechnic +mohamed +burmese +authentic +nikki +genetics +grandparents +homestead +gaza +postponed +metacritic +una +##sby +##bat +unstable +dissertation +##rial +##cian +curls +obscure +uncovered +bronx +praying +disappearing +##hoe +prehistoric +coke +turret +mutations +nonprofit +pits +monaco +##ي +##usion +prominently +dispatched +podium +##mir +uci +##uation +133 +fortifications +birthplace +kendall +##lby +##oll +preacher +rack +goodman +##rman +persistent +##ott +countless +jaime +recorder +lexington +persecution +jumps +renewal +wagons +##11 +crushing +##holder +decorations +##lake +abundance +wrath +laundry +£1 +garde +##rp +jeanne +beetles +peasant +##sl +splitting +caste +sergei +##rer +##ema +scripts +##ively +rub +satellites +##vor +inscribed +verlag +scrapped +gale +packages +chick +potato +slogan +kathleen +arabs +##culture +counterparts +reminiscent +choral +##tead +rand +retains +bushes +dane +accomplish +courtesy +closes +##oth +slaughter +hague +krakow +lawson +tailed +elias +ginger +##ttes +canopy +betrayal +rebuilding +turf +##hof +frowning +allegiance +brigades +kicks +rebuild +polls +alias +nationalism +td +rowan +audition +bowie +fortunately +recognizes +harp +dillon +horrified +##oro +renault +##tics +ropes +##α +presumed +rewarded +infrared +wiping +accelerated +illustration +##rid +presses +practitioners +badminton +##iard +detained +##tera +recognizing +relates +misery +##sies +##tly +reproduction +piercing +potatoes +thornton +esther +manners +hbo +##aan +ours +bullshit +ernie +perennial +sensitivity +illuminated +rupert +##jin +##iss +##ear +rfc +nassau +##dock +staggered +socialism +##haven +appointments +nonsense +prestige +sharma +haul +##tical +solidarity +gps +##ook +##rata +igor +pedestrian +##uit +baxter +tenants +wires +medication +unlimited +guiding +impacts +diabetes +##rama +sasha +pas +clive +extraction +131 +continually +constraints +##bilities +sonata +hunted +sixteenth +chu +planting +quote +mayer +pretended +abs +spat +##hua +ceramic +##cci +curtains +pigs +pitching +##dad +latvian +sore +dayton +##sted +##qi +patrols +slice +playground +##nted +shone +stool +apparatus +inadequate +mates +treason +##ija +desires +##liga +##croft +somalia +laurent +mir +leonardo +oracle +grape +obliged +chevrolet +thirteenth +stunning +enthusiastic +##ede +accounted +concludes +currents +basil +##kovic +drought +##rica +mai +##aire +shove +posting +##shed +pilgrimage +humorous +packing +fry +pencil +wines +smells +144 +marilyn +aching +newest +clung +bon +neighbours +sanctioned +##pie +mug +##stock +drowning +##mma +hydraulic +##vil +hiring +reminder +lilly +investigators +##ncies +sour +##eous +compulsory +packet +##rion +##graphic +##elle +cannes +##inate +depressed +##rit +heroic +importantly +theresa +##tled +conway +saturn +marginal +rae +##xia +corresponds +royce +pact +jasper +explosives +packaging +aluminium +##ttered +denotes +rhythmic +spans +assignments +hereditary +outlined +originating +sundays +lad +reissued +greeting +beatrice +##dic +pillar +marcos +plots +handbook +alcoholic +judiciary +avant +slides +extract +masculine +blur +##eum +##force +homage +trembled +owens +hymn +trey +omega +signaling +socks +accumulated +reacted +attic +theo +lining +angie +distraction +primera +talbot +##key +1200 +ti +creativity +billed +##hey +deacon +eduardo +identifies +proposition +dizzy +gunner +hogan +##yam +##pping +##hol +ja +##chan +jensen +reconstructed +##berger +clearance +darius +##nier +abe +harlem +plea +dei +circled +emotionally +notation +fascist +neville +exceeded +upwards +viable +ducks +##fo +workforce +racer +limiting +shri +##lson +possesses +1600 +kerr +moths +devastating +laden +disturbing +locking +##cture +gal +fearing +accreditation +flavor +aide +1870s +mountainous +##baum +melt +##ures +motel +texture +servers +soda +##mb +herd +##nium +erect +puzzled +hum +peggy +examinations +gould +testified +geoff +ren +devised +sacks +##law +denial +posters +grunted +cesar +tutor +ec +gerry +offerings +byrne +falcons +combinations +ct +incoming +pardon +rocking +26th +avengers +flared +mankind +seller +uttar +loch +nadia +stroking +exposing +##hd +fertile +ancestral +instituted +##has +noises +prophecy +taxation +eminent +vivid +pol +##bol +dart +indirect +multimedia +notebook +upside +displaying +adrenaline +referenced +geometric +##iving +progression +##ddy +blunt +announce +##far +implementing +##lav +aggression +liaison +cooler +cares +headache +plantations +gorge +dots +impulse +thickness +ashamed +averaging +kathy +obligation +precursor +137 +fowler +symmetry +thee +225 +hears +##rai +undergoing +ads +butcher +bowler +##lip +cigarettes +subscription +goodness +##ically +browne +##hos +##tech +kyoto +donor +##erty +damaging +friction +drifting +expeditions +hardened +prostitution +152 +fauna +blankets +claw +tossing +snarled +butterflies +recruits +investigative +coated +healed +138 +communal +hai +xiii +academics +boone +psychologist +restless +lahore +stephens +mba +brendan +foreigners +printer +##pc +ached +explode +27th +deed +scratched +dared +##pole +cardiac +1780 +okinawa +proto +commando +compelled +oddly +electrons +##base +replica +thanksgiving +##rist +sheila +deliberate +stafford +tidal +representations +hercules +ou +##path +##iated +kidnapping +lenses +##tling +deficit +samoa +mouths +consuming +computational +maze +granting +smirk +razor +fixture +ideals +inviting +aiden +nominal +##vs +issuing +julio +pitt +ramsey +docks +##oss +exhaust +##owed +bavarian +draped +anterior +mating +ethiopian +explores +noticing +##nton +discarded +convenience +hoffman +endowment +beasts +cartridge +mormon +paternal +probe +sleeves +interfere +lump +deadline +##rail +jenks +bulldogs +scrap +alternating +justified +reproductive +nam +seize +descending +secretariat +kirby +coupe +grouped +smash +panther +sedan +tapping +##18 +lola +cheer +germanic +unfortunate +##eter +unrelated +##fan +subordinate +##sdale +suzanne +advertisement +##ility +horsepower +##lda +cautiously +discourse +luigi +##mans +##fields +noun +prevalent +mao +schneider +everett +surround +governorate +kira +##avia +westward +##take +misty +rails +sustainability +134 +unused +##rating +packs +toast +unwilling +regulate +thy +suffrage +nile +awe +assam +definitions +travelers +affordable +##rb +conferred +sells +undefeated +beneficial +torso +basal +repeating +remixes +##pass +bahrain +cables +fang +##itated +excavated +numbering +statutory +##rey +deluxe +##lian +forested +ramirez +derbyshire +zeus +slamming +transfers +astronomer +banana +lottery +berg +histories +bamboo +##uchi +resurrection +posterior +bowls +vaguely +##thi +thou +preserving +tensed +offence +##inas +meyrick +callum +ridden +watt +langdon +tying +lowland +snorted +daring +truman +##hale +##girl +aura +overly +filing +weighing +goa +infections +philanthropist +saunders +eponymous +##owski +latitude +perspectives +reviewing +mets +commandant +radial +##kha +flashlight +reliability +koch +vowels +amazed +ada +elaine +supper +##rth +##encies +predator +debated +soviets +cola +##boards +##nah +compartment +crooked +arbitrary +fourteenth +##ctive +havana +majors +steelers +clips +profitable +ambush +exited +packers +##tile +nude +cracks +fungi +##е +limb +trousers +josie +shelby +tens +frederic +##ος +definite +smoothly +constellation +insult +baton +discs +lingering +##nco +conclusions +lent +staging +becker +grandpa +shaky +##tron +einstein +obstacles +sk +adverse +elle +economically +##moto +mccartney +thor +dismissal +motions +readings +nostrils +treatise +##pace +squeezing +evidently +prolonged +1783 +venezuelan +je +marguerite +beirut +takeover +shareholders +##vent +denise +digit +airplay +norse +##bbling +imaginary +pills +hubert +blaze +vacated +eliminating +##ello +vine +mansfield +##tty +retrospective +barrow +borne +clutch +bail +forensic +weaving +##nett +##witz +desktop +citadel +promotions +worrying +dorset +ieee +subdivided +##iating +manned +expeditionary +pickup +synod +chuckle +185 +barney +##rz +##ffin +functionality +karachi +litigation +meanings +uc +lick +turbo +anders +##ffed +execute +curl +oppose +ankles +typhoon +##د +##ache +##asia +linguistics +compassion +pressures +grazing +perfection +##iting +immunity +monopoly +muddy +backgrounds +136 +namibia +francesca +monitors +attracting +stunt +tuition +##ии +vegetable +##mates +##quent +mgm +jen +complexes +forts +##ond +cellar +bites +seventeenth +royals +flemish +failures +mast +charities +##cular +peruvian +capitals +macmillan +ipswich +outward +frigate +postgraduate +folds +employing +##ouse +concurrently +fiery +##tai +contingent +nightmares +monumental +nicaragua +##kowski +lizard +mal +fielding +gig +reject +##pad +harding +##ipe +coastline +##cin +##nos +beethoven +humphrey +innovations +##tam +##nge +norris +doris +solicitor +huang +obey +141 +##lc +niagara +##tton +shelves +aug +bourbon +curry +nightclub +specifications +hilton +##ndo +centennial +dispersed +worm +neglected +briggs +sm +font +kuala +uneasy +plc +##nstein +##bound +##aking +##burgh +awaiting +pronunciation +##bbed +##quest +eh +optimal +zhu +raped +greens +presided +brenda +worries +##life +venetian +marxist +turnout +##lius +refined +braced +sins +grasped +sunderland +nickel +speculated +lowell +cyrillic +communism +fundraising +resembling +colonists +mutant +freddie +usc +##mos +gratitude +##run +mural +##lous +chemist +wi +reminds +28th +steals +tess +pietro +##ingen +promoter +ri +microphone +honoured +rai +sant +##qui +feather +##nson +burlington +kurdish +terrorists +deborah +sickness +##wed +##eet +hazard +irritated +desperation +veil +clarity +##rik +jewels +xv +##gged +##ows +##cup +berkshire +unfair +mysteries +orchid +winced +exhaustion +renovations +stranded +obe +infinity +##nies +adapt +redevelopment +thanked +registry +olga +domingo +noir +tudor +ole +##atus +commenting +behaviors +##ais +crisp +pauline +probable +stirling +wigan +##bian +paralympics +panting +surpassed +##rew +luca +barred +pony +famed +##sters +cassandra +waiter +carolyn +exported +##orted +andres +destructive +deeds +jonah +castles +vacancy +suv +##glass +1788 +orchard +yep +famine +belarusian +sprang +##forth +skinny +##mis +administrators +rotterdam +zambia +zhao +boiler +discoveries +##ride +##physics +lucius +disappointing +outreach +spoon +##frame +qualifications +unanimously +enjoys +regency +##iidae +stade +realism +veterinary +rodgers +dump +alain +chestnut +castile +censorship +rumble +gibbs +##itor +communion +reggae +inactivated +logs +loads +##houses +homosexual +##iano +ale +informs +##cas +phrases +plaster +linebacker +ambrose +kaiser +fascinated +850 +limerick +recruitment +forge +mastered +##nding +leinster +rooted +threaten +##strom +borneo +##hes +suggestions +scholarships +propeller +documentaries +patronage +coats +constructing +invest +neurons +comet +entirety +shouts +identities +annoying +unchanged +wary +##antly +##ogy +neat +oversight +##kos +phillies +replay +constance +##kka +incarnation +humble +skies +minus +##acy +smithsonian +##chel +guerrilla +jar +cadets +##plate +surplus +audit +##aru +cracking +joanna +louisa +pacing +##lights +intentionally +##iri +diner +nwa +imprint +australians +tong +unprecedented +bunker +naive +specialists +ark +nichols +railing +leaked +pedal +##uka +shrub +longing +roofs +v8 +captains +neural +tuned +##ntal +##jet +emission +medina +frantic +codex +definitive +sid +abolition +intensified +stocks +enrique +sustain +genoa +oxide +##written +clues +cha +##gers +tributaries +fragment +venom +##rity +##ente +##sca +muffled +vain +sire +laos +##ingly +##hana +hastily +snapping +surfaced +sentiment +motive +##oft +contests +approximate +mesa +luckily +dinosaur +exchanges +propelled +accord +bourne +relieve +tow +masks +offended +##ues +cynthia +##mmer +rains +bartender +zinc +reviewers +lois +##sai +legged +arrogant +rafe +rosie +comprise +handicap +blockade +inlet +lagoon +copied +drilling +shelley +petals +##inian +mandarin +obsolete +##inated +onward +arguably +productivity +cindy +praising +seldom +busch +discusses +raleigh +shortage +ranged +stanton +encouragement +firstly +conceded +overs +temporal +##uke +cbe +##bos +woo +certainty +pumps +##pton +stalked +##uli +lizzie +periodic +thieves +weaker +##night +gases +shoving +chooses +wc +##chemical +prompting +weights +##kill +robust +flanked +sticky +hu +tuberculosis +##eb +##eal +christchurch +resembled +wallet +reese +inappropriate +pictured +distract +fixing +fiddle +giggled +burger +heirs +hairy +mechanic +torque +apache +obsessed +chiefly +cheng +logging +##tag +extracted +meaningful +numb +##vsky +gloucestershire +reminding +##bay +unite +##lit +breeds +diminished +clown +glove +1860s +##ن +##ug +archibald +focal +freelance +sliced +depiction +##yk +organism +switches +sights +stray +crawling +##ril +lever +leningrad +interpretations +loops +anytime +reel +alicia +delighted +##ech +inhaled +xiv +suitcase +bernie +vega +licenses +northampton +exclusion +induction +monasteries +racecourse +homosexuality +##right +##sfield +##rky +dimitri +michele +alternatives +ions +commentators +genuinely +objected +pork +hospitality +fencing +stephan +warships +peripheral +wit +drunken +wrinkled +quentin +spends +departing +chung +numerical +spokesperson +##zone +johannesburg +caliber +killers +##udge +assumes +neatly +demographic +abigail +bloc +##vel +mounting +##lain +bentley +slightest +xu +recipients +##jk +merlin +##writer +seniors +prisons +blinking +hindwings +flickered +kappa +##hel +80s +strengthening +appealing +brewing +gypsy +mali +lashes +hulk +unpleasant +harassment +bio +treaties +predict +instrumentation +pulp +troupe +boiling +mantle +##ffe +ins +##vn +dividing +handles +verbs +##onal +coconut +senegal +340 +thorough +gum +momentarily +##sto +cocaine +panicked +destined +##turing +teatro +denying +weary +captained +mans +##hawks +##code +wakefield +bollywood +thankfully +##16 +cyril +##wu +amendments +##bahn +consultation +stud +reflections +kindness +1787 +internally +##ovo +tex +mosaic +distribute +paddy +seeming +143 +##hic +piers +##15 +##mura +##verse +popularly +winger +kang +sentinel +mccoy +##anza +covenant +##bag +verge +fireworks +suppress +thrilled +dominate +##jar +swansea +##60 +142 +reconciliation +##ndi +stiffened +cue +dorian +##uf +damascus +amor +ida +foremost +##aga +porsche +unseen +dir +##had +##azi +stony +lexi +melodies +##nko +angular +integer +podcast +ants +inherent +jaws +justify +persona +##olved +josephine +##nr +##ressed +customary +flashes +gala +cyrus +glaring +backyard +ariel +physiology +greenland +html +stir +avon +atletico +finch +methodology +ked +##lent +mas +catholicism +townsend +branding +quincy +fits +containers +1777 +ashore +aragon +##19 +forearm +poisoning +##sd +adopting +conquer +grinding +amnesty +keller +finances +evaluate +forged +lankan +instincts +##uto +guam +bosnian +photographed +workplace +desirable +protector +##dog +allocation +intently +encourages +willy +##sten +bodyguard +electro +brighter +##ν +bihar +##chev +lasts +opener +amphibious +sal +verde +arte +##cope +captivity +vocabulary +yields +##tted +agreeing +desmond +pioneered +##chus +strap +campaigned +railroads +##ович +emblem +##dre +stormed +501 +##ulous +marijuana +northumberland +##gn +##nath +bowen +landmarks +beaumont +##qua +danube +##bler +attorneys +th +ge +flyers +critique +villains +cass +mutation +acc +##0s +colombo +mckay +motif +sampling +concluding +syndicate +##rell +neon +stables +ds +warnings +clint +mourning +wilkinson +##tated +merrill +leopard +evenings +exhaled +emil +sonia +ezra +discrete +stove +farrell +fifteenth +prescribed +superhero +##rier +worms +helm +wren +##duction +##hc +expo +##rator +hq +unfamiliar +antony +prevents +acceleration +fiercely +mari +painfully +calculations +cheaper +ign +clifton +irvine +davenport +mozambique +##np +pierced +##evich +wonders +##wig +##cate +##iling +crusade +ware +##uel +enzymes +reasonably +mls +##coe +mater +ambition +bunny +eliot +kernel +##fin +asphalt +headmaster +torah +aden +lush +pins +waived +##care +##yas +joao +substrate +enforce +##grad +##ules +alvarez +selections +epidemic +tempted +##bit +bremen +translates +ensured +waterfront +29th +forrest +manny +malone +kramer +reigning +cookies +simpler +absorption +205 +engraved +##ffy +evaluated +1778 +haze +146 +comforting +crossover +##abe +thorn +##rift +##imo +##pop +suppression +fatigue +cutter +##tr +201 +wurttemberg +##orf +enforced +hovering +proprietary +gb +samurai +syllable +ascent +lacey +tick +lars +tractor +merchandise +rep +bouncing +defendants +##yre +huntington +##ground +##oko +standardized +##hor +##hima +assassinated +nu +predecessors +rainy +liar +assurance +lyrical +##uga +secondly +flattened +ios +parameter +undercover +##mity +bordeaux +punish +ridges +markers +exodus +inactive +hesitate +debbie +nyc +pledge +savoy +nagar +offset +organist +##tium +hesse +marin +converting +##iver +diagram +propulsion +pu +validity +reverted +supportive +##dc +ministries +clans +responds +proclamation +##inae +##ø +##rea +ein +pleading +patriot +sf +birch +islanders +strauss +hates +##dh +brandenburg +concession +rd +##ob +1900s +killings +textbook +antiquity +cinematography +wharf +embarrassing +setup +creed +farmland +inequality +centred +signatures +fallon +370 +##ingham +##uts +ceylon +gazing +directive +laurie +##tern +globally +##uated +##dent +allah +excavation +threads +##cross +148 +frantically +icc +utilize +determines +respiratory +thoughtful +receptions +##dicate +merging +chandra +seine +147 +builders +builds +diagnostic +dev +visibility +goddamn +analyses +dhaka +cho +proves +chancel +concurrent +curiously +canadians +pumped +restoring +1850s +turtles +jaguar +sinister +spinal +traction +declan +vows +1784 +glowed +capitalism +swirling +install +universidad +##lder +##oat +soloist +##genic +##oor +coincidence +beginnings +nissan +dip +resorts +caucasus +combustion +infectious +##eno +pigeon +serpent +##itating +conclude +masked +salad +jew +##gr +surreal +toni +##wc +harmonica +151 +##gins +##etic +##coat +fishermen +intending +bravery +##wave +klaus +titan +wembley +taiwanese +ransom +40th +incorrect +hussein +eyelids +jp +cooke +dramas +utilities +##etta +##print +eisenhower +principally +granada +lana +##rak +openings +concord +##bl +bethany +connie +morality +sega +##mons +##nard +earnings +##kara +##cine +wii +communes +##rel +coma +composing +softened +severed +grapes +##17 +nguyen +analyzed +warlord +hubbard +heavenly +behave +slovenian +##hit +##ony +hailed +filmmakers +trance +caldwell +skye +unrest +coward +likelihood +##aging +bern +sci +taliban +honolulu +propose +##wang +1700 +browser +imagining +cobra +contributes +dukes +instinctively +conan +violinist +##ores +accessories +gradual +##amp +quotes +sioux +##dating +undertake +intercepted +sparkling +compressed +139 +fungus +tombs +haley +imposing +rests +degradation +lincolnshire +retailers +wetlands +tulsa +distributor +dungeon +nun +greenhouse +convey +atlantis +aft +exits +oman +dresser +lyons +##sti +joking +eddy +judgement +omitted +digits +##cts +##game +juniors +##rae +cents +stricken +une +##ngo +wizards +weir +breton +nan +technician +fibers +liking +royalty +##cca +154 +persia +terribly +magician +##rable +##unt +vance +cafeteria +booker +camille +warmer +##static +consume +cavern +gaps +compass +contemporaries +foyer +soothing +graveyard +maj +plunged +blush +##wear +cascade +demonstrates +ordinance +##nov +boyle +##lana +rockefeller +shaken +banjo +izzy +##ense +breathless +vines +##32 +##eman +alterations +chromosome +dwellings +feudal +mole +153 +catalonia +relics +tenant +mandated +##fm +fridge +hats +honesty +patented +raul +heap +cruisers +accusing +enlightenment +infants +wherein +chatham +contractors +zen +affinity +hc +osborne +piston +156 +traps +maturity +##rana +lagos +##zal +peering +##nay +attendant +dealers +protocols +subset +prospects +biographical +##cre +artery +##zers +insignia +nuns +endured +##eration +recommend +schwartz +serbs +berger +cromwell +crossroads +##ctor +enduring +clasped +grounded +##bine +marseille +twitched +abel +choke +https +catalyst +moldova +italians +##tist +disastrous +wee +##oured +##nti +wwf +nope +##piration +##asa +expresses +thumbs +167 +##nza +coca +1781 +cheating +##ption +skipped +sensory +heidelberg +spies +satan +dangers +semifinal +202 +bohemia +whitish +confusing +shipbuilding +relies +surgeons +landings +ravi +baku +moor +suffix +alejandro +##yana +litre +upheld +##unk +rajasthan +##rek +coaster +insists +posture +scenarios +etienne +favoured +appoint +transgender +elephants +poked +greenwood +defences +fulfilled +militant +somali +1758 +chalk +potent +##ucci +migrants +wink +assistants +nos +restriction +activism +niger +##ario +colon +shaun +##sat +daphne +##erated +swam +congregations +reprise +considerations +magnet +playable +xvi +##р +overthrow +tobias +knob +chavez +coding +##mers +propped +katrina +orient +newcomer +##suke +temperate +##pool +farmhouse +interrogation +##vd +committing +##vert +forthcoming +strawberry +joaquin +macau +ponds +shocking +siberia +##cellular +chant +contributors +##nant +##ologists +sped +absorb +hail +1782 +spared +##hore +barbados +karate +opus +originates +saul +##xie +evergreen +leaped +##rock +correlation +exaggerated +weekday +unification +bump +tracing +brig +afb +pathways +utilizing +##ners +mod +mb +disturbance +kneeling +##stad +##guchi +100th +pune +##thy +decreasing +168 +manipulation +miriam +academia +ecosystem +occupational +rbi +##lem +rift +##14 +rotary +stacked +incorporation +awakening +generators +guerrero +racist +##omy +cyber +derivatives +culminated +allie +annals +panzer +sainte +wikipedia +pops +zu +austro +##vate +algerian +politely +nicholson +mornings +educate +tastes +thrill +dartmouth +##gating +db +##jee +regan +differing +concentrating +choreography +divinity +##media +pledged +alexandre +routing +gregor +madeline +##idal +apocalypse +##hora +gunfire +culminating +elves +fined +liang +lam +programmed +tar +guessing +transparency +gabrielle +##gna +cancellation +flexibility +##lining +accession +shea +stronghold +nets +specializes +##rgan +abused +hasan +sgt +ling +exceeding +##₄ +admiration +supermarket +##ark +photographers +specialised +tilt +resonance +hmm +perfume +380 +sami +threatens +garland +botany +guarding +boiled +greet +puppy +russo +supplier +wilmington +vibrant +vijay +##bius +paralympic +grumbled +paige +faa +licking +margins +hurricanes +##gong +fest +grenade +ripping +##uz +counseling +weigh +##sian +needles +wiltshire +edison +costly +##not +fulton +tramway +redesigned +staffordshire +cache +gasping +watkins +sleepy +candidacy +##group +monkeys +timeline +throbbing +##bid +##sos +berth +uzbekistan +vanderbilt +bothering +overturned +ballots +gem +##iger +sunglasses +subscribers +hooker +compelling +ang +exceptionally +saloon +stab +##rdi +carla +terrifying +rom +##vision +coil +##oids +satisfying +vendors +31st +mackay +deities +overlooked +ambient +bahamas +felipe +olympia +whirled +botanist +advertised +tugging +##dden +disciples +morales +unionist +rites +foley +morse +motives +creepy +##₀ +soo +##sz +bargain +highness +frightening +turnpike +tory +reorganization +##cer +depict +biographer +##walk +unopposed +manifesto +##gles +institut +emile +accidental +kapoor +##dam +kilkenny +cortex +lively +##13 +romanesque +jain +shan +cannons +##ood +##ske +petrol +echoing +amalgamated +disappears +cautious +proposes +sanctions +trenton +##ر +flotilla +aus +contempt +tor +canary +cote +theirs +##hun +conceptual +deleted +fascinating +paso +blazing +elf +honourable +hutchinson +##eiro +##outh +##zin +surveyor +tee +amidst +wooded +reissue +intro +##ono +cobb +shelters +newsletter +hanson +brace +encoding +confiscated +dem +caravan +marino +scroll +melodic +cows +imam +##adi +##aneous +northward +searches +biodiversity +cora +310 +roaring +##bers +connell +theologian +halo +compose +pathetic +unmarried +dynamo +##oot +az +calculation +toulouse +deserves +humour +nr +forgiveness +tam +undergone +martyr +pamela +myths +whore +counselor +hicks +290 +heavens +battleship +electromagnetic +##bbs +stellar +establishments +presley +hopped +##chin +temptation +90s +wills +nas +##yuan +nhs +##nya +seminars +##yev +adaptations +gong +asher +lex +indicator +sikh +tobago +cites +goin +##yte +satirical +##gies +characterised +correspond +bubbles +lure +participates +##vid +eruption +skate +therapeutic +1785 +canals +wholesale +defaulted +sac +460 +petit +##zzled +virgil +leak +ravens +256 +portraying +##yx +ghetto +creators +dams +portray +vicente +##rington +fae +namesake +bounty +##arium +joachim +##ota +##iser +aforementioned +axle +snout +depended +dismantled +reuben +480 +##ibly +gallagher +##lau +##pd +earnest +##ieu +##iary +inflicted +objections +##llar +asa +gritted +##athy +jericho +##sea +##was +flick +underside +ceramics +undead +substituted +195 +eastward +undoubtedly +wheeled +chimney +##iche +guinness +cb +##ager +siding +##bell +traitor +baptiste +disguised +inauguration +149 +tipperary +choreographer +perched +warmed +stationary +eco +##ike +##ntes +bacterial +##aurus +flores +phosphate +##core +attacker +invaders +alvin +intersects +a1 +indirectly +immigrated +businessmen +cornelius +valves +narrated +pill +sober +ul +nationale +monastic +applicants +scenery +##jack +161 +motifs +constitutes +cpu +##osh +jurisdictions +sd +tuning +irritation +woven +##uddin +fertility +gao +##erie +antagonist +impatient +glacial +hides +boarded +denominations +interception +##jas +cookie +nicola +##tee +algebraic +marquess +bahn +parole +buyers +bait +turbines +paperwork +bestowed +natasha +renee +oceans +purchases +157 +vaccine +215 +##tock +fixtures +playhouse +integrate +jai +oswald +intellectuals +##cky +booked +nests +mortimer +##isi +obsession +sept +##gler +##sum +440 +scrutiny +simultaneous +squinted +##shin +collects +oven +shankar +penned +remarkably +##я +slips +luggage +spectral +1786 +collaborations +louie +consolidation +##ailed +##ivating +420 +hoover +blackpool +harness +ignition +vest +tails +belmont +mongol +skinner +##nae +visually +mage +derry +##tism +##unce +stevie +transitional +##rdy +redskins +drying +prep +prospective +##21 +annoyance +oversee +##loaded +fills +##books +##iki +announces +fda +scowled +respects +prasad +mystic +tucson +##vale +revue +springer +bankrupt +1772 +aristotle +salvatore +habsburg +##geny +dal +natal +nut +pod +chewing +darts +moroccan +walkover +rosario +lenin +punjabi +##ße +grossed +scattering +wired +invasive +hui +polynomial +corridors +wakes +gina +portrays +##cratic +arid +retreating +erich +irwin +sniper +##dha +linen +lindsey +maneuver +butch +shutting +socio +bounce +commemorative +postseason +jeremiah +pines +275 +mystical +beads +bp +abbas +furnace +bidding +consulted +assaulted +empirical +rubble +enclosure +sob +weakly +cancel +polly +yielded +##emann +curly +prediction +battered +70s +vhs +jacqueline +render +sails +barked +detailing +grayson +riga +sloane +raging +##yah +herbs +bravo +##athlon +alloy +giggle +imminent +suffers +assumptions +waltz +##itate +accomplishments +##ited +bathing +remixed +deception +prefix +##emia +deepest +##tier +##eis +balkan +frogs +##rong +slab +##pate +philosophers +peterborough +grains +imports +dickinson +rwanda +##atics +1774 +dirk +lan +tablets +##rove +clone +##rice +caretaker +hostilities +mclean +##gre +regimental +treasures +norms +impose +tsar +tango +diplomacy +variously +complain +192 +recognise +arrests +1779 +celestial +pulitzer +##dus +bing +libretto +##moor +adele +splash +##rite +expectation +lds +confronts +##izer +spontaneous +harmful +wedge +entrepreneurs +buyer +##ope +bilingual +translate +rugged +conner +circulated +uae +eaton +##gra +##zzle +lingered +lockheed +vishnu +reelection +alonso +##oom +joints +yankee +headline +cooperate +heinz +laureate +invading +##sford +echoes +scandinavian +##dham +hugging +vitamin +salute +micah +hind +trader +##sper +radioactive +##ndra +militants +poisoned +ratified +remark +campeonato +deprived +wander +prop +##dong +outlook +##tani +##rix +##eye +chiang +darcy +##oping +mandolin +spice +statesman +babylon +182 +walled +forgetting +afro +##cap +158 +giorgio +buffer +##polis +planetary +##gis +overlap +terminals +kinda +centenary +##bir +arising +manipulate +elm +ke +1770 +ak +##tad +chrysler +mapped +moose +pomeranian +quad +macarthur +assemblies +shoreline +recalls +stratford +##rted +noticeable +##evic +imp +##rita +##sque +accustomed +supplying +tents +disgusted +vogue +sipped +filters +khz +reno +selecting +luftwaffe +mcmahon +tyne +masterpiece +carriages +collided +dunes +exercised +flare +remembers +muzzle +##mobile +heck +##rson +burgess +lunged +middleton +boycott +bilateral +##sity +hazardous +lumpur +multiplayer +spotlight +jackets +goldman +liege +porcelain +rag +waterford +benz +attracts +hopeful +battling +ottomans +kensington +baked +hymns +cheyenne +lattice +levine +borrow +polymer +clashes +michaels +monitored +commitments +denounced +##25 +##von +cavity +##oney +hobby +akin +##holders +futures +intricate +cornish +patty +##oned +illegally +dolphin +##lag +barlow +yellowish +maddie +apologized +luton +plagued +##puram +nana +##rds +sway +fanny +łodz +##rino +psi +suspicions +hanged +##eding +initiate +charlton +##por +nak +competent +235 +analytical +annex +wardrobe +reservations +##rma +sect +162 +fairfax +hedge +piled +buckingham +uneven +bauer +simplicity +snyder +interpret +accountability +donors +moderately +byrd +continents +##cite +##max +disciple +hr +jamaican +ping +nominees +##uss +mongolian +diver +attackers +eagerly +ideological +pillows +miracles +apartheid +revolver +sulfur +clinics +moran +163 +##enko +ile +katy +rhetoric +##icated +chronology +recycling +##hrer +elongated +mughal +pascal +profiles +vibration +databases +domination +##fare +##rant +matthias +digest +rehearsal +polling +weiss +initiation +reeves +clinging +flourished +impress +ngo +##hoff +##ume +buckley +symposium +rhythms +weed +emphasize +transforming +##taking +##gence +##yman +accountant +analyze +flicker +foil +priesthood +voluntarily +decreases +##80 +##hya +slater +sv +charting +mcgill +##lde +moreno +##iu +besieged +zur +robes +##phic +admitting +api +deported +turmoil +peyton +earthquakes +##ares +nationalists +beau +clair +brethren +interrupt +welch +curated +galerie +requesting +164 +##ested +impending +steward +viper +##vina +complaining +beautifully +brandy +foam +nl +1660 +##cake +alessandro +punches +laced +explanations +##lim +attribute +clit +reggie +discomfort +##cards +smoothed +whales +##cene +adler +countered +duffy +disciplinary +widening +recipe +reliance +conducts +goats +gradient +preaching +##shaw +matilda +quasi +striped +meridian +cannabis +cordoba +certificates +##agh +##tering +graffiti +hangs +pilgrims +repeats +##ych +revive +urine +etat +##hawk +fueled +belts +fuzzy +susceptible +##hang +mauritius +salle +sincere +beers +hooks +##cki +arbitration +entrusted +advise +sniffed +seminar +junk +donnell +processors +principality +strapped +celia +mendoza +everton +fortunes +prejudice +starving +reassigned +steamer +##lund +tuck +evenly +foreman +##ffen +dans +375 +envisioned +slit +##xy +baseman +liberia +rosemary +##weed +electrified +periodically +potassium +stride +contexts +sperm +slade +mariners +influx +bianca +subcommittee +##rane +spilling +icao +estuary +##nock +delivers +iphone +##ulata +isa +mira +bohemian +dessert +##sbury +welcoming +proudly +slowing +##chs +musee +ascension +russ +##vian +waits +##psy +africans +exploit +##morphic +gov +eccentric +crab +peck +##ull +entrances +formidable +marketplace +groom +bolted +metabolism +patton +robbins +courier +payload +endure +##ifier +andes +refrigerator +##pr +ornate +##uca +ruthless +illegitimate +masonry +strasbourg +bikes +adobe +##³ +apples +quintet +willingly +niche +bakery +corpses +energetic +##cliffe +##sser +##ards +177 +centimeters +centro +fuscous +cretaceous +rancho +##yde +andrei +telecom +tottenham +oasis +ordination +vulnerability +presiding +corey +cp +penguins +sims +##pis +malawi +piss +##48 +correction +##cked +##ffle +##ryn +countdown +detectives +psychiatrist +psychedelic +dinosaurs +blouse +##get +choi +vowed +##oz +randomly +##pol +49ers +scrub +blanche +bruins +dusseldorf +##using +unwanted +##ums +212 +dominique +elevations +headlights +om +laguna +##oga +1750 +famously +ignorance +shrewsbury +##aine +ajax +breuning +che +confederacy +greco +overhaul +##screen +paz +skirts +disagreement +cruelty +jagged +phoebe +shifter +hovered +viruses +##wes +mandy +##lined +##gc +landlord +squirrel +dashed +##ι +ornamental +gag +wally +grange +literal +spurs +undisclosed +proceeding +yin +##text +billie +orphan +spanned +humidity +indy +weighted +presentations +explosions +lucian +##tary +vaughn +hindus +##anga +##hell +psycho +171 +daytona +protects +efficiently +rematch +sly +tandem +##oya +rebranded +impaired +hee +metropolis +peach +godfrey +diaspora +ethnicity +prosperous +gleaming +dar +grossing +playback +##rden +stripe +pistols +##tain +births +labelled +##cating +172 +rudy +alba +##onne +aquarium +hostility +##gb +##tase +shudder +sumatra +hardest +lakers +consonant +creeping +demos +homicide +capsule +zeke +liberties +expulsion +pueblo +##comb +trait +transporting +##ddin +##neck +##yna +depart +gregg +mold +ledge +hangar +oldham +playboy +termination +analysts +gmbh +romero +##itic +insist +cradle +filthy +brightness +slash +shootout +deposed +bordering +##truct +isis +microwave +tumbled +sheltered +cathy +werewolves +messy +andersen +convex +clapped +clinched +satire +wasting +edo +vc +rufus +##jak +mont +##etti +poznan +##keeping +restructuring +transverse +##rland +azerbaijani +slovene +gestures +roommate +choking +shear +##quist +vanguard +oblivious +##hiro +disagreed +baptism +##lich +coliseum +##aceae +salvage +societe +cory +locke +relocation +relying +versailles +ahl +swelling +##elo +cheerful +##word +##edes +gin +sarajevo +obstacle +diverted +##nac +messed +thoroughbred +fluttered +utrecht +chewed +acquaintance +assassins +dispatch +mirza +##wart +nike +salzburg +swell +yen +##gee +idle +ligue +samson +##nds +##igh +playful +spawned +##cise +tease +##case +burgundy +##bot +stirring +skeptical +interceptions +marathi +##dies +bedrooms +aroused +pinch +##lik +preferences +tattoos +buster +digitally +projecting +rust +##ital +kitten +priorities +addison +pseudo +##guard +dusk +icons +sermon +##psis +##iba +bt +##lift +##xt +ju +truce +rink +##dah +##wy +defects +psychiatry +offences +calculate +glucose +##iful +##rized +##unda +francaise +##hari +richest +warwickshire +carly +1763 +purity +redemption +lending +##cious +muse +bruises +cerebral +aero +carving +##name +preface +terminology +invade +monty +##int +anarchist +blurred +##iled +rossi +treats +guts +shu +foothills +ballads +undertaking +premise +cecilia +affiliates +blasted +conditional +wilder +minors +drone +rudolph +buffy +swallowing +horton +attested +##hop +rutherford +howell +primetime +livery +penal +##bis +minimize +hydro +wrecked +wrought +palazzo +##gling +cans +vernacular +friedman +nobleman +shale +walnut +danielle +##ection +##tley +sears +##kumar +chords +lend +flipping +streamed +por +dracula +gallons +sacrifices +gamble +orphanage +##iman +mckenzie +##gible +boxers +daly +##balls +##ان +208 +##ific +##rative +##iq +exploited +slated +##uity +circling +hillary +pinched +goldberg +provost +campaigning +lim +piles +ironically +jong +mohan +successors +usaf +##tem +##ught +autobiographical +haute +preserves +##ending +acquitted +comparisons +203 +hydroelectric +gangs +cypriot +torpedoes +rushes +chrome +derive +bumps +instability +fiat +pets +##mbe +silas +dye +reckless +settler +##itation +info +heats +##writing +176 +canonical +maltese +fins +mushroom +stacy +aspen +avid +##kur +##loading +vickers +gaston +hillside +statutes +wilde +gail +kung +sabine +comfortably +motorcycles +##rgo +169 +pneumonia +fetch +##sonic +axel +faintly +parallels +##oop +mclaren +spouse +compton +interdisciplinary +miner +##eni +181 +clamped +##chal +##llah +separates +versa +##mler +scarborough +labrador +##lity +##osing +rutgers +hurdles +como +166 +burt +divers +##100 +wichita +cade +coincided +##erson +bruised +mla +##pper +vineyard +##ili +##brush +notch +mentioning +jase +hearted +kits +doe +##acle +pomerania +##ady +ronan +seizure +pavel +problematic +##zaki +domenico +##ulin +catering +penelope +dependence +parental +emilio +ministerial +atkinson +##bolic +clarkson +chargers +colby +grill +peeked +arises +summon +##aged +fools +##grapher +faculties +qaeda +##vial +garner +refurbished +##hwa +geelong +disasters +nudged +bs +shareholder +lori +algae +reinstated +rot +##ades +##nous +invites +stainless +183 +inclusive +##itude +diocesan +til +##icz +denomination +##xa +benton +floral +registers +##ider +##erman +##kell +absurd +brunei +guangzhou +hitter +retaliation +##uled +##eve +blanc +nh +consistency +contamination +##eres +##rner +dire +palermo +broadcasters +diaries +inspire +vols +brewer +tightening +ky +mixtape +hormone +##tok +stokes +##color +##dly +##ssi +pg +##ometer +##lington +sanitation +##tility +intercontinental +apps +##adt +¹⁄₂ +cylinders +economies +favourable +unison +croix +gertrude +odyssey +vanity +dangling +##logists +upgrades +dice +middleweight +practitioner +##ight +206 +henrik +parlor +orion +angered +lac +python +blurted +##rri +sensual +intends +swings +angled +##phs +husky +attain +peerage +precinct +textiles +cheltenham +shuffled +dai +confess +tasting +bhutan +##riation +tyrone +segregation +abrupt +ruiz +##rish +smirked +blackwell +confidential +browning +amounted +##put +vase +scarce +fabulous +raided +staple +guyana +unemployed +glider +shay +##tow +carmine +troll +intervene +squash +superstar +##uce +cylindrical +len +roadway +researched +handy +##rium +##jana +meta +lao +declares +##rring +##tadt +##elin +##kova +willem +shrubs +napoleonic +realms +skater +qi +volkswagen +##ł +tad +hara +archaeologist +awkwardly +eerie +##kind +wiley +##heimer +##24 +titus +organizers +cfl +crusaders +lama +usb +vent +enraged +thankful +occupants +maximilian +##gaard +possessing +textbooks +##oran +collaborator +quaker +##ulo +avalanche +mono +silky +straits +isaiah +mustang +surged +resolutions +potomac +descend +cl +kilograms +plato +strains +saturdays +##olin +bernstein +##ype +holstein +ponytail +##watch +belize +conversely +heroine +perpetual +##ylus +charcoal +piedmont +glee +negotiating +backdrop +prologue +##jah +##mmy +pasadena +climbs +ramos +sunni +##holm +##tner +##tri +anand +deficiency +hertfordshire +stout +##avi +aperture +orioles +##irs +doncaster +intrigued +bombed +coating +otis +##mat +cocktail +##jit +##eto +amir +arousal +sar +##proof +##act +##ories +dixie +pots +##bow +whereabouts +159 +##fted +drains +bullying +cottages +scripture +coherent +fore +poe +appetite +##uration +sampled +##ators +##dp +derrick +rotor +jays +peacock +installment +##rro +advisors +##coming +rodeo +scotch +##mot +##db +##fen +##vant +ensued +rodrigo +dictatorship +martyrs +twenties +##н +towed +incidence +marta +rainforest +sai +scaled +##cles +oceanic +qualifiers +symphonic +mcbride +dislike +generalized +aubrey +colonization +##iation +##lion +##ssing +disliked +lublin +salesman +##ulates +spherical +whatsoever +sweating +avalon +contention +punt +severity +alderman +atari +##dina +##grant +##rop +scarf +seville +vertices +annexation +fairfield +fascination +inspiring +launches +palatinate +regretted +##rca +feral +##iom +elk +nap +olsen +reddy +yong +##leader +##iae +garment +transports +feng +gracie +outrage +viceroy +insides +##esis +breakup +grady +organizer +softer +grimaced +222 +murals +galicia +arranging +vectors +##rsten +bas +##sb +##cens +sloan +##eka +bitten +ara +fender +nausea +bumped +kris +banquet +comrades +detector +persisted +##llan +adjustment +endowed +cinemas +##shot +sellers +##uman +peek +epa +kindly +neglect +simpsons +talon +mausoleum +runaway +hangul +lookout +##cic +rewards +coughed +acquainted +chloride +##ald +quicker +accordion +neolithic +##qa +artemis +coefficient +lenny +pandora +tx +##xed +ecstasy +litter +segunda +chairperson +gemma +hiss +rumor +vow +nasal +antioch +compensate +patiently +transformers +##eded +judo +morrow +penis +posthumous +philips +bandits +husbands +denote +flaming +##any +##phones +langley +yorker +1760 +walters +##uo +##kle +gubernatorial +fatty +samsung +leroy +outlaw +##nine +unpublished +poole +jakob +##ᵢ +##ₙ +crete +distorted +superiority +##dhi +intercept +crust +mig +claus +crashes +positioning +188 +stallion +301 +frontal +armistice +##estinal +elton +aj +encompassing +camel +commemorated +malaria +woodward +calf +cigar +penetrate +##oso +willard +##rno +##uche +illustrate +amusing +convergence +noteworthy +##lma +##rva +journeys +realise +manfred +##sable +410 +##vocation +hearings +fiance +##posed +educators +provoked +adjusting +##cturing +modular +stockton +paterson +vlad +rejects +electors +selena +maureen +##tres +uber +##rce +swirled +##num +proportions +nanny +pawn +naturalist +parma +apostles +awoke +ethel +wen +##bey +monsoon +overview +##inating +mccain +rendition +risky +adorned +##ih +equestrian +germain +nj +conspicuous +confirming +##yoshi +shivering +##imeter +milestone +rumours +flinched +bounds +smacked +token +##bei +lectured +automobiles +##shore +impacted +##iable +nouns +nero +##leaf +ismail +prostitute +trams +##lace +bridget +sud +stimulus +impressions +reins +revolves +##oud +##gned +giro +honeymoon +##swell +criterion +##sms +##uil +libyan +prefers +##osition +211 +preview +sucks +accusation +bursts +metaphor +diffusion +tolerate +faye +betting +cinematographer +liturgical +specials +bitterly +humboldt +##ckle +flux +rattled +##itzer +archaeologists +odor +authorised +marshes +discretion +##ов +alarmed +archaic +inverse +##leton +explorers +##pine +drummond +tsunami +woodlands +##minate +##tland +booklet +insanity +owning +insert +crafted +calculus +##tore +receivers +##bt +stung +##eca +##nched +prevailing +travellers +eyeing +lila +graphs +##borne +178 +julien +##won +morale +adaptive +therapist +erica +cw +libertarian +bowman +pitches +vita +##ional +crook +##ads +##entation +caledonia +mutiny +##sible +1840s +automation +##ß +flock +##pia +ironic +pathology +##imus +remarried +##22 +joker +withstand +energies +##att +shropshire +hostages +madeleine +tentatively +conflicting +mateo +recipes +euros +ol +mercenaries +nico +##ndon +albuquerque +augmented +mythical +bel +freud +##child +cough +##lica +365 +freddy +lillian +genetically +nuremberg +calder +209 +bonn +outdoors +paste +suns +urgency +vin +restraint +tyson +##cera +##selle +barrage +bethlehem +kahn +##par +mounts +nippon +barony +happier +ryu +makeshift +sheldon +blushed +castillo +barking +listener +taped +bethel +fluent +headlines +pornography +rum +disclosure +sighing +mace +doubling +gunther +manly +##plex +rt +interventions +physiological +forwards +emerges +##tooth +##gny +compliment +rib +recession +visibly +barge +faults +connector +exquisite +prefect +##rlin +patio +##cured +elevators +brandt +italics +pena +173 +wasp +satin +ea +botswana +graceful +respectable +##jima +##rter +##oic +franciscan +generates +##dl +alfredo +disgusting +##olate +##iously +sherwood +warns +cod +promo +cheryl +sino +##ة +##escu +twitch +##zhi +brownish +thom +ortiz +##dron +densely +##beat +carmel +reinforce +##bana +187 +anastasia +downhill +vertex +contaminated +remembrance +harmonic +homework +##sol +fiancee +gears +olds +angelica +loft +ramsay +quiz +colliery +sevens +##cape +autism +##hil +walkway +##boats +ruben +abnormal +ounce +khmer +##bbe +zachary +bedside +morphology +punching +##olar +sparrow +convinces +##35 +hewitt +queer +remastered +rods +mabel +solemn +notified +lyricist +symmetric +##xide +174 +encore +passports +wildcats +##uni +baja +##pac +mildly +##ease +bleed +commodity +mounds +glossy +orchestras +##omo +damian +prelude +ambitions +##vet +awhile +remotely +##aud +asserts +imply +##iques +distinctly +modelling +remedy +##dded +windshield +dani +xiao +##endra +audible +powerplant +1300 +invalid +elemental +acquisitions +##hala +immaculate +libby +plata +smuggling +ventilation +denoted +minh +##morphism +430 +differed +dion +kelley +lore +mocking +sabbath +spikes +hygiene +drown +runoff +stylized +tally +liberated +aux +interpreter +righteous +aba +siren +reaper +pearce +millie +##cier +##yra +gaius +##iso +captures +##ttering +dorm +claudio +##sic +benches +knighted +blackness +##ored +discount +fumble +oxidation +routed +##ς +novak +perpendicular +spoiled +fracture +splits +##urt +pads +topology +##cats +axes +fortunate +offenders +protestants +esteem +221 +broadband +convened +frankly +hound +prototypes +isil +facilitated +keel +##sher +sahara +awaited +bubba +orb +prosecutors +186 +hem +520 +##xing +relaxing +remnant +romney +sorted +slalom +stefano +ulrich +##active +exemption +folder +pauses +foliage +hitchcock +epithet +204 +criticisms +##aca +ballistic +brody +hinduism +chaotic +youths +equals +##pala +pts +thicker +analogous +capitalist +improvised +overseeing +sinatra +ascended +beverage +##tl +straightforward +##kon +curran +##west +bois +325 +induce +surveying +emperors +sax +unpopular +##kk +cartoonist +fused +##mble +unto +##yuki +localities +##cko +##ln +darlington +slain +academie +lobbying +sediment +puzzles +##grass +defiance +dickens +manifest +tongues +alumnus +arbor +coincide +184 +appalachian +mustafa +examiner +cabaret +traumatic +yves +bracelet +draining +heroin +magnum +baths +odessa +consonants +mitsubishi +##gua +kellan +vaudeville +##fr +joked +null +straps +probation +##ław +ceded +interfaces +##pas +##zawa +blinding +viet +224 +rothschild +museo +640 +huddersfield +##vr +tactic +##storm +brackets +dazed +incorrectly +##vu +reg +glazed +fearful +manifold +benefited +irony +##sun +stumbling +##rte +willingness +balkans +mei +wraps +##aba +injected +##lea +gu +syed +harmless +##hammer +bray +takeoff +poppy +timor +cardboard +astronaut +purdue +weeping +southbound +cursing +stalls +diagonal +##neer +lamar +bryce +comte +weekdays +harrington +##uba +negatively +##see +lays +grouping +##cken +##henko +affirmed +halle +modernist +##lai +hodges +smelling +aristocratic +baptized +dismiss +justification +oilers +##now +coupling +qin +snack +healer +##qing +gardener +layla +battled +formulated +stephenson +gravitational +##gill +##jun +1768 +granny +coordinating +suites +##cd +##ioned +monarchs +##cote +##hips +sep +blended +apr +barrister +deposition +fia +mina +policemen +paranoid +##pressed +churchyard +covert +crumpled +creep +abandoning +tr +transmit +conceal +barr +understands +readiness +spire +##cology +##enia +##erry +610 +startling +unlock +vida +bowled +slots +##nat +##islav +spaced +trusting +admire +rig +##ink +slack +##70 +mv +207 +casualty +##wei +classmates +##odes +##rar +##rked +amherst +furnished +evolve +foundry +menace +mead +##lein +flu +wesleyan +##kled +monterey +webber +##vos +wil +##mith +##на +bartholomew +justices +restrained +##cke +amenities +191 +mediated +sewage +trenches +ml +mainz +##thus +1800s +##cula +##inski +caine +bonding +213 +converts +spheres +superseded +marianne +crypt +sweaty +ensign +historia +##br +spruce +##post +##ask +forks +thoughtfully +yukon +pamphlet +ames +##uter +karma +##yya +bryn +negotiation +sighs +incapable +##mbre +##ntial +actresses +taft +##mill +luce +prevailed +##amine +1773 +motionless +envoy +testify +investing +sculpted +instructors +provence +kali +cullen +horseback +##while +goodwin +##jos +gaa +norte +##ldon +modify +wavelength +abd +214 +skinned +sprinter +forecast +scheduling +marries +squared +tentative +##chman +boer +##isch +bolts +swap +fisherman +assyrian +impatiently +guthrie +martins +murdoch +194 +tanya +nicely +dolly +lacy +med +##45 +syn +decks +fashionable +millionaire +##ust +surfing +##ml +##ision +heaved +tammy +consulate +attendees +routinely +197 +fuse +saxophonist +backseat +malaya +##lord +scowl +tau +##ishly +193 +sighted +steaming +##rks +303 +911 +##holes +##hong +ching +##wife +bless +conserved +jurassic +stacey +unix +zion +chunk +rigorous +blaine +198 +peabody +slayer +dismay +brewers +nz +##jer +det +##glia +glover +postwar +int +penetration +sylvester +imitation +vertically +airlift +heiress +knoxville +viva +##uin +390 +macon +##rim +##fighter +##gonal +janice +##orescence +##wari +marius +belongings +leicestershire +196 +blanco +inverted +preseason +sanity +sobbing +##due +##elt +##dled +collingwood +regeneration +flickering +shortest +##mount +##osi +feminism +##lat +sherlock +cabinets +fumbled +northbound +precedent +snaps +##mme +researching +##akes +guillaume +insights +manipulated +vapor +neighbour +sap +gangster +frey +f1 +stalking +scarcely +callie +barnett +tendencies +audi +doomed +assessing +slung +panchayat +ambiguous +bartlett +##etto +distributing +violating +wolverhampton +##hetic +swami +histoire +##urus +liable +pounder +groin +hussain +larsen +popping +surprises +##atter +vie +curt +##station +mute +relocate +musicals +authorization +richter +##sef +immortality +tna +bombings +##press +deteriorated +yiddish +##acious +robbed +colchester +cs +pmid +ao +verified +balancing +apostle +swayed +recognizable +oxfordshire +retention +nottinghamshire +contender +judd +invitational +shrimp +uhf +##icient +cleaner +longitudinal +tanker +##mur +acronym +broker +koppen +sundance +suppliers +##gil +4000 +clipped +fuels +petite +##anne +landslide +helene +diversion +populous +landowners +auspices +melville +quantitative +##xes +ferries +nicky +##llus +doo +haunting +roche +carver +downed +unavailable +##pathy +approximation +hiroshima +##hue +garfield +valle +comparatively +keyboardist +traveler +##eit +congestion +calculating +subsidiaries +##bate +serb +modernization +fairies +deepened +ville +averages +##lore +inflammatory +tonga +##itch +co₂ +squads +##hea +gigantic +serum +enjoyment +retailer +verona +35th +cis +##phobic +magna +technicians +##vati +arithmetic +##sport +levin +##dation +amtrak +chow +sienna +##eyer +backstage +entrepreneurship +##otic +learnt +tao +##udy +worcestershire +formulation +baggage +hesitant +bali +sabotage +##kari +barren +enhancing +murmur +pl +freshly +putnam +syntax +aces +medicines +resentment +bandwidth +##sier +grins +chili +guido +##sei +framing +implying +gareth +lissa +genevieve +pertaining +admissions +geo +thorpe +proliferation +sato +bela +analyzing +parting +##gor +awakened +##isman +huddled +secrecy +##kling +hush +gentry +540 +dungeons +##ego +coasts +##utz +sacrificed +##chule +landowner +mutually +prevalence +programmer +adolescent +disrupted +seaside +gee +trusts +vamp +georgie +##nesian +##iol +schedules +sindh +##market +etched +hm +sparse +bey +beaux +scratching +gliding +unidentified +216 +collaborating +gems +jesuits +oro +accumulation +shaping +mbe +anal +##xin +231 +enthusiasts +newscast +##egan +janata +dewey +parkinson +179 +ankara +biennial +towering +dd +inconsistent +950 +##chet +thriving +terminate +cabins +furiously +eats +advocating +donkey +marley +muster +phyllis +leiden +##user +grassland +glittering +iucn +loneliness +217 +memorandum +armenians +##ddle +popularized +rhodesia +60s +lame +##illon +sans +bikini +header +orbits +##xx +##finger +##ulator +sharif +spines +biotechnology +strolled +naughty +yates +##wire +fremantle +milo +##mour +abducted +removes +##atin +humming +wonderland +##chrome +##ester +hume +pivotal +##rates +armand +grams +believers +elector +rte +apron +bis +scraped +##yria +endorsement +initials +##llation +eps +dotted +hints +buzzing +emigration +nearer +##tom +indicators +##ulu +coarse +neutron +protectorate +##uze +directional +exploits +pains +loire +1830s +proponents +guggenheim +rabbits +ritchie +305 +hectare +inputs +hutton +##raz +verify +##ako +boilers +longitude +##lev +skeletal +yer +emilia +citrus +compromised +##gau +pokemon +prescription +paragraph +eduard +cadillac +attire +categorized +kenyan +weddings +charley +##bourg +entertain +monmouth +##lles +nutrients +davey +mesh +incentive +practised +ecosystems +kemp +subdued +overheard +##rya +bodily +maxim +##nius +apprenticeship +ursula +##fight +lodged +rug +silesian +unconstitutional +patel +inspected +coyote +unbeaten +##hak +34th +disruption +convict +parcel +##cl +##nham +collier +implicated +mallory +##iac +##lab +susannah +winkler +##rber +shia +phelps +sediments +graphical +robotic +##sner +adulthood +mart +smoked +##isto +kathryn +clarified +##aran +divides +convictions +oppression +pausing +burying +##mt +federico +mathias +eileen +##tana +kite +hunched +##acies +189 +##atz +disadvantage +liza +kinetic +greedy +paradox +yokohama +dowager +trunks +ventured +##gement +gupta +vilnius +olaf +##thest +crimean +hopper +##ej +progressively +arturo +mouthed +arrondissement +##fusion +rubin +simulcast +oceania +##orum +##stra +##rred +busiest +intensely +navigator +cary +##vine +##hini +##bies +fife +rowe +rowland +posing +insurgents +shafts +lawsuits +activate +conor +inward +culturally +garlic +265 +##eering +eclectic +##hui +##kee +##nl +furrowed +vargas +meteorological +rendezvous +##aus +culinary +commencement +##dition +quota +##notes +mommy +salaries +overlapping +mule +##iology +##mology +sums +wentworth +##isk +##zione +mainline +subgroup +##illy +hack +plaintiff +verdi +bulb +differentiation +engagements +multinational +supplemented +bertrand +caller +regis +##naire +##sler +##arts +##imated +blossom +propagation +kilometer +viaduct +vineyards +##uate +beckett +optimization +golfer +songwriters +seminal +semitic +thud +volatile +evolving +ridley +##wley +trivial +distributions +scandinavia +jiang +##ject +wrestled +insistence +##dio +emphasizes +napkin +##ods +adjunct +rhyme +##ricted +##eti +hopeless +surrounds +tremble +32nd +smoky +##ntly +oils +medicinal +padded +steer +wilkes +219 +255 +concessions +hue +uniquely +blinded +landon +yahoo +##lane +hendrix +commemorating +dex +specify +chicks +##ggio +intercity +1400 +morley +##torm +highlighting +##oting +pang +oblique +stalled +##liner +flirting +newborn +1769 +bishopric +shaved +232 +currie +##ush +dharma +spartan +##ooped +favorites +smug +novella +sirens +abusive +creations +espana +##lage +paradigm +semiconductor +sheen +##rdo +##yen +##zak +nrl +renew +##pose +##tur +adjutant +marches +norma +##enity +ineffective +weimar +grunt +##gat +lordship +plotting +expenditure +infringement +lbs +refrain +av +mimi +mistakenly +postmaster +1771 +##bara +ras +motorsports +tito +199 +subjective +##zza +bully +stew +##kaya +prescott +1a +##raphic +##zam +bids +styling +paranormal +reeve +sneaking +exploding +katz +akbar +migrant +syllables +indefinitely +##ogical +destroys +replaces +applause +##phine +pest +##fide +218 +articulated +bertie +##thing +##cars +##ptic +courtroom +crowley +aesthetics +cummings +tehsil +hormones +titanic +dangerously +##ibe +stadion +jaenelle +auguste +ciudad +##chu +mysore +partisans +##sio +lucan +philipp +##aly +debating +henley +interiors +##rano +##tious +homecoming +beyonce +usher +henrietta +prepares +weeds +##oman +ely +plucked +##pire +##dable +luxurious +##aq +artifact +password +pasture +juno +maddy +minsk +##dder +##ologies +##rone +assessments +martian +royalist +1765 +examines +##mani +##rge +nino +223 +parry +scooped +relativity +##eli +##uting +##cao +congregational +noisy +traverse +##agawa +strikeouts +nickelodeon +obituary +transylvania +binds +depictions +polk +trolley +##yed +##lard +breeders +##under +dryly +hokkaido +1762 +strengths +stacks +bonaparte +connectivity +neared +prostitutes +stamped +anaheim +gutierrez +sinai +##zzling +bram +fresno +madhya +##86 +proton +##lena +##llum +##phon +reelected +wanda +##anus +##lb +ample +distinguishing +##yler +grasping +sermons +tomato +bland +stimulation +avenues +##eux +spreads +scarlett +fern +pentagon +assert +baird +chesapeake +ir +calmed +distortion +fatalities +##olis +correctional +pricing +##astic +##gina +prom +dammit +ying +collaborate +##chia +welterweight +33rd +pointer +substitution +bonded +umpire +communicating +multitude +paddle +##obe +federally +intimacy +##insky +betray +ssr +##lett +##lean +##lves +##therapy +airbus +##tery +functioned +ud +bearer +biomedical +netflix +##hire +##nca +condom +brink +ik +##nical +macy +##bet +flap +gma +experimented +jelly +lavender +##icles +##ulia +munro +##mian +##tial +rye +##rle +60th +gigs +hottest +rotated +predictions +fuji +bu +##erence +##omi +barangay +##fulness +##sas +clocks +##rwood +##liness +cereal +roe +wight +decker +uttered +babu +onion +xml +forcibly +##df +petra +sarcasm +hartley +peeled +storytelling +##42 +##xley +##ysis +##ffa +fibre +kiel +auditor +fig +harald +greenville +##berries +geographically +nell +quartz +##athic +cemeteries +##lr +crossings +nah +holloway +reptiles +chun +sichuan +snowy +660 +corrections +##ivo +zheng +ambassadors +blacksmith +fielded +fluids +hardcover +turnover +medications +melvin +academies +##erton +ro +roach +absorbing +spaniards +colton +##founded +outsider +espionage +kelsey +245 +edible +##ulf +dora +establishes +##sham +##tries +contracting +##tania +cinematic +costello +nesting +##uron +connolly +duff +##nology +mma +##mata +fergus +sexes +gi +optics +spectator +woodstock +banning +##hee +##fle +differentiate +outfielder +refinery +226 +312 +gerhard +horde +lair +drastically +##udi +landfall +##cheng +motorsport +odi +##achi +predominant +quay +skins +##ental +edna +harshly +complementary +murdering +##aves +wreckage +##90 +ono +outstretched +lennox +munitions +galen +reconcile +470 +scalp +bicycles +gillespie +questionable +rosenberg +guillermo +hostel +jarvis +kabul +volvo +opium +yd +##twined +abuses +decca +outpost +##cino +sensible +neutrality +##64 +ponce +anchorage +atkins +turrets +inadvertently +disagree +libre +vodka +reassuring +weighs +##yal +glide +jumper +ceilings +repertory +outs +stain +##bial +envy +##ucible +smashing +heightened +policing +hyun +mixes +lai +prima +##ples +celeste +##bina +lucrative +intervened +kc +manually +##rned +stature +staffed +bun +bastards +nairobi +priced +##auer +thatcher +##kia +tripped +comune +##ogan +##pled +brasil +incentives +emanuel +hereford +musica +##kim +benedictine +biennale +##lani +eureka +gardiner +rb +knocks +sha +##ael +##elled +##onate +efficacy +ventura +masonic +sanford +maize +leverage +##feit +capacities +santana +##aur +novelty +vanilla +##cter +##tour +benin +##oir +##rain +neptune +drafting +tallinn +##cable +humiliation +##boarding +schleswig +fabian +bernardo +liturgy +spectacle +sweeney +pont +routledge +##tment +cosmos +ut +hilt +sleek +universally +##eville +##gawa +typed +##dry +favors +allegheny +glaciers +##rly +recalling +aziz +##log +parasite +requiem +auf +##berto +##llin +illumination +##breaker +##issa +festivities +bows +govern +vibe +vp +333 +sprawled +larson +pilgrim +bwf +leaping +##rts +##ssel +alexei +greyhound +hoarse +##dler +##oration +seneca +##cule +gaping +##ulously +##pura +cinnamon +##gens +##rricular +craven +fantasies +houghton +engined +reigned +dictator +supervising +##oris +bogota +commentaries +unnatural +fingernails +spirituality +tighten +##tm +canadiens +protesting +intentional +cheers +sparta +##ytic +##iere +##zine +widen +belgarath +controllers +dodd +iaaf +navarre +##ication +defect +squire +steiner +whisky +##mins +560 +inevitably +tome +##gold +chew +##uid +##lid +elastic +##aby +streaked +alliances +jailed +regal +##ined +##phy +czechoslovak +narration +absently +##uld +bluegrass +guangdong +quran +criticizing +hose +hari +##liest +##owa +skier +streaks +deploy +##lom +raft +bose +dialed +huff +##eira +haifa +simplest +bursting +endings +ib +sultanate +##titled +franks +whitman +ensures +sven +##ggs +collaborators +forster +organising +ui +banished +napier +injustice +teller +layered +thump +##otti +roc +battleships +evidenced +fugitive +sadie +robotics +##roud +equatorial +geologist +##iza +yielding +##bron +##sr +internationale +mecca +##diment +sbs +skyline +toad +uploaded +reflective +undrafted +lal +leafs +bayern +##dai +lakshmi +shortlisted +##stick +##wicz +camouflage +donate +af +christi +lau +##acio +disclosed +nemesis +1761 +assemble +straining +northamptonshire +tal +##asi +bernardino +premature +heidi +42nd +coefficients +galactic +reproduce +buzzed +sensations +zionist +monsieur +myrtle +##eme +archery +strangled +musically +viewpoint +antiquities +bei +trailers +seahawks +cured +pee +preferring +tasmanian +lange +sul +##mail +##working +colder +overland +lucivar +massey +gatherings +haitian +##smith +disapproval +flaws +##cco +##enbach +1766 +npr +##icular +boroughs +creole +forums +techno +1755 +dent +abdominal +streetcar +##eson +##stream +procurement +gemini +predictable +##tya +acheron +christoph +feeder +fronts +vendor +bernhard +jammu +tumors +slang +##uber +goaltender +twists +curving +manson +vuelta +mer +peanut +confessions +pouch +unpredictable +allowance +theodor +vascular +##factory +bala +authenticity +metabolic +coughing +nanjing +##cea +pembroke +##bard +splendid +36th +ff +hourly +##ahu +elmer +handel +##ivate +awarding +thrusting +dl +experimentation +##hesion +##46 +caressed +entertained +steak +##rangle +biologist +orphans +baroness +oyster +stepfather +##dridge +mirage +reefs +speeding +##31 +barons +1764 +227 +inhabit +preached +repealed +##tral +honoring +boogie +captives +administer +johanna +##imate +gel +suspiciously +1767 +sobs +##dington +backbone +hayward +garry +##folding +##nesia +maxi +##oof +##ppe +ellison +galileo +##stand +crimea +frenzy +amour +bumper +matrices +natalia +baking +garth +palestinians +##grove +smack +conveyed +ensembles +gardening +##manship +##rup +##stituting +1640 +harvesting +topography +jing +shifters +dormitory +##carriage +##lston +ist +skulls +##stadt +dolores +jewellery +sarawak +##wai +##zier +fences +christy +confinement +tumbling +credibility +fir +stench +##bria +##plication +##nged +##sam +virtues +##belt +marjorie +pba +##eem +##made +celebrates +schooner +agitated +barley +fulfilling +anthropologist +##pro +restrict +novi +regulating +##nent +padres +##rani +##hesive +loyola +tabitha +milky +olson +proprietor +crambidae +guarantees +intercollegiate +ljubljana +hilda +##sko +ignorant +hooded +##lts +sardinia +##lidae +##vation +frontman +privileged +witchcraft +##gp +jammed +laude +poking +##than +bracket +amazement +yunnan +##erus +maharaja +linnaeus +264 +commissioning +milano +peacefully +##logies +akira +rani +regulator +##36 +grasses +##rance +luzon +crows +compiler +gretchen +seaman +edouard +tab +buccaneers +ellington +hamlets +whig +socialists +##anto +directorial +easton +mythological +##kr +##vary +rhineland +semantic +taut +dune +inventions +succeeds +##iter +replication +branched +##pired +jul +prosecuted +kangaroo +penetrated +##avian +middlesbrough +doses +bleak +madam +predatory +relentless +##vili +reluctance +##vir +hailey +crore +silvery +1759 +monstrous +swimmers +transmissions +hawthorn +informing +##eral +toilets +caracas +crouch +kb +##sett +295 +cartel +hadley +##aling +alexia +yvonne +##biology +cinderella +eton +superb +blizzard +stabbing +industrialist +maximus +##gm +##orus +groves +maud +clade +oversized +comedic +##bella +rosen +nomadic +fulham +montane +beverages +galaxies +redundant +swarm +##rot +##folia +##llis +buckinghamshire +fen +bearings +bahadur +##rom +gilles +phased +dynamite +faber +benoit +vip +##ount +##wd +booking +fractured +tailored +anya +spices +westwood +cairns +auditions +inflammation +steamed +##rocity +##acion +##urne +skyla +thereof +watford +torment +archdeacon +transforms +lulu +demeanor +fucked +serge +##sor +mckenna +minas +entertainer +##icide +caress +originate +residue +##sty +1740 +##ilised +##org +beech +##wana +subsidies +##ghton +emptied +gladstone +ru +firefighters +voodoo +##rcle +het +nightingale +tamara +edmond +ingredient +weaknesses +silhouette +285 +compatibility +withdrawing +hampson +##mona +anguish +giggling +##mber +bookstore +##jiang +southernmost +tilting +##vance +bai +economical +rf +briefcase +dreadful +hinted +projections +shattering +totaling +##rogate +analogue +indicted +periodical +fullback +##dman +haynes +##tenberg +##ffs +##ishment +1745 +thirst +stumble +penang +vigorous +##ddling +##kor +##lium +octave +##ove +##enstein +##inen +##ones +siberian +##uti +cbn +repeal +swaying +##vington +khalid +tanaka +unicorn +otago +plastered +lobe +riddle +##rella +perch +##ishing +croydon +filtered +graeme +tripoli +##ossa +crocodile +##chers +sufi +mined +##tung +inferno +lsu +##phi +swelled +utilizes +£2 +cale +periodicals +styx +hike +informally +coop +lund +##tidae +ala +hen +qui +transformations +disposed +sheath +chickens +##cade +fitzroy +sas +silesia +unacceptable +odisha +1650 +sabrina +pe +spokane +ratios +athena +massage +shen +dilemma +##drum +##riz +##hul +corona +doubtful +niall +##pha +##bino +fines +cite +acknowledging +bangor +ballard +bathurst +##resh +huron +mustered +alzheimer +garments +kinase +tyre +warship +##cp +flashback +pulmonary +braun +cheat +kamal +cyclists +constructions +grenades +ndp +traveller +excuses +stomped +signalling +trimmed +futsal +mosques +relevance +##wine +wta +##23 +##vah +##lter +hoc +##riding +optimistic +##´s +deco +sim +interacting +rejecting +moniker +waterways +##ieri +##oku +mayors +gdansk +outnumbered +pearls +##ended +##hampton +fairs +totals +dominating +262 +notions +stairway +compiling +pursed +commodities +grease +yeast +##jong +carthage +griffiths +residual +amc +contraction +laird +sapphire +##marine +##ivated +amalgamation +dissolve +inclination +lyle +packaged +altitudes +suez +canons +graded +lurched +narrowing +boasts +guise +wed +enrico +##ovsky +rower +scarred +bree +cub +iberian +protagonists +bargaining +proposing +trainers +voyages +vans +fishes +##aea +##ivist +##verance +encryption +artworks +kazan +sabre +cleopatra +hepburn +rotting +supremacy +mecklenburg +##brate +burrows +hazards +outgoing +flair +organizes +##ctions +scorpion +##usions +boo +234 +chevalier +dunedin +slapping +##34 +ineligible +pensions +##38 +##omic +manufactures +emails +bismarck +238 +weakening +blackish +ding +mcgee +quo +##rling +northernmost +xx +manpower +greed +sampson +clicking +##ange +##horpe +##inations +##roving +torre +##eptive +##moral +symbolism +38th +asshole +meritorious +outfits +splashed +biographies +sprung +astros +##tale +302 +737 +filly +raoul +nw +tokugawa +linden +clubhouse +##apa +tracts +romano +##pio +putin +tags +##note +chained +dickson +gunshot +moe +gunn +rashid +##tails +zipper +##bas +##nea +contrasted +##ply +##udes +plum +pharaoh +##pile +aw +comedies +ingrid +sandwiches +subdivisions +1100 +mariana +nokia +kamen +hz +delaney +veto +herring +##words +possessive +outlines +##roup +siemens +stairwell +rc +gallantry +messiah +palais +yells +233 +zeppelin +##dm +bolivar +##cede +smackdown +mckinley +##mora +##yt +muted +geologic +finely +unitary +avatar +hamas +maynard +rees +bog +contrasting +##rut +liv +chico +disposition +pixel +##erate +becca +dmitry +yeshiva +narratives +##lva +##ulton +mercenary +sharpe +tempered +navigate +stealth +amassed +keynes +##lini +untouched +##rrie +havoc +lithium +##fighting +abyss +graf +southward +wolverine +balloons +implements +ngos +transitions +##icum +ambushed +concacaf +dormant +economists +##dim +costing +csi +rana +universite +boulders +verity +##llon +collin +mellon +misses +cypress +fluorescent +lifeless +spence +##ulla +crewe +shepard +pak +revelations +##م +jolly +gibbons +paw +##dro +##quel +freeing +##test +shack +fries +palatine +##51 +##hiko +accompaniment +cruising +recycled +##aver +erwin +sorting +synthesizers +dyke +realities +sg +strides +enslaved +wetland +##ghan +competence +gunpowder +grassy +maroon +reactors +objection +##oms +carlson +gearbox +macintosh +radios +shelton +##sho +clergyman +prakash +254 +mongols +trophies +oricon +228 +stimuli +twenty20 +cantonese +cortes +mirrored +##saurus +bhp +cristina +melancholy +##lating +enjoyable +nuevo +##wny +downfall +schumacher +##ind +banging +lausanne +rumbled +paramilitary +reflex +ax +amplitude +migratory +##gall +##ups +midi +barnard +lastly +sherry +##hp +##nall +keystone +##kra +carleton +slippery +##53 +coloring +foe +socket +otter +##rgos +mats +##tose +consultants +bafta +bison +topping +##km +490 +primal +abandonment +transplant +atoll +hideous +mort +pained +reproduced +tae +howling +##turn +unlawful +billionaire +hotter +poised +lansing +##chang +dinamo +retro +messing +nfc +domesday +##mina +blitz +timed +##athing +##kley +ascending +gesturing +##izations +signaled +tis +chinatown +mermaid +savanna +jameson +##aint +catalina +##pet +##hers +cochrane +cy +chatting +##kus +alerted +computation +mused +noelle +majestic +mohawk +campo +octagonal +##sant +##hend +241 +aspiring +##mart +comprehend +iona +paralyzed +shimmering +swindon +rhone +##eley +reputed +configurations +pitchfork +agitation +francais +gillian +lipstick +##ilo +outsiders +pontifical +resisting +bitterness +sewer +rockies +##edd +##ucher +misleading +1756 +exiting +galloway +##nging +risked +##heart +246 +commemoration +schultz +##rka +integrating +##rsa +poses +shrieked +##weiler +guineas +gladys +jerking +owls +goldsmith +nightly +penetrating +##unced +lia +##33 +ignited +betsy +##aring +##thorpe +follower +vigorously +##rave +coded +kiran +knit +zoology +tbilisi +##28 +##bered +repository +govt +deciduous +dino +growling +##bba +enhancement +unleashed +chanting +pussy +biochemistry +##eric +kettle +repression +toxicity +nrhp +##arth +##kko +##bush +ernesto +commended +outspoken +242 +mca +parchment +sms +kristen +##aton +bisexual +raked +glamour +navajo +a2 +conditioned +showcased +##hma +spacious +youthful +##esa +usl +appliances +junta +brest +layne +conglomerate +enchanted +chao +loosened +picasso +circulating +inspect +montevideo +##centric +##kti +piazza +spurred +##aith +bari +freedoms +poultry +stamford +lieu +##ect +indigo +sarcastic +bahia +stump +attach +dvds +frankenstein +lille +approx +scriptures +pollen +##script +nmi +overseen +##ivism +tides +proponent +newmarket +inherit +milling +##erland +centralized +##rou +distributors +credentials +drawers +abbreviation +##lco +##xon +downing +uncomfortably +ripe +##oes +erase +franchises +##ever +populace +##bery +##khar +decomposition +pleas +##tet +daryl +sabah +##stle +##wide +fearless +genie +lesions +annette +##ogist +oboe +appendix +nair +dripped +petitioned +maclean +mosquito +parrot +rpg +hampered +1648 +operatic +reservoirs +##tham +irrelevant +jolt +summarized +##fp +medallion +##taff +##− +clawed +harlow +narrower +goddard +marcia +bodied +fremont +suarez +altering +tempest +mussolini +porn +##isms +sweetly +oversees +walkers +solitude +grimly +shrines +hk +ich +supervisors +hostess +dietrich +legitimacy +brushes +expressive +##yp +dissipated +##rse +localized +systemic +##nikov +gettysburg +##js +##uaries +dialogues +muttering +251 +housekeeper +sicilian +discouraged +##frey +beamed +kaladin +halftime +kidnap +##amo +##llet +1754 +synonymous +depleted +instituto +insulin +reprised +##opsis +clashed +##ctric +interrupting +radcliffe +insisting +medici +1715 +ejected +playfully +turbulent +##47 +starvation +##rini +shipment +rebellious +petersen +verification +merits +##rified +cakes +##charged +1757 +milford +shortages +spying +fidelity +##aker +emitted +storylines +harvested +seismic +##iform +cheung +kilda +theoretically +barbie +lynx +##rgy +##tius +goblin +mata +poisonous +##nburg +reactive +residues +obedience +##евич +conjecture +##rac +401 +hating +sixties +kicker +moaning +motown +##bha +emancipation +neoclassical +##hering +consoles +ebert +professorship +##tures +sustaining +assaults +obeyed +affluent +incurred +tornadoes +##eber +##zow +emphasizing +highlanders +cheated +helmets +##ctus +internship +terence +bony +executions +legislators +berries +peninsular +tinged +##aco +1689 +amplifier +corvette +ribbons +lavish +pennant +##lander +worthless +##chfield +##forms +mariano +pyrenees +expenditures +##icides +chesterfield +mandir +tailor +39th +sergey +nestled +willed +aristocracy +devotees +goodnight +raaf +rumored +weaponry +remy +appropriations +harcourt +burr +riaa +##lence +limitation +unnoticed +guo +soaking +swamps +##tica +collapsing +tatiana +descriptive +brigham +psalm +##chment +maddox +##lization +patti +caliph +##aja +akron +injuring +serra +##ganj +basins +##sari +astonished +launcher +##church +hilary +wilkins +sewing +##sf +stinging +##fia +##ncia +underwood +startup +##ition +compilations +vibrations +embankment +jurist +##nity +bard +juventus +groundwater +kern +palaces +helium +boca +cramped +marissa +soto +##worm +jae +princely +##ggy +faso +bazaar +warmly +##voking +229 +pairing +##lite +##grate +##nets +wien +freaked +ulysses +rebirth +##alia +##rent +mummy +guzman +jimenez +stilled +##nitz +trajectory +tha +woken +archival +professions +##pts +##pta +hilly +shadowy +shrink +##bolt +norwood +glued +migrate +stereotypes +devoid +##pheus +625 +evacuate +horrors +infancy +gotham +knowles +optic +downloaded +sachs +kingsley +parramatta +darryl +mor +##onale +shady +commence +confesses +kan +##meter +##placed +marlborough +roundabout +regents +frigates +io +##imating +gothenburg +revoked +carvings +clockwise +convertible +intruder +##sche +banged +##ogo +vicky +bourgeois +##mony +dupont +footing +##gum +pd +##real +buckle +yun +penthouse +sane +720 +serviced +stakeholders +neumann +bb +##eers +comb +##gam +catchment +pinning +rallies +typing +##elles +forefront +freiburg +sweetie +giacomo +widowed +goodwill +worshipped +aspirations +midday +##vat +fishery +##trick +bournemouth +turk +243 +hearth +ethanol +guadalajara +murmurs +sl +##uge +afforded +scripted +##hta +wah +##jn +coroner +translucent +252 +memorials +puck +progresses +clumsy +##race +315 +candace +recounted +##27 +##slin +##uve +filtering +##mac +howl +strata +heron +leveled +##ays +dubious +##oja +##т +##wheel +citations +exhibiting +##laya +##mics +##pods +turkic +##lberg +injunction +##ennial +##mit +antibodies +##44 +organise +##rigues +cardiovascular +cushion +inverness +##zquez +dia +cocoa +sibling +##tman +##roid +expanse +feasible +tunisian +algiers +##relli +rus +bloomberg +dso +westphalia +bro +tacoma +281 +downloads +##ours +konrad +duran +##hdi +continuum +jett +compares +legislator +secession +##nable +##gues +##zuka +translating +reacher +##gley +##ła +aleppo +##agi +tc +orchards +trapping +linguist +versatile +drumming +postage +calhoun +superiors +##mx +barefoot +leary +##cis +ignacio +alfa +kaplan +##rogen +bratislava +mori +##vot +disturb +haas +313 +cartridges +gilmore +radiated +salford +tunic +hades +##ulsive +archeological +delilah +magistrates +auditioned +brewster +charters +empowerment +blogs +cappella +dynasties +iroquois +whipping +##krishna +raceway +truths +myra +weaken +judah +mcgregor +##horse +mic +refueling +37th +burnley +bosses +markus +premio +query +##gga +dunbar +##economic +darkest +lyndon +sealing +commendation +reappeared +##mun +addicted +ezio +slaughtered +satisfactory +shuffle +##eves +##thic +##uj +fortification +warrington +##otto +resurrected +fargo +mane +##utable +##lei +##space +foreword +ox +##aris +##vern +abrams +hua +##mento +sakura +##alo +uv +sentimental +##skaya +midfield +##eses +sturdy +scrolls +macleod +##kyu +entropy +##lance +mitochondrial +cicero +excelled +thinner +convoys +perceive +##oslav +##urable +systematically +grind +burkina +287 +##tagram +ops +##aman +guantanamo +##cloth +##tite +forcefully +wavy +##jou +pointless +##linger +##tze +layton +portico +superficial +clerical +outlaws +##hism +burials +muir +##inn +creditors +hauling +rattle +##leg +calais +monde +archers +reclaimed +dwell +wexford +hellenic +falsely +remorse +##tek +dough +furnishings +##uttered +gabon +neurological +novice +##igraphy +contemplated +pulpit +nightstand +saratoga +##istan +documenting +pulsing +taluk +##firmed +busted +marital +##rien +disagreements +wasps +##yes +hodge +mcdonnell +mimic +fran +pendant +dhabi +musa +##nington +congratulations +argent +darrell +concussion +losers +regrets +thessaloniki +reversal +donaldson +hardwood +thence +achilles +ritter +##eran +demonic +jurgen +prophets +goethe +eki +classmate +buff +##cking +yank +irrational +##inging +perished +seductive +qur +sourced +##crat +##typic +mustard +ravine +barre +horizontally +characterization +phylogenetic +boise +##dit +##runner +##tower +brutally +intercourse +seduce +##bbing +fay +ferris +ogden +amar +nik +unarmed +##inator +evaluating +kyrgyzstan +sweetness +##lford +##oki +mccormick +meiji +notoriety +stimulate +disrupt +figuring +instructional +mcgrath +##zoo +groundbreaking +##lto +flinch +khorasan +agrarian +bengals +mixer +radiating +##sov +ingram +pitchers +nad +tariff +##cript +tata +##codes +##emi +##ungen +appellate +lehigh +##bled +##giri +brawl +duct +texans +##ciation +##ropolis +skipper +speculative +vomit +doctrines +stresses +253 +davy +graders +whitehead +jozef +timely +cumulative +haryana +paints +appropriately +boon +cactus +##ales +##pid +dow +legions +##pit +perceptions +1730 +picturesque +##yse +periphery +rune +wr +##aha +celtics +sentencing +whoa +##erin +confirms +variance +425 +moines +mathews +spade +rave +m1 +fronted +fx +blending +alleging +reared +##gl +237 +##paper +grassroots +eroded +##free +##physical +directs +ordeal +##sław +accelerate +hacker +rooftop +##inia +lev +buys +cebu +devote +##lce +specialising +##ulsion +choreographed +repetition +warehouses +##ryl +paisley +tuscany +analogy +sorcerer +hash +huts +shards +descends +exclude +nix +chaplin +gaga +ito +vane +##drich +causeway +misconduct +limo +orchestrated +glands +jana +##kot +u2 +##mple +##sons +branching +contrasts +scoop +longed +##virus +chattanooga +##75 +syrup +cornerstone +##tized +##mind +##iaceae +careless +precedence +frescoes +##uet +chilled +consult +modelled +snatch +peat +##thermal +caucasian +humane +relaxation +spins +temperance +##lbert +occupations +lambda +hybrids +moons +mp3 +##oese +247 +rolf +societal +yerevan +ness +##ssler +befriended +mechanized +nominate +trough +boasted +cues +seater +##hom +bends +##tangle +conductors +emptiness +##lmer +eurasian +adriatic +tian +##cie +anxiously +lark +propellers +chichester +jock +ev +2a +##holding +credible +recounts +tori +loyalist +abduction +##hoot +##redo +nepali +##mite +ventral +tempting +##ango +##crats +steered +##wice +javelin +dipping +laborers +prentice +looming +titanium +##ː +badges +emir +tensor +##ntation +egyptians +rash +denies +hawthorne +lombard +showers +wehrmacht +dietary +trojan +##reus +welles +executing +horseshoe +lifeboat +##lak +elsa +infirmary +nearing +roberta +boyer +mutter +trillion +joanne +##fine +##oked +sinks +vortex +uruguayan +clasp +sirius +##block +accelerator +prohibit +sunken +byu +chronological +diplomats +ochreous +510 +symmetrical +1644 +maia +##tology +salts +reigns +atrocities +##ия +hess +bared +issn +##vyn +cater +saturated +##cycle +##isse +sable +voyager +dyer +yusuf +##inge +fountains +wolff +##39 +##nni +engraving +rollins +atheist +ominous +##ault +herr +chariot +martina +strung +##fell +##farlane +horrific +sahib +gazes +saetan +erased +ptolemy +##olic +flushing +lauderdale +analytic +##ices +530 +navarro +beak +gorilla +herrera +broom +guadalupe +raiding +sykes +311 +bsc +deliveries +1720 +invasions +carmichael +tajikistan +thematic +ecumenical +sentiments +onstage +##rians +##brand +##sume +catastrophic +flanks +molten +##arns +waller +aimee +terminating +##icing +alternately +##oche +nehru +printers +outraged +##eving +empires +template +banners +repetitive +za +##oise +vegetarian +##tell +guiana +opt +cavendish +lucknow +synthesized +##hani +##mada +finalized +##ctable +fictitious +mayoral +unreliable +##enham +embracing +peppers +rbis +##chio +##neo +inhibition +slashed +togo +orderly +embroidered +safari +salty +236 +barron +benito +totaled +##dak +pubs +simulated +caden +devin +tolkien +momma +welding +sesame +##ept +gottingen +hardness +630 +shaman +temeraire +620 +adequately +pediatric +##kit +ck +assertion +radicals +composure +cadence +seafood +beaufort +lazarus +mani +warily +cunning +kurdistan +249 +cantata +##kir +ares +##41 +##clusive +nape +townland +geared +insulted +flutter +boating +violate +draper +dumping +malmo +##hh +##romatic +firearm +alta +bono +obscured +##clave +exceeds +panorama +unbelievable +##train +preschool +##essed +disconnected +installing +rescuing +secretaries +accessibility +##castle +##drive +##ifice +##film +bouts +slug +waterway +mindanao +##buro +##ratic +halves +##ل +calming +liter +maternity +adorable +bragg +electrification +mcc +##dote +roxy +schizophrenia +##body +munoz +kaye +whaling +239 +mil +tingling +tolerant +##ago +unconventional +volcanoes +##finder +deportivo +##llie +robson +kaufman +neuroscience +wai +deportation +masovian +scraping +converse +##bh +hacking +bulge +##oun +administratively +yao +580 +amp +mammoth +booster +claremont +hooper +nomenclature +pursuits +mclaughlin +melinda +##sul +catfish +barclay +substrates +taxa +zee +originals +kimberly +packets +padma +##ality +borrowing +ostensibly +solvent +##bri +##genesis +##mist +lukas +shreveport +veracruz +##ь +##lou +##wives +cheney +tt +anatolia +hobbs +##zyn +cyclic +radiant +alistair +greenish +siena +dat +independents +##bation +conform +pieter +hyper +applicant +bradshaw +spores +telangana +vinci +inexpensive +nuclei +322 +jang +nme +soho +spd +##ign +cradled +receptionist +pow +##43 +##rika +fascism +##ifer +experimenting +##ading +##iec +##region +345 +jocelyn +maris +stair +nocturnal +toro +constabulary +elgin +##kker +msc +##giving +##schen +##rase +doherty +doping +sarcastically +batter +maneuvers +##cano +##apple +##gai +##git +intrinsic +##nst +##stor +1753 +showtime +cafes +gasps +lviv +ushered +##thed +fours +restart +astonishment +transmitting +flyer +shrugs +##sau +intriguing +cones +dictated +mushrooms +medial +##kovsky +##elman +escorting +gaped +##26 +godfather +##door +##sell +djs +recaptured +timetable +vila +1710 +3a +aerodrome +mortals +scientology +##orne +angelina +mag +convection +unpaid +insertion +intermittent +lego +##nated +endeavor +kota +pereira +##lz +304 +bwv +glamorgan +insults +agatha +fey +##cend +fleetwood +mahogany +protruding +steamship +zeta +##arty +mcguire +suspense +##sphere +advising +urges +##wala +hurriedly +meteor +gilded +inline +arroyo +stalker +##oge +excitedly +revered +##cure +earle +introductory +##break +##ilde +mutants +puff +pulses +reinforcement +##haling +curses +lizards +stalk +correlated +##fixed +fallout +macquarie +##unas +bearded +denton +heaving +802 +##ocation +winery +assign +dortmund +##lkirk +everest +invariant +charismatic +susie +##elling +bled +lesley +telegram +sumner +bk +##ogen +##к +wilcox +needy +colbert +duval +##iferous +##mbled +allotted +attends +imperative +##hita +replacements +hawker +##inda +insurgency +##zee +##eke +casts +##yla +680 +ives +transitioned +##pack +##powering +authoritative +baylor +flex +cringed +plaintiffs +woodrow +##skie +drastic +ape +aroma +unfolded +commotion +nt +preoccupied +theta +routines +lasers +privatization +wand +domino +ek +clenching +nsa +strategically +showered +bile +handkerchief +pere +storing +christophe +insulting +316 +nakamura +romani +asiatic +magdalena +palma +cruises +stripping +405 +konstantin +soaring +##berman +colloquially +forerunner +havilland +incarcerated +parasites +sincerity +##utus +disks +plank +saigon +##ining +corbin +homo +ornaments +powerhouse +##tlement +chong +fastened +feasibility +idf +morphological +usable +##nish +##zuki +aqueduct +jaguars +keepers +##flies +aleksandr +faust +assigns +ewing +bacterium +hurled +tricky +hungarians +integers +wallis +321 +yamaha +##isha +hushed +oblivion +aviator +evangelist +friars +##eller +monograph +ode +##nary +airplanes +labourers +charms +##nee +1661 +hagen +tnt +rudder +fiesta +transcript +dorothea +ska +inhibitor +maccabi +retorted +raining +encompassed +clauses +menacing +1642 +lineman +##gist +vamps +##ape +##dick +gloom +##rera +dealings +easing +seekers +##nut +##pment +helens +unmanned +##anu +##isson +basics +##amy +##ckman +adjustments +1688 +brutality +horne +##zell +sui +##55 +##mable +aggregator +##thal +rhino +##drick +##vira +counters +zoom +##01 +##rting +mn +montenegrin +packard +##unciation +##♭ +##kki +reclaim +scholastic +thugs +pulsed +##icia +syriac +quan +saddam +banda +kobe +blaming +buddies +dissent +##lusion +##usia +corbett +jaya +delle +erratic +lexie +##hesis +435 +amiga +hermes +##pressing +##leen +chapels +gospels +jamal +##uating +compute +revolving +warp +##sso +##thes +armory +##eras +##gol +antrim +loki +##kow +##asian +##good +##zano +braid +handwriting +subdistrict +funky +pantheon +##iculate +concurrency +estimation +improper +juliana +##his +newcomers +johnstone +staten +communicated +##oco +##alle +sausage +stormy +##stered +##tters +superfamily +##grade +acidic +collateral +tabloid +##oped +##rza +bladder +austen +##ellant +mcgraw +##hay +hannibal +mein +aquino +lucifer +wo +badger +boar +cher +christensen +greenberg +interruption +##kken +jem +244 +mocked +bottoms +cambridgeshire +##lide +sprawling +##bbly +eastwood +ghent +synth +##buck +advisers +##bah +nominally +hapoel +qu +daggers +estranged +fabricated +towels +vinnie +wcw +misunderstanding +anglia +nothin +unmistakable +##dust +##lova +chilly +marquette +truss +##edge +##erine +reece +##lty +##chemist +##connected +272 +308 +41st +bash +raion +waterfalls +##ump +##main +labyrinth +queue +theorist +##istle +bharatiya +flexed +soundtracks +rooney +leftist +patrolling +wharton +plainly +alleviate +eastman +schuster +topographic +engages +immensely +unbearable +fairchild +1620 +dona +lurking +parisian +oliveira +ia +indictment +hahn +bangladeshi +##aster +vivo +##uming +##ential +antonia +expects +indoors +kildare +harlan +##logue +##ogenic +##sities +forgiven +##wat +childish +tavi +##mide +##orra +plausible +grimm +successively +scooted +##bola +##dget +##rith +spartans +emery +flatly +azure +epilogue +##wark +flourish +##iny +##tracted +##overs +##oshi +bestseller +distressed +receipt +spitting +hermit +topological +##cot +drilled +subunit +francs +##layer +eel +##fk +##itas +octopus +footprint +petitions +ufo +##say +##foil +interfering +leaking +palo +##metry +thistle +valiant +##pic +narayan +mcpherson +##fast +gonzales +##ym +##enne +dustin +novgorod +solos +##zman +doin +##raph +##patient +##meyer +soluble +ashland +cuffs +carole +pendleton +whistling +vassal +##river +deviation +revisited +constituents +rallied +rotate +loomed +##eil +##nting +amateurs +augsburg +auschwitz +crowns +skeletons +##cona +bonnet +257 +dummy +globalization +simeon +sleeper +mandal +differentiated +##crow +##mare +milne +bundled +exasperated +talmud +owes +segregated +##feng +##uary +dentist +piracy +props +##rang +devlin +##torium +malicious +paws +##laid +dependency +##ergy +##fers +##enna +258 +pistons +rourke +jed +grammatical +tres +maha +wig +512 +ghostly +jayne +##achal +##creen +##ilis +##lins +##rence +designate +##with +arrogance +cambodian +clones +showdown +throttle +twain +##ception +lobes +metz +nagoya +335 +braking +##furt +385 +roaming +##minster +amin +crippled +##37 +##llary +indifferent +hoffmann +idols +intimidating +1751 +261 +influenza +memo +onions +1748 +bandage +consciously +##landa +##rage +clandestine +observes +swiped +tangle +##ener +##jected +##trum +##bill +##lta +hugs +congresses +josiah +spirited +##dek +humanist +managerial +filmmaking +inmate +rhymes +debuting +grimsby +ur +##laze +duplicate +vigor +##tf +republished +bolshevik +refurbishment +antibiotics +martini +methane +newscasts +royale +horizons +levant +iain +visas +##ischen +paler +##around +manifestation +snuck +alf +chop +futile +pedestal +rehab +##kat +bmg +kerman +res +fairbanks +jarrett +abstraction +saharan +##zek +1746 +procedural +clearer +kincaid +sash +luciano +##ffey +crunch +helmut +##vara +revolutionaries +##tute +creamy +leach +##mmon +1747 +permitting +nes +plight +wendell +##lese +contra +ts +clancy +ipa +mach +staples +autopsy +disturbances +nueva +karin +pontiac +##uding +proxy +venerable +haunt +leto +bergman +expands +##helm +wal +##pipe +canning +celine +cords +obesity +##enary +intrusion +planner +##phate +reasoned +sequencing +307 +harrow +##chon +##dora +marred +mcintyre +repay +tarzan +darting +248 +harrisburg +margarita +repulsed +##hur +##lding +belinda +hamburger +novo +compliant +runways +bingham +registrar +skyscraper +ic +cuthbert +improvisation +livelihood +##corp +##elial +admiring +##dened +sporadic +believer +casablanca +popcorn +##29 +asha +shovel +##bek +##dice +coiled +tangible +##dez +casper +elsie +resin +tenderness +rectory +##ivision +avail +sonar +##mori +boutique +##dier +guerre +bathed +upbringing +vaulted +sandals +blessings +##naut +##utnant +1680 +306 +foxes +pia +corrosion +hesitantly +confederates +crystalline +footprints +shapiro +tirana +valentin +drones +45th +microscope +shipments +texted +inquisition +wry +guernsey +unauthorized +resigning +760 +ripple +schubert +stu +reassure +felony +##ardo +brittle +koreans +##havan +##ives +dun +implicit +tyres +##aldi +##lth +magnolia +##ehan +##puri +##poulos +aggressively +fei +gr +familiarity +##poo +indicative +##trust +fundamentally +jimmie +overrun +395 +anchors +moans +##opus +britannia +armagh +##ggle +purposely +seizing +##vao +bewildered +mundane +avoidance +cosmopolitan +geometridae +quartermaster +caf +415 +chatter +engulfed +gleam +purge +##icate +juliette +jurisprudence +guerra +revisions +##bn +casimir +brew +##jm +1749 +clapton +cloudy +conde +hermitage +278 +simulations +torches +vincenzo +matteo +##rill +hidalgo +booming +westbound +accomplishment +tentacles +unaffected +##sius +annabelle +flopped +sloping +##litz +dreamer +interceptor +vu +##loh +consecration +copying +messaging +breaker +climates +hospitalized +1752 +torino +afternoons +winfield +witnessing +##teacher +breakers +choirs +sawmill +coldly +##ege +sipping +haste +uninhabited +conical +bibliography +pamphlets +severn +edict +##oca +deux +illnesses +grips +##pl +rehearsals +sis +thinkers +tame +##keepers +1690 +acacia +reformer +##osed +##rys +shuffling +##iring +##shima +eastbound +ionic +rhea +flees +littered +##oum +rocker +vomiting +groaning +champ +overwhelmingly +civilizations +paces +sloop +adoptive +##tish +skaters +##vres +aiding +mango +##joy +nikola +shriek +##ignon +pharmaceuticals +##mg +tuna +calvert +gustavo +stocked +yearbook +##urai +##mana +computed +subsp +riff +hanoi +kelvin +hamid +moors +pastures +summons +jihad +nectar +##ctors +bayou +untitled +pleasing +vastly +republics +intellect +##η +##ulio +##tou +crumbling +stylistic +sb +##ی +consolation +frequented +h₂o +walden +widows +##iens +404 +##ignment +chunks +improves +288 +grit +recited +##dev +snarl +sociological +##arte +##gul +inquired +##held +bruise +clube +consultancy +homogeneous +hornets +multiplication +pasta +prick +savior +##grin +##kou +##phile +yoon +##gara +grimes +vanishing +cheering +reacting +bn +distillery +##quisite +##vity +coe +dockyard +massif +##jord +escorts +voss +##valent +byte +chopped +hawke +illusions +workings +floats +##koto +##vac +kv +annapolis +madden +##onus +alvaro +noctuidae +##cum +##scopic +avenge +steamboat +forte +illustrates +erika +##trip +570 +dew +nationalities +bran +manifested +thirsty +diversified +muscled +reborn +##standing +arson +##lessness +##dran +##logram +##boys +##kushima +##vious +willoughby +##phobia +286 +alsace +dashboard +yuki +##chai +granville +myspace +publicized +tricked +##gang +adjective +##ater +relic +reorganisation +enthusiastically +indications +saxe +##lassified +consolidate +iec +padua +helplessly +ramps +renaming +regulars +pedestrians +accents +convicts +inaccurate +lowers +mana +##pati +barrie +bjp +outta +someplace +berwick +flanking +invoked +marrow +sparsely +excerpts +clothed +rei +##ginal +wept +##straße +##vish +alexa +excel +##ptive +membranes +aquitaine +creeks +cutler +sheppard +implementations +ns +##dur +fragrance +budge +concordia +magnesium +marcelo +##antes +gladly +vibrating +##rral +##ggles +montrose +##omba +lew +seamus +1630 +cocky +##ament +##uen +bjorn +##rrick +fielder +fluttering +##lase +methyl +kimberley +mcdowell +reductions +barbed +##jic +##tonic +aeronautical +condensed +distracting +##promising +huffed +##cala +##sle +claudius +invincible +missy +pious +balthazar +ci +##lang +butte +combo +orson +##dication +myriad +1707 +silenced +##fed +##rh +coco +netball +yourselves +##oza +clarify +heller +peg +durban +etudes +offender +roast +blackmail +curvature +##woods +vile +309 +illicit +suriname +##linson +overture +1685 +bubbling +gymnast +tucking +##mming +##ouin +maldives +##bala +gurney +##dda +##eased +##oides +backside +pinto +jars +racehorse +tending +##rdial +baronetcy +wiener +duly +##rke +barbarian +cupping +flawed +##thesis +bertha +pleistocene +puddle +swearing +##nob +##tically +fleeting +prostate +amulet +educating +##mined +##iti +##tler +75th +jens +respondents +analytics +cavaliers +papacy +raju +##iente +##ulum +##tip +funnel +271 +disneyland +##lley +sociologist +##iam +2500 +faulkner +louvre +menon +##dson +276 +##ower +afterlife +mannheim +peptide +referees +comedians +meaningless +##anger +##laise +fabrics +hurley +renal +sleeps +##bour +##icle +breakout +kristin +roadside +animator +clover +disdain +unsafe +redesign +##urity +firth +barnsley +portage +reset +narrows +268 +commandos +expansive +speechless +tubular +##lux +essendon +eyelashes +smashwords +##yad +##bang +##claim +craved +sprinted +chet +somme +astor +wrocław +orton +266 +bane +##erving +##uing +mischief +##amps +##sund +scaling +terre +##xious +impairment +offenses +undermine +moi +soy +contiguous +arcadia +inuit +seam +##tops +macbeth +rebelled +##icative +##iot +590 +elaborated +frs +uniformed +##dberg +259 +powerless +priscilla +stimulated +980 +qc +arboretum +frustrating +trieste +bullock +##nified +enriched +glistening +intern +##adia +locus +nouvelle +ollie +ike +lash +starboard +ee +tapestry +headlined +hove +rigged +##vite +pollock +##yme +thrive +clustered +cas +roi +gleamed +olympiad +##lino +pressured +regimes +##hosis +##lick +ripley +##ophone +kickoff +gallon +rockwell +##arable +crusader +glue +revolutions +scrambling +1714 +grover +##jure +englishman +aztec +263 +contemplating +coven +ipad +preach +triumphant +tufts +##esian +rotational +##phus +328 +falkland +##brates +strewn +clarissa +rejoin +environmentally +glint +banded +drenched +moat +albanians +johor +rr +maestro +malley +nouveau +shaded +taxonomy +v6 +adhere +bunk +airfields +##ritan +1741 +encompass +remington +tran +##erative +amelie +mazda +friar +morals +passions +##zai +breadth +vis +##hae +argus +burnham +caressing +insider +rudd +##imov +##mini +##rso +italianate +murderous +textual +wainwright +armada +bam +weave +timer +##taken +##nh +fra +##crest +ardent +salazar +taps +tunis +##ntino +allegro +gland +philanthropic +##chester +implication +##optera +esq +judas +noticeably +wynn +##dara +inched +indexed +crises +villiers +bandit +royalties +patterned +cupboard +interspersed +accessory +isla +kendrick +entourage +stitches +##esthesia +headwaters +##ior +interlude +distraught +draught +1727 +##basket +biased +sy +transient +triad +subgenus +adapting +kidd +shortstop +##umatic +dimly +spiked +mcleod +reprint +nellie +pretoria +windmill +##cek +singled +##mps +273 +reunite +##orous +747 +bankers +outlying +##omp +##ports +##tream +apologies +cosmetics +patsy +##deh +##ocks +##yson +bender +nantes +serene +##nad +lucha +mmm +323 +##cius +##gli +cmll +coinage +nestor +juarez +##rook +smeared +sprayed +twitching +sterile +irina +embodied +juveniles +enveloped +miscellaneous +cancers +dq +gulped +luisa +crested +swat +donegal +ref +##anov +##acker +hearst +mercantile +##lika +doorbell +ua +vicki +##alla +##som +bilbao +psychologists +stryker +sw +horsemen +turkmenistan +wits +##national +anson +mathew +screenings +##umb +rihanna +##agne +##nessy +aisles +##iani +##osphere +hines +kenton +saskatoon +tasha +truncated +##champ +##itan +mildred +advises +fredrik +interpreting +inhibitors +##athi +spectroscopy +##hab +##kong +karim +panda +##oia +##nail +##vc +conqueror +kgb +leukemia +##dity +arrivals +cheered +pisa +phosphorus +shielded +##riated +mammal +unitarian +urgently +chopin +sanitary +##mission +spicy +drugged +hinges +##tort +tipping +trier +impoverished +westchester +##caster +267 +epoch +nonstop +##gman +##khov +aromatic +centrally +cerro +##tively +##vio +billions +modulation +sedimentary +283 +facilitating +outrageous +goldstein +##eak +##kt +ld +maitland +penultimate +pollard +##dance +fleets +spaceship +vertebrae +##nig +alcoholism +als +recital +##bham +##ference +##omics +m2 +##bm +trois +##tropical +##в +commemorates +##meric +marge +##raction +1643 +670 +cosmetic +ravaged +##ige +catastrophe +eng +##shida +albrecht +arterial +bellamy +decor +harmon +##rde +bulbs +synchronized +vito +easiest +shetland +shielding +wnba +##glers +##ssar +##riam +brianna +cumbria +##aceous +##rard +cores +thayer +##nsk +brood +hilltop +luminous +carts +keynote +larkin +logos +##cta +##ا +##mund +##quay +lilith +tinted +277 +wrestle +mobilization +##uses +sequential +siam +bloomfield +takahashi +274 +##ieving +presenters +ringo +blazed +witty +##oven +##ignant +devastation +haydn +harmed +newt +therese +##peed +gershwin +molina +rabbis +sudanese +001 +innate +restarted +##sack +##fus +slices +wb +##shah +enroll +hypothetical +hysterical +1743 +fabio +indefinite +warped +##hg +exchanging +525 +unsuitable +##sboro +gallo +1603 +bret +cobalt +homemade +##hunter +mx +operatives +##dhar +terraces +durable +latch +pens +whorls +##ctuated +##eaux +billing +ligament +succumbed +##gly +regulators +spawn +##brick +##stead +filmfare +rochelle +##nzo +1725 +circumstance +saber +supplements +##nsky +##tson +crowe +wellesley +carrot +##9th +##movable +primate +drury +sincerely +topical +##mad +##rao +callahan +kyiv +smarter +tits +undo +##yeh +announcements +anthologies +barrio +nebula +##islaus +##shaft +##tyn +bodyguards +2021 +assassinate +barns +emmett +scully +##mah +##yd +##eland +##tino +##itarian +demoted +gorman +lashed +prized +adventist +writ +##gui +alla +invertebrates +##ausen +1641 +amman +1742 +align +healy +redistribution +##gf +##rize +insulation +##drop +adherents +hezbollah +vitro +ferns +yanking +269 +php +registering +uppsala +cheerleading +confines +mischievous +tully +##ross +49th +docked +roam +stipulated +pumpkin +##bry +prompt +##ezer +blindly +shuddering +craftsmen +frail +scented +katharine +scramble +shaggy +sponge +helix +zaragoza +279 +##52 +43rd +backlash +fontaine +seizures +posse +cowan +nonfiction +telenovela +wwii +hammered +undone +##gpur +encircled +irs +##ivation +artefacts +oneself +searing +smallpox +##belle +##osaurus +shandong +breached +upland +blushing +rankin +infinitely +psyche +tolerated +docking +evicted +##col +unmarked +##lving +gnome +lettering +litres +musique +##oint +benevolent +##jal +blackened +##anna +mccall +racers +tingle +##ocene +##orestation +introductions +radically +292 +##hiff +##باد +1610 +1739 +munchen +plead +##nka +condo +scissors +##sight +##tens +apprehension +##cey +##yin +hallmark +watering +formulas +sequels +##llas +aggravated +bae +commencing +##building +enfield +prohibits +marne +vedic +civilized +euclidean +jagger +beforehand +blasts +dumont +##arney +##nem +740 +conversions +hierarchical +rios +simulator +##dya +##lellan +hedges +oleg +thrusts +shadowed +darby +maximize +1744 +gregorian +##nded +##routed +sham +unspecified +##hog +emory +factual +##smo +##tp +fooled +##rger +ortega +wellness +marlon +##oton +##urance +casket +keating +ley +enclave +##ayan +char +influencing +jia +##chenko +412 +ammonia +erebidae +incompatible +violins +cornered +##arat +grooves +astronauts +columbian +rampant +fabrication +kyushu +mahmud +vanish +##dern +mesopotamia +##lete +ict +##rgen +caspian +kenji +pitted +##vered +999 +grimace +roanoke +tchaikovsky +twinned +##analysis +##awan +xinjiang +arias +clemson +kazakh +sizable +1662 +##khand +##vard +plunge +tatum +vittorio +##nden +cholera +##dana +##oper +bracing +indifference +projectile +superliga +##chee +realises +upgrading +299 +porte +retribution +##vies +nk +stil +##resses +ama +bureaucracy +blackberry +bosch +testosterone +collapses +greer +##pathic +ioc +fifties +malls +##erved +bao +baskets +adolescents +siegfried +##osity +##tosis +mantra +detecting +existent +fledgling +##cchi +dissatisfied +gan +telecommunication +mingled +sobbed +6000 +controversies +outdated +taxis +##raus +fright +slams +##lham +##fect +##tten +detectors +fetal +tanned +##uw +fray +goth +olympian +skipping +mandates +scratches +sheng +unspoken +hyundai +tracey +hotspur +restrictive +##buch +americana +mundo +##bari +burroughs +diva +vulcan +##6th +distinctions +thumping +##ngen +mikey +sheds +fide +rescues +springsteen +vested +valuation +##ece +##ely +pinnacle +rake +sylvie +##edo +almond +quivering +##irus +alteration +faltered +##wad +51st +hydra +ticked +##kato +recommends +##dicated +antigua +arjun +stagecoach +wilfred +trickle +pronouns +##pon +aryan +nighttime +##anian +gall +pea +stitch +##hei +leung +milos +##dini +eritrea +nexus +starved +snowfall +kant +parasitic +cot +discus +hana +strikers +appleton +kitchens +##erina +##partisan +##itha +##vius +disclose +metis +##channel +1701 +tesla +##vera +fitch +1735 +blooded +##tila +decimal +##tang +##bai +cyclones +eun +bottled +peas +pensacola +basha +bolivian +crabs +boil +lanterns +partridge +roofed +1645 +necks +##phila +opined +patting +##kla +##lland +chuckles +volta +whereupon +##nche +devout +euroleague +suicidal +##dee +inherently +involuntary +knitting +nasser +##hide +puppets +colourful +courageous +southend +stills +miraculous +hodgson +richer +rochdale +ethernet +greta +uniting +prism +umm +##haya +##itical +##utation +deterioration +pointe +prowess +##ropriation +lids +scranton +billings +subcontinent +##koff +##scope +brute +kellogg +psalms +degraded +##vez +stanisław +##ructured +ferreira +pun +astonishing +gunnar +##yat +arya +prc +gottfried +##tight +excursion +##ographer +dina +##quil +##nare +huffington +illustrious +wilbur +gundam +verandah +##zard +naacp +##odle +constructive +fjord +kade +##naud +generosity +thrilling +baseline +cayman +frankish +plastics +accommodations +zoological +##fting +cedric +qb +motorized +##dome +##otted +squealed +tackled +canucks +budgets +situ +asthma +dail +gabled +grasslands +whimpered +writhing +judgments +##65 +minnie +pv +##carbon +bananas +grille +domes +monique +odin +maguire +markham +tierney +##estra +##chua +libel +poke +speedy +atrium +laval +notwithstanding +##edly +fai +kala +##sur +robb +##sma +listings +luz +supplementary +tianjin +##acing +enzo +jd +ric +scanner +croats +transcribed +##49 +arden +cv +##hair +##raphy +##lver +##uy +357 +seventies +staggering +alam +horticultural +hs +regression +timbers +blasting +##ounded +montagu +manipulating +##cit +catalytic +1550 +troopers +##meo +condemnation +fitzpatrick +##oire +##roved +inexperienced +1670 +castes +##lative +outing +314 +dubois +flicking +quarrel +ste +learners +1625 +iq +whistled +##class +282 +classify +tariffs +temperament +355 +folly +liszt +##yles +immersed +jordanian +ceasefire +apparel +extras +maru +fished +##bio +harta +stockport +assortment +craftsman +paralysis +transmitters +##cola +blindness +##wk +fatally +proficiency +solemnly +##orno +repairing +amore +groceries +ultraviolet +##chase +schoolhouse +##tua +resurgence +nailed +##otype +##× +ruse +saliva +diagrams +##tructing +albans +rann +thirties +1b +antennas +hilarious +cougars +paddington +stats +##eger +breakaway +ipod +reza +authorship +prohibiting +scoffed +##etz +##ttle +conscription +defected +trondheim +##fires +ivanov +keenan +##adan +##ciful +##fb +##slow +locating +##ials +##tford +cadiz +basalt +blankly +interned +rags +rattling +##tick +carpathian +reassured +sync +bum +guildford +iss +staunch +##onga +astronomers +sera +sofie +emergencies +susquehanna +##heard +duc +mastery +vh1 +williamsburg +bayer +buckled +craving +##khan +##rdes +bloomington +##write +alton +barbecue +##bians +justine +##hri +##ndt +delightful +smartphone +newtown +photon +retrieval +peugeot +hissing +##monium +##orough +flavors +lighted +relaunched +tainted +##games +##lysis +anarchy +microscopic +hopping +adept +evade +evie +##beau +inhibit +sinn +adjustable +hurst +intuition +wilton +cisco +44th +lawful +lowlands +stockings +thierry +##dalen +##hila +##nai +fates +prank +tb +maison +lobbied +provocative +1724 +4a +utopia +##qual +carbonate +gujarati +purcell +##rford +curtiss +##mei +overgrown +arenas +mediation +swallows +##rnik +respectful +turnbull +##hedron +##hope +alyssa +ozone +##ʻi +ami +gestapo +johansson +snooker +canteen +cuff +declines +empathy +stigma +##ags +##iner +##raine +taxpayers +gui +volga +##wright +##copic +lifespan +overcame +tattooed +enactment +giggles +##ador +##camp +barrington +bribe +obligatory +orbiting +peng +##enas +elusive +sucker +##vating +cong +hardship +empowered +anticipating +estrada +cryptic +greasy +detainees +planck +sudbury +plaid +dod +marriott +kayla +##ears +##vb +##zd +mortally +##hein +cognition +radha +319 +liechtenstein +meade +richly +argyle +harpsichord +liberalism +trumpets +lauded +tyrant +salsa +tiled +lear +promoters +reused +slicing +trident +##chuk +##gami +##lka +cantor +checkpoint +##points +gaul +leger +mammalian +##tov +##aar +##schaft +doha +frenchman +nirvana +##vino +delgado +headlining +##eron +##iography +jug +tko +1649 +naga +intersections +##jia +benfica +nawab +##suka +ashford +gulp +##deck +##vill +##rug +brentford +frazier +pleasures +dunne +potsdam +shenzhen +dentistry +##tec +flanagan +##dorff +##hear +chorale +dinah +prem +quezon +##rogated +relinquished +sutra +terri +##pani +flaps +##rissa +poly +##rnet +homme +aback +##eki +linger +womb +##kson +##lewood +doorstep +orthodoxy +threaded +westfield +##rval +dioceses +fridays +subsided +##gata +loyalists +##biotic +##ettes +letterman +lunatic +prelate +tenderly +invariably +souza +thug +winslow +##otide +furlongs +gogh +jeopardy +##runa +pegasus +##umble +humiliated +standalone +tagged +##roller +freshmen +klan +##bright +attaining +initiating +transatlantic +logged +viz +##uance +1723 +combatants +intervening +stephane +chieftain +despised +grazed +317 +cdc +galveston +godzilla +macro +simulate +##planes +parades +##esses +960 +##ductive +##unes +equator +overdose +##cans +##hosh +##lifting +joshi +epstein +sonora +treacherous +aquatics +manchu +responsive +##sation +supervisory +##christ +##llins +##ibar +##balance +##uso +kimball +karlsruhe +mab +##emy +ignores +phonetic +reuters +spaghetti +820 +almighty +danzig +rumbling +tombstone +designations +lured +outset +##felt +supermarkets +##wt +grupo +kei +kraft +susanna +##blood +comprehension +genealogy +##aghan +##verted +redding +##ythe +1722 +bowing +##pore +##roi +lest +sharpened +fulbright +valkyrie +sikhs +##unds +swans +bouquet +merritt +##tage +##venting +commuted +redhead +clerks +leasing +cesare +dea +hazy +##vances +fledged +greenfield +servicemen +##gical +armando +blackout +dt +sagged +downloadable +intra +potion +pods +##4th +##mism +xp +attendants +gambia +stale +##ntine +plump +asteroids +rediscovered +buds +flea +hive +##neas +1737 +classifications +debuts +##eles +olympus +scala +##eurs +##gno +##mute +hummed +sigismund +visuals +wiggled +await +pilasters +clench +sulfate +##ances +bellevue +enigma +trainee +snort +##sw +clouded +denim +##rank +##rder +churning +hartman +lodges +riches +sima +##missible +accountable +socrates +regulates +mueller +##cr +1702 +avoids +solids +himalayas +nutrient +pup +##jevic +squat +fades +nec +##lates +##pina +##rona +##ου +privateer +tequila +##gative +##mpton +apt +hornet +immortals +##dou +asturias +cleansing +dario +##rries +##anta +etymology +servicing +zhejiang +##venor +##nx +horned +erasmus +rayon +relocating +£10 +##bags +escalated +promenade +stubble +2010s +artisans +axial +liquids +mora +sho +yoo +##tsky +bundles +oldies +##nally +notification +bastion +##ths +sparkle +##lved +1728 +leash +pathogen +highs +##hmi +immature +880 +gonzaga +ignatius +mansions +monterrey +sweets +bryson +##loe +polled +regatta +brightest +pei +rosy +squid +hatfield +payroll +addict +meath +cornerback +heaviest +lodging +##mage +capcom +rippled +##sily +barnet +mayhem +ymca +snuggled +rousseau +##cute +blanchard +284 +fragmented +leighton +chromosomes +risking +##md +##strel +##utter +corinne +coyotes +cynical +hiroshi +yeomanry +##ractive +ebook +grading +mandela +plume +agustin +magdalene +##rkin +bea +femme +trafford +##coll +##lun +##tance +52nd +fourier +upton +##mental +camilla +gust +iihf +islamabad +longevity +##kala +feldman +netting +##rization +endeavour +foraging +mfa +orr +##open +greyish +contradiction +graz +##ruff +handicapped +marlene +tweed +oaxaca +spp +campos +miocene +pri +configured +cooks +pluto +cozy +pornographic +##entes +70th +fairness +glided +jonny +lynne +rounding +sired +##emon +##nist +remade +uncover +##mack +complied +lei +newsweek +##jured +##parts +##enting +##pg +293 +finer +guerrillas +athenian +deng +disused +stepmother +accuse +gingerly +seduction +521 +confronting +##walker +##going +gora +nostalgia +sabres +virginity +wrenched +##minated +syndication +wielding +eyre +##56 +##gnon +##igny +behaved +taxpayer +sweeps +##growth +childless +gallant +##ywood +amplified +geraldine +scrape +##ffi +babylonian +fresco +##rdan +##kney +##position +1718 +restricting +tack +fukuoka +osborn +selector +partnering +##dlow +318 +gnu +kia +tak +whitley +gables +##54 +##mania +mri +softness +immersion +##bots +##evsky +1713 +chilling +insignificant +pcs +##uis +elites +lina +purported +supplemental +teaming +##americana +##dding +##inton +proficient +rouen +##nage +##rret +niccolo +selects +##bread +fluffy +1621 +gruff +knotted +mukherjee +polgara +thrash +nicholls +secluded +smoothing +thru +corsica +loaf +whitaker +inquiries +##rrier +##kam +indochina +289 +marlins +myles +peking +##tea +extracts +pastry +superhuman +connacht +vogel +##ditional +##het +##udged +##lash +gloss +quarries +refit +teaser +##alic +##gaon +20s +materialized +sling +camped +pickering +tung +tracker +pursuant +##cide +cranes +soc +##cini +##typical +##viere +anhalt +overboard +workout +chores +fares +orphaned +stains +##logie +fenton +surpassing +joyah +triggers +##itte +grandmaster +##lass +##lists +clapping +fraudulent +ledger +nagasaki +##cor +##nosis +##tsa +eucalyptus +tun +##icio +##rney +##tara +dax +heroism +ina +wrexham +onboard +unsigned +##dates +moshe +galley +winnie +droplets +exiles +praises +watered +noodles +##aia +fein +adi +leland +multicultural +stink +bingo +comets +erskine +modernized +canned +constraint +domestically +chemotherapy +featherweight +stifled +##mum +darkly +irresistible +refreshing +hasty +isolate +##oys +kitchener +planners +##wehr +cages +yarn +implant +toulon +elects +childbirth +yue +##lind +##lone +cn +rightful +sportsman +junctions +remodeled +specifies +##rgh +291 +##oons +complimented +##urgent +lister +ot +##logic +bequeathed +cheekbones +fontana +gabby +##dial +amadeus +corrugated +maverick +resented +triangles +##hered +##usly +nazareth +tyrol +1675 +assent +poorer +sectional +aegean +##cous +296 +nylon +ghanaian +##egorical +##weig +cushions +forbid +fusiliers +obstruction +somerville +##scia +dime +earrings +elliptical +leyte +oder +polymers +timmy +atm +midtown +piloted +settles +continual +externally +mayfield +##uh +enrichment +henson +keane +persians +1733 +benji +braden +pep +324 +##efe +contenders +pepsi +valet +##isches +298 +##asse +##earing +goofy +stroll +##amen +authoritarian +occurrences +adversary +ahmedabad +tangent +toppled +dorchester +1672 +modernism +marxism +islamist +charlemagne +exponential +racks +unicode +brunette +mbc +pic +skirmish +##bund +##lad +##powered +##yst +hoisted +messina +shatter +##ctum +jedi +vantage +##music +##neil +clemens +mahmoud +corrupted +authentication +lowry +nils +##washed +omnibus +wounding +jillian +##itors +##opped +serialized +narcotics +handheld +##arm +##plicity +intersecting +stimulating +##onis +crate +fellowships +hemingway +casinos +climatic +fordham +copeland +drip +beatty +leaflets +robber +brothel +madeira +##hedral +sphinx +ultrasound +##vana +valor +forbade +leonid +villas +##aldo +duane +marquez +##cytes +disadvantaged +forearms +kawasaki +reacts +consular +lax +uncles +uphold +##hopper +concepcion +dorsey +lass +##izan +arching +passageway +1708 +researches +tia +internationals +##graphs +##opers +distinguishes +javanese +divert +##uven +plotted +##listic +##rwin +##erik +##tify +affirmative +signifies +validation +##bson +kari +felicity +georgina +zulu +##eros +##rained +##rath +overcoming +##dot +argyll +##rbin +1734 +chiba +ratification +windy +earls +parapet +##marks +hunan +pristine +astrid +punta +##gart +brodie +##kota +##oder +malaga +minerva +rouse +##phonic +bellowed +pagoda +portals +reclamation +##gur +##odies +##⁄₄ +parentheses +quoting +allergic +palette +showcases +benefactor +heartland +nonlinear +##tness +bladed +cheerfully +scans +##ety +##hone +1666 +girlfriends +pedersen +hiram +sous +##liche +##nator +1683 +##nery +##orio +##umen +bobo +primaries +smiley +##cb +unearthed +uniformly +fis +metadata +1635 +ind +##oted +recoil +##titles +##tura +##ια +406 +hilbert +jamestown +mcmillan +tulane +seychelles +##frid +antics +coli +fated +stucco +##grants +1654 +bulky +accolades +arrays +caledonian +carnage +optimism +puebla +##tative +##cave +enforcing +rotherham +seo +dunlop +aeronautics +chimed +incline +zoning +archduke +hellenistic +##oses +##sions +candi +thong +##ople +magnate +rustic +##rsk +projective +slant +##offs +danes +hollis +vocalists +##ammed +congenital +contend +gesellschaft +##ocating +##pressive +douglass +quieter +##cm +##kshi +howled +salim +spontaneously +townsville +buena +southport +##bold +kato +1638 +faerie +stiffly +##vus +##rled +297 +flawless +realising +taboo +##7th +bytes +straightening +356 +jena +##hid +##rmin +cartwright +berber +bertram +soloists +411 +noses +417 +coping +fission +hardin +inca +##cen +1717 +mobilized +vhf +##raf +biscuits +curate +##85 +##anial +331 +gaunt +neighbourhoods +1540 +##abas +blanca +bypassed +sockets +behold +coincidentally +##bane +nara +shave +splinter +terrific +##arion +##erian +commonplace +juris +redwood +waistband +boxed +caitlin +fingerprints +jennie +naturalized +##ired +balfour +craters +jody +bungalow +hugely +quilt +glitter +pigeons +undertaker +bulging +constrained +goo +##sil +##akh +assimilation +reworked +##person +persuasion +##pants +felicia +##cliff +##ulent +1732 +explodes +##dun +##inium +##zic +lyman +vulture +hog +overlook +begs +northwards +ow +spoil +##urer +fatima +favorably +accumulate +sargent +sorority +corresponded +dispersal +kochi +toned +##imi +##lita +internacional +newfound +##agger +##lynn +##rigue +booths +peanuts +##eborg +medicare +muriel +nur +##uram +crates +millennia +pajamas +worsened +##breakers +jimi +vanuatu +yawned +##udeau +carousel +##hony +hurdle +##ccus +##mounted +##pod +rv +##eche +airship +ambiguity +compulsion +recapture +##claiming +arthritis +##osomal +1667 +asserting +ngc +sniffing +dade +discontent +glendale +ported +##amina +defamation +rammed +##scent +fling +livingstone +##fleet +875 +##ppy +apocalyptic +comrade +lcd +##lowe +cessna +eine +persecuted +subsistence +demi +hoop +reliefs +710 +coptic +progressing +stemmed +perpetrators +1665 +priestess +##nio +dobson +ebony +rooster +itf +tortricidae +##bbon +##jian +cleanup +##jean +##øy +1721 +eighties +taxonomic +holiness +##hearted +##spar +antilles +showcasing +stabilized +##nb +gia +mascara +michelangelo +dawned +##uria +##vinsky +extinguished +fitz +grotesque +£100 +##fera +##loid +##mous +barges +neue +throbbed +cipher +johnnie +##a1 +##mpt +outburst +##swick +spearheaded +administrations +c1 +heartbreak +pixels +pleasantly +##enay +lombardy +plush +##nsed +bobbie +##hly +reapers +tremor +xiang +minogue +substantive +hitch +barak +##wyl +kwan +##encia +910 +obscene +elegance +indus +surfer +bribery +conserve +##hyllum +##masters +horatio +##fat +apes +rebound +psychotic +##pour +iteration +##mium +##vani +botanic +horribly +antiques +dispose +paxton +##hli +##wg +timeless +1704 +disregard +engraver +hounds +##bau +##version +looted +uno +facilitates +groans +masjid +rutland +antibody +disqualification +decatur +footballers +quake +slacks +48th +rein +scribe +stabilize +commits +exemplary +tho +##hort +##chison +pantry +traversed +##hiti +disrepair +identifiable +vibrated +baccalaureate +##nnis +csa +interviewing +##iensis +##raße +greaves +wealthiest +343 +classed +jogged +£5 +##58 +##atal +illuminating +knicks +respecting +##uno +scrubbed +##iji +##dles +kruger +moods +growls +raider +silvia +chefs +kam +vr +cree +percival +##terol +gunter +counterattack +defiant +henan +ze +##rasia +##riety +equivalence +submissions +##fra +##thor +bautista +mechanically +##heater +cornice +herbal +templar +##mering +outputs +ruining +ligand +renumbered +extravagant +mika +blockbuster +eta +insurrection +##ilia +darkening +ferocious +pianos +strife +kinship +##aer +melee +##anor +##iste +##may +##oue +decidedly +weep +##jad +##missive +##ppel +354 +puget +unease +##gnant +1629 +hammering +kassel +ob +wessex +##lga +bromwich +egan +paranoia +utilization +##atable +##idad +contradictory +provoke +##ols +##ouring +##tangled +knesset +##very +##lette +plumbing +##sden +##¹ +greensboro +occult +sniff +338 +zev +beaming +gamer +haggard +mahal +##olt +##pins +mendes +utmost +briefing +gunnery +##gut +##pher +##zh +##rok +1679 +khalifa +sonya +##boot +principals +urbana +wiring +##liffe +##minating +##rrado +dahl +nyu +skepticism +np +townspeople +ithaca +lobster +somethin +##fur +##arina +##−1 +freighter +zimmerman +biceps +contractual +##herton +amend +hurrying +subconscious +##anal +336 +meng +clermont +spawning +##eia +##lub +dignitaries +impetus +snacks +spotting +twigs +##bilis +##cz +##ouk +libertadores +nic +skylar +##aina +##firm +gustave +asean +##anum +dieter +legislatures +flirt +bromley +trolls +umar +##bbies +##tyle +blah +parc +bridgeport +crank +negligence +##nction +46th +constantin +molded +bandages +seriousness +00pm +siegel +carpets +compartments +upbeat +statehood +##dner +##edging +marko +730 +platt +##hane +paving +##iy +1738 +abbess +impatience +limousine +nbl +##talk +441 +lucille +mojo +nightfall +robbers +##nais +karel +brisk +calves +replicate +ascribed +telescopes +##olf +intimidated +##reen +ballast +specialization +##sit +aerodynamic +caliphate +rainer +visionary +##arded +epsilon +##aday +##onte +aggregation +auditory +boosted +reunification +kathmandu +loco +robyn +402 +acknowledges +appointing +humanoid +newell +redeveloped +restraints +##tained +barbarians +chopper +1609 +italiana +##lez +##lho +investigates +wrestlemania +##anies +##bib +690 +##falls +creaked +dragoons +gravely +minions +stupidity +volley +##harat +##week +musik +##eries +##uously +fungal +massimo +semantics +malvern +##ahl +##pee +discourage +embryo +imperialism +1910s +profoundly +##ddled +jiangsu +sparkled +stat +##holz +sweatshirt +tobin +##iction +sneered +##cheon +##oit +brit +causal +smyth +##neuve +diffuse +perrin +silvio +##ipes +##recht +detonated +iqbal +selma +##nism +##zumi +roasted +##riders +tay +##ados +##mament +##mut +##rud +840 +completes +nipples +cfa +flavour +hirsch +##laus +calderon +sneakers +moravian +##ksha +1622 +rq +294 +##imeters +bodo +##isance +##pre +##ronia +anatomical +excerpt +##lke +dh +kunst +##tablished +##scoe +biomass +panted +unharmed +gael +housemates +montpellier +##59 +coa +rodents +tonic +hickory +singleton +##taro +451 +1719 +aldo +breaststroke +dempsey +och +rocco +##cuit +merton +dissemination +midsummer +serials +##idi +haji +polynomials +##rdon +gs +enoch +prematurely +shutter +taunton +£3 +##grating +##inates +archangel +harassed +##asco +326 +archway +dazzling +##ecin +1736 +sumo +wat +##kovich +1086 +honneur +##ently +##nostic +##ttal +##idon +1605 +403 +1716 +blogger +rents +##gnan +hires +##ikh +##dant +howie +##rons +handler +retracted +shocks +1632 +arun +duluth +kepler +trumpeter +##lary +peeking +seasoned +trooper +##mara +laszlo +##iciencies +##rti +heterosexual +##inatory +##ssion +indira +jogging +##inga +##lism +beit +dissatisfaction +malice +##ately +nedra +peeling +##rgeon +47th +stadiums +475 +vertigo +##ains +iced +restroom +##plify +##tub +illustrating +pear +##chner +##sibility +inorganic +rappers +receipts +watery +##kura +lucinda +##oulos +reintroduced +##8th +##tched +gracefully +saxons +nutritional +wastewater +rained +favourites +bedrock +fisted +hallways +likeness +upscale +##lateral +1580 +blinds +prequel +##pps +##tama +deter +humiliating +restraining +tn +vents +1659 +laundering +recess +rosary +tractors +coulter +federer +##ifiers +##plin +persistence +##quitable +geschichte +pendulum +quakers +##beam +bassett +pictorial +buffet +koln +##sitor +drills +reciprocal +shooters +##57 +##cton +##tees +converge +pip +dmitri +donnelly +yamamoto +aqua +azores +demographics +hypnotic +spitfire +suspend +wryly +roderick +##rran +sebastien +##asurable +mavericks +##fles +##200 +himalayan +prodigy +##iance +transvaal +demonstrators +handcuffs +dodged +mcnamara +sublime +1726 +crazed +##efined +##till +ivo +pondered +reconciled +shrill +sava +##duk +bal +cad +heresy +jaipur +goran +##nished +341 +lux +shelly +whitehall +##hre +israelis +peacekeeping +##wled +1703 +demetrius +ousted +##arians +##zos +beale +anwar +backstroke +raged +shrinking +cremated +##yck +benign +towing +wadi +darmstadt +landfill +parana +soothe +colleen +sidewalks +mayfair +tumble +hepatitis +ferrer +superstructure +##gingly +##urse +##wee +anthropological +translators +##mies +closeness +hooves +##pw +mondays +##roll +##vita +landscaping +##urized +purification +sock +thorns +thwarted +jalan +tiberius +##taka +saline +##rito +confidently +khyber +sculptors +##ij +brahms +hammersmith +inspectors +battista +fivb +fragmentation +hackney +##uls +arresting +exercising +antoinette +bedfordshire +##zily +dyed +##hema +1656 +racetrack +variability +##tique +1655 +austrians +deteriorating +madman +theorists +aix +lehman +weathered +1731 +decreed +eruptions +1729 +flaw +quinlan +sorbonne +flutes +nunez +1711 +adored +downwards +fable +rasped +1712 +moritz +mouthful +renegade +shivers +stunts +dysfunction +restrain +translit +327 +pancakes +##avio +##cision +##tray +351 +vial +##lden +bain +##maid +##oxide +chihuahua +malacca +vimes +##rba +##rnier +1664 +donnie +plaques +##ually +337 +bangs +floppy +huntsville +loretta +nikolay +##otte +eater +handgun +ubiquitous +##hett +eras +zodiac +1634 +##omorphic +1820s +##zog +cochran +##bula +##lithic +warring +##rada +dalai +excused +blazers +mcconnell +reeling +bot +este +##abi +geese +hoax +taxon +##bla +guitarists +##icon +condemning +hunts +inversion +moffat +taekwondo +##lvis +1624 +stammered +##rest +##rzy +sousa +fundraiser +marylebone +navigable +uptown +cabbage +daniela +salman +shitty +whimper +##kian +##utive +programmers +protections +rm +##rmi +##rued +forceful +##enes +fuss +##tao +##wash +brat +oppressive +reykjavik +spartak +ticking +##inkles +##kiewicz +adolph +horst +maui +protege +straighten +cpc +landau +concourse +clements +resultant +##ando +imaginative +joo +reactivated +##rem +##ffled +##uising +consultative +##guide +flop +kaitlyn +mergers +parenting +somber +##vron +supervise +vidhan +##imum +courtship +exemplified +harmonies +medallist +refining +##rrow +##ка +amara +##hum +780 +goalscorer +sited +overshadowed +rohan +displeasure +secretive +multiplied +osman +##orth +engravings +padre +##kali +##veda +miniatures +mis +##yala +clap +pali +rook +##cana +1692 +57th +antennae +astro +oskar +1628 +bulldog +crotch +hackett +yucatan +##sure +amplifiers +brno +ferrara +migrating +##gree +thanking +turing +##eza +mccann +ting +andersson +onslaught +gaines +ganga +incense +standardization +##mation +sentai +scuba +stuffing +turquoise +waivers +alloys +##vitt +regaining +vaults +##clops +##gizing +digger +furry +memorabilia +probing +##iad +payton +rec +deutschland +filippo +opaque +seamen +zenith +afrikaans +##filtration +disciplined +inspirational +##merie +banco +confuse +grafton +tod +##dgets +championed +simi +anomaly +biplane +##ceptive +electrode +##para +1697 +cleavage +crossbow +swirl +informant +##lars +##osta +afi +bonfire +spec +##oux +lakeside +slump +##culus +##lais +##qvist +##rrigan +1016 +facades +borg +inwardly +cervical +xl +pointedly +050 +stabilization +##odon +chests +1699 +hacked +ctv +orthogonal +suzy +##lastic +gaulle +jacobite +rearview +##cam +##erted +ashby +##drik +##igate +##mise +##zbek +affectionately +canine +disperse +latham +##istles +##ivar +spielberg +##orin +##idium +ezekiel +cid +##sg +durga +middletown +##cina +customized +frontiers +harden +##etano +##zzy +1604 +bolsheviks +##66 +coloration +yoko +##bedo +briefs +slabs +debra +liquidation +plumage +##oin +blossoms +dementia +subsidy +1611 +proctor +relational +jerseys +parochial +ter +##ici +esa +peshawar +cavalier +loren +cpi +idiots +shamrock +1646 +dutton +malabar +mustache +##endez +##ocytes +referencing +terminates +marche +yarmouth +##sop +acton +mated +seton +subtly +baptised +beige +extremes +jolted +kristina +telecast +##actic +safeguard +waldo +##baldi +##bular +endeavors +sloppy +subterranean +##ensburg +##itung +delicately +pigment +tq +##scu +1626 +##ound +collisions +coveted +herds +##personal +##meister +##nberger +chopra +##ricting +abnormalities +defective +galician +lucie +##dilly +alligator +likened +##genase +burundi +clears +complexion +derelict +deafening +diablo +fingered +champaign +dogg +enlist +isotope +labeling +mrna +##erre +brilliance +marvelous +##ayo +1652 +crawley +ether +footed +dwellers +deserts +hamish +rubs +warlock +skimmed +##lizer +870 +buick +embark +heraldic +irregularities +##ajan +kiara +##kulam +##ieg +antigen +kowalski +##lge +oakley +visitation +##mbit +vt +##suit +1570 +murderers +##miento +##rites +chimneys +##sling +condemn +custer +exchequer +havre +##ghi +fluctuations +##rations +dfb +hendricks +vaccines +##tarian +nietzsche +biking +juicy +##duced +brooding +scrolling +selangor +##ragan +352 +annum +boomed +seminole +sugarcane +##dna +departmental +dismissing +innsbruck +arteries +ashok +batavia +daze +kun +overtook +##rga +##tlan +beheaded +gaddafi +holm +electronically +faulty +galilee +fractures +kobayashi +##lized +gunmen +magma +aramaic +mala +eastenders +inference +messengers +bf +##qu +407 +bathrooms +##vere +1658 +flashbacks +ideally +misunderstood +##jali +##weather +mendez +##grounds +505 +uncanny +##iii +1709 +friendships +##nbc +sacrament +accommodated +reiterated +logistical +pebbles +thumped +##escence +administering +decrees +drafts +##flight +##cased +##tula +futuristic +picket +intimidation +winthrop +##fahan +interfered +339 +afar +francoise +morally +uta +cochin +croft +dwarfs +##bruck +##dents +##nami +biker +##hner +##meral +nano +##isen +##ometric +##pres +##ан +brightened +meek +parcels +securely +gunners +##jhl +##zko +agile +hysteria +##lten +##rcus +bukit +champs +chevy +cuckoo +leith +sadler +theologians +welded +##section +1663 +jj +plurality +xander +##rooms +##formed +shredded +temps +intimately +pau +tormented +##lok +##stellar +1618 +charred +ems +essen +##mmel +alarms +spraying +ascot +blooms +twinkle +##abia +##apes +internment +obsidian +##chaft +snoop +##dav +##ooping +malibu +##tension +quiver +##itia +hays +mcintosh +travers +walsall +##ffie +1623 +beverley +schwarz +plunging +structurally +m3 +rosenthal +vikram +##tsk +770 +ghz +##onda +##tiv +chalmers +groningen +pew +reckon +unicef +##rvis +55th +##gni +1651 +sulawesi +avila +cai +metaphysical +screwing +turbulence +##mberg +augusto +samba +56th +baffled +momentary +toxin +##urian +##wani +aachen +condoms +dali +steppe +##3d +##app +##oed +##year +adolescence +dauphin +electrically +inaccessible +microscopy +nikita +##ega +atv +##cel +##enter +##oles +##oteric +##ы +accountants +punishments +wrongly +bribes +adventurous +clinch +flinders +southland +##hem +##kata +gough +##ciency +lads +soared +##ה +undergoes +deformation +outlawed +rubbish +##arus +##mussen +##nidae +##rzburg +arcs +##ingdon +##tituted +1695 +wheelbase +wheeling +bombardier +campground +zebra +##lices +##oj +##bain +lullaby +##ecure +donetsk +wylie +grenada +##arding +##ης +squinting +eireann +opposes +##andra +maximal +runes +##broken +##cuting +##iface +##ror +##rosis +additive +britney +adultery +triggering +##drome +detrimental +aarhus +containment +jc +swapped +vichy +##ioms +madly +##oric +##rag +brant +##ckey +##trix +1560 +1612 +broughton +rustling +##stems +##uder +asbestos +mentoring +##nivorous +finley +leaps +##isan +apical +pry +slits +substitutes +##dict +intuitive +fantasia +insistent +unreasonable +##igen +##vna +domed +hannover +margot +ponder +##zziness +impromptu +jian +lc +rampage +stemming +##eft +andrey +gerais +whichever +amnesia +appropriated +anzac +clicks +modifying +ultimatum +cambrian +maids +verve +yellowstone +##mbs +conservatoire +##scribe +adherence +dinners +spectra +imperfect +mysteriously +sidekick +tatar +tuba +##aks +##ifolia +distrust +##athan +##zle +c2 +ronin +zac +##pse +celaena +instrumentalist +scents +skopje +##mbling +comical +compensated +vidal +condor +intersect +jingle +wavelengths +##urrent +mcqueen +##izzly +carp +weasel +422 +kanye +militias +postdoctoral +eugen +gunslinger +##ɛ +faux +hospice +##for +appalled +derivation +dwarves +##elis +dilapidated +##folk +astoria +philology +##lwyn +##otho +##saka +inducing +philanthropy +##bf +##itative +geek +markedly +sql +##yce +bessie +indices +rn +##flict +495 +frowns +resolving +weightlifting +tugs +cleric +contentious +1653 +mania +rms +##miya +##reate +##ruck +##tucket +bien +eels +marek +##ayton +##cence +discreet +unofficially +##ife +leaks +##bber +1705 +332 +dung +compressor +hillsborough +pandit +shillings +distal +##skin +381 +##tat +##you +nosed +##nir +mangrove +undeveloped +##idia +textures +##inho +##500 +##rise +ae +irritating +nay +amazingly +bancroft +apologetic +compassionate +kata +symphonies +##lovic +airspace +##lch +930 +gifford +precautions +fulfillment +sevilla +vulgar +martinique +##urities +looting +piccolo +tidy +##dermott +quadrant +armchair +incomes +mathematicians +stampede +nilsson +##inking +##scan +foo +quarterfinal +##ostal +shang +shouldered +squirrels +##owe +344 +vinegar +##bner +##rchy +##systems +delaying +##trics +ars +dwyer +rhapsody +sponsoring +##gration +bipolar +cinder +starters +##olio +##urst +421 +signage +##nty +aground +figurative +mons +acquaintances +duets +erroneously +soyuz +elliptic +recreated +##cultural +##quette +##ssed +##tma +##zcz +moderator +scares +##itaire +##stones +##udence +juniper +sighting +##just +##nsen +britten +calabria +ry +bop +cramer +forsyth +stillness +##л +airmen +gathers +unfit +##umber +##upt +taunting +##rip +seeker +streamlined +##bution +holster +schumann +tread +vox +##gano +##onzo +strive +dil +reforming +covent +newbury +predicting +##orro +decorate +tre +##puted +andover +ie +asahi +dept +dunkirk +gills +##tori +buren +huskies +##stis +##stov +abstracts +bets +loosen +##opa +1682 +yearning +##glio +##sir +berman +effortlessly +enamel +napoli +persist +##peration +##uez +attache +elisa +b1 +invitations +##kic +accelerating +reindeer +boardwalk +clutches +nelly +polka +starbucks +##kei +adamant +huey +lough +unbroken +adventurer +embroidery +inspecting +stanza +##ducted +naia +taluka +##pone +##roids +chases +deprivation +florian +##jing +##ppet +earthly +##lib +##ssee +colossal +foreigner +vet +freaks +patrice +rosewood +triassic +upstate +##pkins +dominates +ata +chants +ks +vo +##400 +##bley +##raya +##rmed +555 +agra +infiltrate +##ailing +##ilation +##tzer +##uppe +##werk +binoculars +enthusiast +fujian +squeak +##avs +abolitionist +almeida +boredom +hampstead +marsden +rations +##ands +inflated +334 +bonuses +rosalie +patna +##rco +329 +detachments +penitentiary +54th +flourishing +woolf +##dion +##etched +papyrus +##lster +##nsor +##toy +bobbed +dismounted +endelle +inhuman +motorola +tbs +wince +wreath +##ticus +hideout +inspections +sanjay +disgrace +infused +pudding +stalks +##urbed +arsenic +leases +##hyl +##rrard +collarbone +##waite +##wil +dowry +##bant +##edance +genealogical +nitrate +salamanca +scandals +thyroid +necessitated +##! +##" +### +##$ +##% +##& +##' +##( +##) +##* +##+ +##, +##- +##. +##/ +##: +##; +##< +##= +##> +##? +##@ +##[ +##\ +##] +##^ +##_ +##` +##{ +##| +##} +##~ +##¡ +##¢ +##£ +##¤ +##¥ +##¦ +##§ +##¨ +##© +##ª +##« +##¬ +##® +##± +##´ +##µ +##¶ +##· +##º +##» +##¼ +##¾ +##¿ +##æ +##ð +##÷ +##þ +##đ +##ħ +##ŋ +##œ +##ƒ +##ɐ +##ɑ +##ɒ +##ɔ +##ɕ +##ə +##ɡ +##ɣ +##ɨ +##ɪ +##ɫ +##ɬ +##ɯ +##ɲ +##ɴ +##ɹ +##ɾ +##ʀ +##ʁ +##ʂ +##ʃ +##ʉ +##ʊ +##ʋ +##ʌ +##ʎ +##ʐ +##ʑ +##ʒ +##ʔ +##ʰ +##ʲ +##ʳ +##ʷ +##ʸ +##ʻ +##ʼ +##ʾ +##ʿ +##ˈ +##ˡ +##ˢ +##ˣ +##ˤ +##β +##γ +##δ +##ε +##ζ +##θ +##κ +##λ +##μ +##ξ +##ο +##π +##ρ +##σ +##τ +##υ +##φ +##χ +##ψ +##ω +##б +##г +##д +##ж +##з +##м +##п +##с +##у +##ф +##х +##ц +##ч +##ш +##щ +##ъ +##э +##ю +##ђ +##є +##і +##ј +##љ +##њ +##ћ +##ӏ +##ա +##բ +##գ +##դ +##ե +##թ +##ի +##լ +##կ +##հ +##մ +##յ +##ն +##ո +##պ +##ս +##վ +##տ +##ր +##ւ +##ք +##־ +##א +##ב +##ג +##ד +##ו +##ז +##ח +##ט +##י +##ך +##כ +##ל +##ם +##מ +##ן +##נ +##ס +##ע +##ף +##פ +##ץ +##צ +##ק +##ר +##ש +##ת +##، +##ء +##ب +##ت +##ث +##ج +##ح +##خ +##ذ +##ز +##س +##ش +##ص +##ض +##ط +##ظ +##ع +##غ +##ـ +##ف +##ق +##ك +##و +##ى +##ٹ +##پ +##چ +##ک +##گ +##ں +##ھ +##ہ +##ے +##अ +##आ +##उ +##ए +##क +##ख +##ग +##च +##ज +##ट +##ड +##ण +##त +##थ +##द +##ध +##न +##प +##ब +##भ +##म +##य +##र +##ल +##व +##श +##ष +##स +##ह +##ा +##ि +##ी +##ो +##। +##॥ +##ং +##অ +##আ +##ই +##উ +##এ +##ও +##ক +##খ +##গ +##চ +##ছ +##জ +##ট +##ড +##ণ +##ত +##থ +##দ +##ধ +##ন +##প +##ব +##ভ +##ম +##য +##র +##ল +##শ +##ষ +##স +##হ +##া +##ি +##ী +##ে +##க +##ச +##ட +##த +##ந +##ன +##ப +##ம +##ய +##ர +##ல +##ள +##வ +##ா +##ி +##ு +##ே +##ை +##ನ +##ರ +##ಾ +##ක +##ය +##ර +##ල +##ව +##ා +##ก +##ง +##ต +##ท +##น +##พ +##ม +##ย +##ร +##ล +##ว +##ส +##อ +##า +##เ +##་ +##། +##ག +##ང +##ད +##ན +##པ +##བ +##མ +##འ +##ར +##ལ +##ས +##မ +##ა +##ბ +##გ +##დ +##ე +##ვ +##თ +##ი +##კ +##ლ +##მ +##ნ +##ო +##რ +##ს +##ტ +##უ +##ᄀ +##ᄂ +##ᄃ +##ᄅ +##ᄆ +##ᄇ +##ᄉ +##ᄊ +##ᄋ +##ᄌ +##ᄎ +##ᄏ +##ᄐ +##ᄑ +##ᄒ +##ᅡ +##ᅢ +##ᅥ +##ᅦ +##ᅧ +##ᅩ +##ᅪ +##ᅭ +##ᅮ +##ᅯ +##ᅲ +##ᅳ +##ᅴ +##ᅵ +##ᆨ +##ᆫ +##ᆯ +##ᆷ +##ᆸ +##ᆼ +##ᴬ +##ᴮ +##ᴰ +##ᴵ +##ᴺ +##ᵀ +##ᵃ +##ᵇ +##ᵈ +##ᵉ +##ᵍ +##ᵏ +##ᵐ +##ᵒ +##ᵖ +##ᵗ +##ᵘ +##ᵣ +##ᵤ +##ᵥ +##ᶜ +##ᶠ +##‐ +##‑ +##‒ +##– +##— +##― +##‖ +##‘ +##’ +##‚ +##“ +##” +##„ +##† +##‡ +##• +##… +##‰ +##′ +##″ +##› +##‿ +##⁄ +##⁰ +##ⁱ +##⁴ +##⁵ +##⁶ +##⁷ +##⁸ +##⁹ +##⁻ +##ⁿ +##₅ +##₆ +##₇ +##₈ +##₉ +##₊ +##₍ +##₎ +##ₐ +##ₑ +##ₒ +##ₓ +##ₕ +##ₖ +##ₗ +##ₘ +##ₚ +##ₛ +##ₜ +##₤ +##₩ +##€ +##₱ +##₹ +##ℓ +##№ +##ℝ +##™ +##⅓ +##⅔ +##← +##↑ +##→ +##↓ +##↔ +##↦ +##⇄ +##⇌ +##⇒ +##∂ +##∅ +##∆ +##∇ +##∈ +##∗ +##∘ +##√ +##∞ +##∧ +##∨ +##∩ +##∪ +##≈ +##≡ +##≤ +##≥ +##⊂ +##⊆ +##⊕ +##⊗ +##⋅ +##─ +##│ +##■ +##▪ +##● +##★ +##☆ +##☉ +##♠ +##♣ +##♥ +##♦ +##♯ +##⟨ +##⟩ +##ⱼ +##⺩ +##⺼ +##⽥ +##、 +##。 +##〈 +##〉 +##《 +##》 +##「 +##」 +##『 +##』 +##〜 +##あ +##い +##う +##え +##お +##か +##き +##く +##け +##こ +##さ +##し +##す +##せ +##そ +##た +##ち +##っ +##つ +##て +##と +##な +##に +##ぬ +##ね +##の +##は +##ひ +##ふ +##へ +##ほ +##ま +##み +##む +##め +##も +##や +##ゆ +##よ +##ら +##り +##る +##れ +##ろ +##を +##ん +##ァ +##ア +##ィ +##イ +##ウ +##ェ +##エ +##オ +##カ +##キ +##ク +##ケ +##コ +##サ +##シ +##ス +##セ +##タ +##チ +##ッ +##ツ +##テ +##ト +##ナ +##ニ +##ノ +##ハ +##ヒ +##フ +##ヘ +##ホ +##マ +##ミ +##ム +##メ +##モ +##ャ +##ュ +##ョ +##ラ +##リ +##ル +##レ +##ロ +##ワ +##ン +##・ +##ー +##一 +##三 +##上 +##下 +##不 +##世 +##中 +##主 +##久 +##之 +##也 +##事 +##二 +##五 +##井 +##京 +##人 +##亻 +##仁 +##介 +##代 +##仮 +##伊 +##会 +##佐 +##侍 +##保 +##信 +##健 +##元 +##光 +##八 +##公 +##内 +##出 +##分 +##前 +##劉 +##力 +##加 +##勝 +##北 +##区 +##十 +##千 +##南 +##博 +##原 +##口 +##古 +##史 +##司 +##合 +##吉 +##同 +##名 +##和 +##囗 +##四 +##国 +##國 +##土 +##地 +##坂 +##城 +##堂 +##場 +##士 +##夏 +##外 +##大 +##天 +##太 +##夫 +##奈 +##女 +##子 +##学 +##宀 +##宇 +##安 +##宗 +##定 +##宣 +##宮 +##家 +##宿 +##寺 +##將 +##小 +##尚 +##山 +##岡 +##島 +##崎 +##川 +##州 +##巿 +##帝 +##平 +##年 +##幸 +##广 +##弘 +##張 +##彳 +##後 +##御 +##德 +##心 +##忄 +##志 +##忠 +##愛 +##成 +##我 +##戦 +##戸 +##手 +##扌 +##政 +##文 +##新 +##方 +##日 +##明 +##星 +##春 +##昭 +##智 +##曲 +##書 +##月 +##有 +##朝 +##木 +##本 +##李 +##村 +##東 +##松 +##林 +##森 +##楊 +##樹 +##橋 +##歌 +##止 +##正 +##武 +##比 +##氏 +##民 +##水 +##氵 +##氷 +##永 +##江 +##沢 +##河 +##治 +##法 +##海 +##清 +##漢 +##瀬 +##火 +##版 +##犬 +##王 +##生 +##田 +##男 +##疒 +##発 +##白 +##的 +##皇 +##目 +##相 +##省 +##真 +##石 +##示 +##社 +##神 +##福 +##禾 +##秀 +##秋 +##空 +##立 +##章 +##竹 +##糹 +##美 +##義 +##耳 +##良 +##艹 +##花 +##英 +##華 +##葉 +##藤 +##行 +##街 +##西 +##見 +##訁 +##語 +##谷 +##貝 +##貴 +##車 +##軍 +##辶 +##道 +##郎 +##郡 +##部 +##都 +##里 +##野 +##金 +##鈴 +##镇 +##長 +##門 +##間 +##阝 +##阿 +##陳 +##陽 +##雄 +##青 +##面 +##風 +##食 +##香 +##馬 +##高 +##龍 +##龸 +##fi +##fl +##！ +##（ +##） +##， +##－ +##． +##／ +##： +##？ +##～ diff --git a/linguistics/pom.xml b/linguistics/pom.xml index a09f2ecb031..d9ab942a0b8 100644 --- a/linguistics/pom.xml +++ b/linguistics/pom.xml @@ -22,6 +22,7 @@ <dependency> <groupId>org.mockito</groupId> <artifactId>mockito-core</artifactId> + <scope>test</scope> </dependency> <dependency> <groupId>com.yahoo.vespa</groupId> @@ -61,10 +62,6 @@ <groupId>org.apache.opennlp</groupId> <artifactId>opennlp-tools</artifactId> </dependency> - <dependency> - <groupId>com.optimaize.languagedetector</groupId> - <artifactId>language-detector</artifactId> - </dependency> </dependencies> <build> <plugins> diff --git a/linguistics/src/main/java/com/yahoo/language/opennlp/DefaultLanguageDetectorContextGenerator.java b/linguistics/src/main/java/com/yahoo/language/opennlp/DefaultLanguageDetectorContextGenerator.java new file mode 100644 index 00000000000..27c23d8d3e6 --- /dev/null +++ b/linguistics/src/main/java/com/yahoo/language/opennlp/DefaultLanguageDetectorContextGenerator.java @@ -0,0 +1,32 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.language.opennlp; + +import opennlp.tools.ngram.NGramCharModel; +import opennlp.tools.util.normalizer.CharSequenceNormalizer; + +import java.util.HashSet; +import java.util.Set; + +/** + * Avoids using the unnecessarily slow {@link NGramCharModel}. + * + * @author jonmv + */ +public class DefaultLanguageDetectorContextGenerator extends opennlp.tools.langdetect.DefaultLanguageDetectorContextGenerator { + + public DefaultLanguageDetectorContextGenerator(int minLength, int maxLength, CharSequenceNormalizer... normalizers) { + super(minLength, maxLength, normalizers); + } + + @Override + public String[] getContext(CharSequence document) { + int[] normalized = normalizer.normalize(document).codePoints().map(Character::toLowerCase).toArray(); + Set<String> grams = new HashSet<>(); + for (int i = 0; i < normalized.length; i++) + for (int j = minLength; j <= maxLength && i + j < normalized.length; j++) + grams.add(new String(normalized, i, j)); + + return grams.toArray(new String[grams.size()]); + } + +} diff --git a/linguistics/src/main/java/com/yahoo/language/opennlp/LanguageDetectorFactory.java b/linguistics/src/main/java/com/yahoo/language/opennlp/LanguageDetectorFactory.java new file mode 100644 index 00000000000..fdca5355008 --- /dev/null +++ b/linguistics/src/main/java/com/yahoo/language/opennlp/LanguageDetectorFactory.java @@ -0,0 +1,28 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.language.opennlp; + +import opennlp.tools.langdetect.LanguageDetectorContextGenerator; +import opennlp.tools.util.normalizer.EmojiCharSequenceNormalizer; +import opennlp.tools.util.normalizer.NumberCharSequenceNormalizer; +import opennlp.tools.util.normalizer.ShrinkCharSequenceNormalizer; +import opennlp.tools.util.normalizer.TwitterCharSequenceNormalizer; + +/** + * Overrides the UrlCharSequenceNormalizer, which has a bad regex, until fixed: https://issues.apache.org/jira/browse/OPENNLP-1350 + * + * @author jonmv + */ +@SuppressWarnings("unused") // Loaded by black magic. +public class LanguageDetectorFactory extends opennlp.tools.langdetect.LanguageDetectorFactory { + + @Override + public LanguageDetectorContextGenerator getContextGenerator() { + return new DefaultLanguageDetectorContextGenerator(1, 3, + EmojiCharSequenceNormalizer.getInstance(), + UrlCharSequenceNormalizer.getInstance(), + TwitterCharSequenceNormalizer.getInstance(), + NumberCharSequenceNormalizer.getInstance(), + ShrinkCharSequenceNormalizer.getInstance()); + } + +} diff --git a/linguistics/src/main/java/com/yahoo/language/opennlp/OpenNlpDetector.java b/linguistics/src/main/java/com/yahoo/language/opennlp/OpenNlpDetector.java new file mode 100644 index 00000000000..d7a7d3a4744 --- /dev/null +++ b/linguistics/src/main/java/com/yahoo/language/opennlp/OpenNlpDetector.java @@ -0,0 +1,92 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.language.opennlp; + +import com.yahoo.language.Language; +import com.yahoo.language.detect.Detection; +import com.yahoo.language.detect.Detector; +import com.yahoo.language.detect.Hint; +import com.yahoo.language.simple.SimpleDetector; +import opennlp.tools.langdetect.LanguageDetectorConfig; +import opennlp.tools.langdetect.LanguageDetectorME; +import opennlp.tools.langdetect.LanguageDetectorModel; + +import java.io.IOException; +import java.io.UncheckedIOException; +import java.nio.ByteBuffer; +import java.nio.charset.Charset; +import java.util.HashMap; +import java.util.Locale; +import java.util.Map; + +import static java.nio.charset.StandardCharsets.UTF_8; + +/** + * Detects text language using patched OpenNLP, with fallback to {@link SimpleDetector} for undetected CJK input. + * + * @author jonmv + */ +class OpenNlpDetector implements Detector { + + private static final Object monitor = new Object(); + private static LanguageDetectorModel model; + + private final SimpleDetector simple = new SimpleDetector(); + private final Map<String, Language> languagesByISO3 = new HashMap<>(); + private final LanguageDetectorME detector; + private final LanguageDetectorConfig config; + + OpenNlpDetector() { + detector = new LanguageDetectorME(loadModel()); + config = new LanguageDetectorConfig(); + config.setMinDiff(0.02); + config.setChunkSize(32); + config.setMaxLength(256); + for (Locale locale : Locale.getAvailableLocales()) { + Language language = Language.fromLocale(locale); + if (language != null) + languagesByISO3.put(locale.getISO3Language(), language); + } + } + + private static LanguageDetectorModel loadModel() { + synchronized (monitor) { + if (model == null) { + try { + model = new LanguageDetectorModel(OpenNlpDetector.class.getResourceAsStream("/models/langdetect-183.bin")); + } + catch (IOException e) { + throw new UncheckedIOException(e); + } + } + } + return model; + } + + @Override + public Detection detect(byte[] input, int offset, int length, Hint hint) { + Charset encoding = Charset.forName(simple.guessEncoding(input, offset, length)); + return new Detection(detectLanguage(new String(input, offset, length, encoding)), encoding.name(), false); + } + + @Override + public Detection detect(ByteBuffer input, Hint hint) { + if (input.hasArray()) + return detect(input.array(), input.arrayOffset() + input.position(), input.remaining(), hint); + + byte[] buffer = new byte[input.remaining()]; + input.get(buffer); + return detect(buffer, 0, buffer.length, hint); + } + + @Override + public Detection detect(String input, Hint hint) { + return new Detection(detectLanguage(input), UTF_8.name(), false); + } + + private Language detectLanguage(String input) { + var prediction = detector.probingPredictLanguages(input, config).getLanguages()[0]; + var result = prediction.getConfidence() > 0.02 ? languagesByISO3.get(prediction.getLang()) : null; + return result != null ? result : simple.guessLanguage(input.substring(0, Math.min(input.length(), 256))); + } + +} diff --git a/linguistics/src/main/java/com/yahoo/language/opennlp/OpenNlpLinguistics.java b/linguistics/src/main/java/com/yahoo/language/opennlp/OpenNlpLinguistics.java index a27e726cda8..c749679024a 100644 --- a/linguistics/src/main/java/com/yahoo/language/opennlp/OpenNlpLinguistics.java +++ b/linguistics/src/main/java/com/yahoo/language/opennlp/OpenNlpLinguistics.java @@ -7,36 +7,21 @@ import com.yahoo.language.detect.Detector; import com.yahoo.language.process.Tokenizer; import com.yahoo.language.simple.SimpleDetector; import com.yahoo.language.simple.SimpleLinguistics; -import java.util.logging.Logger; -import java.util.logging.Level; +import opennlp.tools.langdetect.LanguageDetectorModel; /** - * Returns a linguistics implementation based on OpenNlp, - * and (optionally, default on) Optimaize for language detection. + * Returns a linguistics implementation based on OpenNlp. * * @author bratseth + * @author jonmv */ public class OpenNlpLinguistics extends SimpleLinguistics { - private static final Logger log = Logger.getLogger(OpenNlpLinguistics.class.getName()); private final Detector detector; - public OpenNlpLinguistics() { - this(true); - } - @Inject - public OpenNlpLinguistics(OpennlpLinguisticsConfig config) { - this(config.detector().enableOptimaize()); - } - - public OpenNlpLinguistics(boolean enableOptimaize) { - this(enableOptimaize ? new OptimaizeDetector() : new SimpleDetector()); - log.log(Level.FINE, "using "+(enableOptimaize ? "Optimaize" : "Simple")+" detector"); - } - - private OpenNlpLinguistics(Detector detector) { - this.detector = detector; + public OpenNlpLinguistics() { + this.detector = new OpenNlpDetector(); } @Override diff --git a/linguistics/src/main/java/com/yahoo/language/opennlp/OptimaizeDetector.java b/linguistics/src/main/java/com/yahoo/language/opennlp/OptimaizeDetector.java deleted file mode 100644 index 83947c795fb..00000000000 --- a/linguistics/src/main/java/com/yahoo/language/opennlp/OptimaizeDetector.java +++ /dev/null @@ -1,107 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.language.opennlp; - -import com.google.common.base.Optional; -import com.optimaize.langdetect.LanguageDetector; -import com.optimaize.langdetect.LanguageDetectorBuilder; -import com.optimaize.langdetect.i18n.LdLocale; -import com.optimaize.langdetect.ngram.NgramExtractors; -import com.optimaize.langdetect.profiles.LanguageProfile; -import com.optimaize.langdetect.profiles.LanguageProfileReader; -import com.optimaize.langdetect.text.CommonTextObjectFactories; -import com.optimaize.langdetect.text.TextObjectFactory; -import com.yahoo.language.Language; -import com.yahoo.language.detect.Detection; -import com.yahoo.language.detect.Detector; -import com.yahoo.language.detect.Hint; -import com.yahoo.language.simple.SimpleDetector; -import com.yahoo.text.Utf8; - -import java.io.IOException; -import java.io.UncheckedIOException; -import java.nio.ByteBuffer; -import java.util.List; -import java.util.Locale; -import java.util.logging.Logger; -import java.util.logging.Level; - -/** - * Detects the language of some sample text using SimpleDetector for CJK and Optimaize otherwise. - * - * @author bratseth - */ -public class OptimaizeDetector implements Detector { - - private static final Object initGuard = new Object(); - private static TextObjectFactory textObjectFactory = null; - private static LanguageDetector languageDetector = null; - private static final Logger log = Logger.getLogger(OptimaizeDetector.class.getName()); - - static private void initOptimaize() { - synchronized (initGuard) { - if ((textObjectFactory != null) && (languageDetector != null)) return; - - // origin: https://github.com/optimaize/language-detector - // load all languages: - List<LanguageProfile> languageProfiles; - try { - languageProfiles = new LanguageProfileReader().readAllBuiltIn(); - } catch (IOException e) { - throw new UncheckedIOException(e); - } - - //build language detector: - languageDetector = LanguageDetectorBuilder.create(NgramExtractors.standard()) - .withProfiles(languageProfiles) - .build(); - - //create a text object factory - textObjectFactory = CommonTextObjectFactories.forDetectingOnLargeText(); - } - } - - private final SimpleDetector simpleDetector = new SimpleDetector(); - - public OptimaizeDetector() { - initOptimaize(); - } - - @Override - public Detection detect(byte[] input, int offset, int length, Hint hint) { - return new Detection(guessLanguage(input, offset, length), simpleDetector.guessEncoding(input), false); - } - - @Override - public Detection detect(ByteBuffer input, Hint hint) { - byte[] buf = new byte[input.remaining()]; - input.get(buf, 0, buf.length); - return detect(buf, 0, buf.length, hint); - } - - @Override - public Detection detect(String input, Hint hint) { - return new Detection(guessLanguage(input), Utf8.getCharset().name(), false); - } - - private Language guessLanguage(byte[] buf, int offset, int length) { - return guessLanguage(Utf8.toString(buf, offset, length)); - } - - public Language guessLanguage(String input) { - if (input == null || input.length() == 0) return Language.UNKNOWN; - - Language result = simpleDetector.guessLanguage(input); - if (result != Language.UNKNOWN) return result; - - return guessLanguageUsingOptimaize(input); - } - - private static Language guessLanguageUsingOptimaize(String input) { - Optional<LdLocale> result = languageDetector.detect(textObjectFactory.forText(input)); - if ( ! result.isPresent()) return Language.UNKNOWN; - log.log(Level.FINE, () -> "guessing language "+result.get()+" from input: "+input); - - return Language.fromLocale(new Locale(result.get().getLanguage())); - } - -} diff --git a/linguistics/src/main/java/com/yahoo/language/opennlp/UrlCharSequenceNormalizer.java b/linguistics/src/main/java/com/yahoo/language/opennlp/UrlCharSequenceNormalizer.java new file mode 100644 index 00000000000..883319e2f8b --- /dev/null +++ b/linguistics/src/main/java/com/yahoo/language/opennlp/UrlCharSequenceNormalizer.java @@ -0,0 +1,31 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.language.opennlp; + +import opennlp.tools.util.normalizer.CharSequenceNormalizer; + +import java.util.regex.Pattern; + +/** + * Modifies {@link opennlp.tools.util.normalizer.UrlCharSequenceNormalizer} to avoid the bad email regex. + * + * @author jonmv + */ +public class UrlCharSequenceNormalizer implements CharSequenceNormalizer { + + private static final Pattern URL_REGEX = + Pattern.compile("https?://[-_.?&~;+=/#0-9A-Za-z]+"); + private static final Pattern MAIL_REGEX = + Pattern.compile("(?<![-+_.0-9A-Za-z])[-+_.0-9A-Za-z]+@[-0-9A-Za-z]+[-.0-9A-Za-z]+"); + + private static final UrlCharSequenceNormalizer INSTANCE = new UrlCharSequenceNormalizer(); + + public static UrlCharSequenceNormalizer getInstance() { + return INSTANCE; + } + + public CharSequence normalize(CharSequence text) { + String modified = URL_REGEX.matcher(text).replaceAll(" "); + return MAIL_REGEX.matcher(modified).replaceAll(" "); + } + +} diff --git a/linguistics/src/main/java/com/yahoo/language/process/Embedder.java b/linguistics/src/main/java/com/yahoo/language/process/Embedder.java index 17ee0419cea..dd9c3847314 100644 --- a/linguistics/src/main/java/com/yahoo/language/process/Embedder.java +++ b/linguistics/src/main/java/com/yahoo/language/process/Embedder.java @@ -3,6 +3,7 @@ package com.yahoo.language.process; import com.yahoo.language.Language; import com.yahoo.tensor.Tensor; +import com.yahoo.tensor.TensorAddress; import com.yahoo.tensor.TensorType; import java.util.List; @@ -34,7 +35,7 @@ public interface Embedder { * @param text the text to embed * @param context the context which may influence an embedder's behavior * @param tensorType the type of the tensor to be returned - * @return the tensor embedding of the text, as the spoecified tensor type + * @return the tensor embedding of the text, as the specified tensor type * @throws IllegalArgumentException if the language or tensor type is not supported by this embedder */ Tensor embed(String text, Context context, TensorType tensorType); @@ -69,7 +70,7 @@ public interface Embedder { /** * Sets the name of the recipient of this tensor. * - * This iseither a query feature name + * This is either a query feature name * ("query(feature)"), or a schema and field name concatenated by a dot ("schema.field"). */ public Context setDestination(String destination) { diff --git a/linguistics/src/main/java/com/yahoo/language/simple/SimpleDetector.java b/linguistics/src/main/java/com/yahoo/language/simple/SimpleDetector.java index 53b8ad7ad70..61d446cd8d0 100644 --- a/linguistics/src/main/java/com/yahoo/language/simple/SimpleDetector.java +++ b/linguistics/src/main/java/com/yahoo/language/simple/SimpleDetector.java @@ -130,10 +130,14 @@ public class SimpleDetector implements Detector { } public String guessEncoding(byte[] input) { + return guessEncoding(input, 0, input.length); + } + + public String guessEncoding(byte[] input, int offset, int length) { boolean isUtf8 = true; boolean hasHighs = false; scan: - for (int i = 0; i < input.length; i++) { + for (int i = offset; i < offset + length; i++) { final int l = isLeadingFor(input[i]); if (l < 0 || i + l >= input.length) { hasHighs = true; diff --git a/linguistics/src/main/java/com/yahoo/language/simple/SimpleLinguistics.java b/linguistics/src/main/java/com/yahoo/language/simple/SimpleLinguistics.java index 3ca46dcc4f1..b10beb8c9af 100644 --- a/linguistics/src/main/java/com/yahoo/language/simple/SimpleLinguistics.java +++ b/linguistics/src/main/java/com/yahoo/language/simple/SimpleLinguistics.java @@ -2,8 +2,7 @@ package com.yahoo.language.simple; import com.google.inject.Inject; -import com.yahoo.collections.Tuple2; -import com.yahoo.component.Version; +import com.yahoo.component.AbstractComponent; import com.yahoo.language.Linguistics; import com.yahoo.language.detect.Detector; import com.yahoo.language.process.CharacterClasses; @@ -16,7 +15,6 @@ import com.yahoo.language.process.Stemmer; import com.yahoo.language.process.StemmerImpl; import com.yahoo.language.process.Tokenizer; import com.yahoo.language.process.Transformer; -import com.yahoo.vespa.configdefinition.SpecialtokensConfig; import java.util.List; diff --git a/linguistics/src/main/resources/configdefinitions/language.opennlp.opennlp-linguistics.def b/linguistics/src/main/resources/configdefinitions/language.opennlp.opennlp-linguistics.def deleted file mode 100644 index 361a8a5f50c..00000000000 --- a/linguistics/src/main/resources/configdefinitions/language.opennlp.opennlp-linguistics.def +++ /dev/null @@ -1,6 +0,0 @@ -# Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -namespace=language.opennlp - -# Enable Optimaize language detector -detector.enableOptimaize bool default=true - diff --git a/linguistics/src/main/resources/models/langdetect-183.bin b/linguistics/src/main/resources/models/langdetect-183.bin Binary files differnew file mode 100644 index 00000000000..c3cde217050 --- /dev/null +++ b/linguistics/src/main/resources/models/langdetect-183.bin diff --git a/linguistics/src/test/java/com/yahoo/language/opennlp/OpenNlpDetectorTestCase.java b/linguistics/src/test/java/com/yahoo/language/opennlp/OpenNlpDetectorTestCase.java new file mode 100644 index 00000000000..746ed10da1c --- /dev/null +++ b/linguistics/src/test/java/com/yahoo/language/opennlp/OpenNlpDetectorTestCase.java @@ -0,0 +1,87 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.language.opennlp; + +import com.yahoo.language.Language; +import com.yahoo.language.detect.Detector; +import org.junit.Test; + +import static org.junit.Assert.assertEquals; + +/** + * @author jonmv + */ +public class OpenNlpDetectorTestCase { + + @Test + public void testDetection() { + Detector detector = new OpenNlpDetector(); + + assertLanguage(Language.UNKNOWN, + "", + detector); + + assertLanguage(Language.UNKNOWN, + "Hello!", + detector); + + // from https://en.wikipedia.org/wiki/Yahoo + assertLanguage(Language.ENGLISH, + "Yahoo became a public company via an initial public offering in April 1996 and its stock price rose 600% within two years.", + detector); + + // from https://de.wikipedia.org/wiki/Yahoo + assertLanguage(Language.GERMAN, + "1996 ging Yahoo mit 46 Angestellten an die Börse. 2009 arbeiteten insgesamt rund 13.500 Mitarbeiter für Yahoo.", + detector); + + // from https://fr.wikipedia.org/wiki/Yahoo + assertLanguage(Language.FRENCH, + "À l'origine, Yahoo! était uniquement un annuaire Web.", + detector); + + // Test fallback to SimpleDetector + assertLanguage(Language.CHINESE_TRADITIONAL, // CHINESE_SIMPLIFIED input + "我能吞下玻璃而不伤身体。", + detector); + + // from https://zh.wikipedia.org/wiki/Yahoo + assertLanguage(Language.CHINESE_TRADITIONAL, + "Yahoo! Next是一个展示雅虎新技术、新产品的场所，目前在测试阶段。", + detector); + + // from https://th.wikipedia.org/wiki/Yahoo + assertLanguage(Language.THAI, + "เดือนกรกฎาคม 2012 Yahoo! ก็ได้ประธานเจ้าหน้าที่บริหารคนใหม่ \"มาริสสา เมเยอร์\" อดีตผู้บริหารจาก Google มาทำหน้าที่พลิกฟื้นบริษัท", + detector); + + // from https://ar.wikipedia.org/wiki/Yahoo + assertLanguage(Language.ARABIC, + "وفقًا لمزودي تحليلات الويب دائما كأليكسا وسميلارويب،وصل موقع ياهولأكثر من 7 مليارات مشاهدة شهريًا - حيث احتل المرتبة السادسة بين أكثر مواقع الويب زيارة على مستوى العالم في عام 2016.", + detector); + + // from https://ko.wikipedia.org/wiki/Yahoo + assertLanguage(Language.KOREAN, + "야후!의 전신인 디렉터리 사이트는 1994년 1월에 스탠퍼드 대학교 출신의 제리 양과 데이비드 파일로가 만들었으며, 회사는 1995년 3월 2일에 설립되었다.", + detector); + + // from https://ja.wikipedia.org/wiki/Yahoo + assertLanguage(Language.JAPANESE, + "日本では、ヤフー株式会社がYahoo!（後にベライゾンがアルタバに売却）とソフトバンクの合弁会社として1996年に設立した。", + detector); + + // from https://ru.wikipedia.org/wiki/Yahoo + assertLanguage(Language.RUSSIAN, + "7 февраля 2000 года Yahoo.com подвергся DDoS атаке и на несколько часов приостановил работу.", + detector); + + // from https://he.wikipedia.org/wiki/Yahoo + assertLanguage(Language.HEBREW, + "אתר יאהו! הוא אחד מאתרי האינטרנט הפופולריים ביותר בעולם, עם מעל 500 מיליון כניסות בכל יום", + detector); + } + + private void assertLanguage(Language language, String input, Detector detector) { + assertEquals(language, detector.detect(input, null).getLanguage()); + } + +} diff --git a/linguistics/src/test/java/com/yahoo/language/opennlp/OptimaizeDetectorTestCase.java b/linguistics/src/test/java/com/yahoo/language/opennlp/OptimaizeDetectorTestCase.java deleted file mode 100644 index 20b5de3b165..00000000000 --- a/linguistics/src/test/java/com/yahoo/language/opennlp/OptimaizeDetectorTestCase.java +++ /dev/null @@ -1,35 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.language.opennlp; - -import com.yahoo.language.Language; -import com.yahoo.language.detect.Detector; -import org.junit.Test; - -import static org.junit.Assert.assertEquals; - -/** - * @author bratseth - */ -public class OptimaizeDetectorTestCase { - - private static final Detector detector = new OptimaizeDetector(); - - @Test - public void testDetection() { - assertLanguage(Language.UNKNOWN, "Hello!"); - - // Test fallback to SimpleDetector - assertLanguage(Language.CHINESE_TRADITIONAL, // CHINESE_SIMPLIFIED input - "\u6211\u80FD\u541E\u4E0B\u73BB\u7483\u800C\u4E0D\u4F24\u8EAB\u4F53\u3002"); - - // from https://ru.wikipedia.org/wiki/%D0%A0%D0%BE%D1%81%D1%81%D0%B8%D1%8F - assertLanguage(Language.RUSSIAN, "Материал из Википедии — свободной энциклопедии"); - // https://he.wikipedia.org/wiki/Yahoo! - assertLanguage(Language.HEBREW, "אתר יאהו! הוא אחד מאתרי האינטרנט הפופולריים ביותר בעולם, עם מעל 500 מיליון כניסות בכל יום"); - } - - private static void assertLanguage(Language language, String input) { - assertEquals(language, detector.detect(input, null).getLanguage()); - } - -} diff --git a/linguistics/src/test/java/com/yahoo/language/opennlp/UrlCharSequenceNormalizerTest.java b/linguistics/src/test/java/com/yahoo/language/opennlp/UrlCharSequenceNormalizerTest.java new file mode 100644 index 00000000000..a8c637bc6ec --- /dev/null +++ b/linguistics/src/test/java/com/yahoo/language/opennlp/UrlCharSequenceNormalizerTest.java @@ -0,0 +1,20 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.language.opennlp; + +import org.junit.Test; + +import static org.junit.Assert.assertEquals; + +/** + * @author jonmv + */ +public class UrlCharSequenceNormalizerTest { + + @Test + public void testNormalization() { + String text = "xxx+yyy_.dude@mail.com foo bar@baz_bax https://host.tld/path?query=boo a@b §boo@boo"; + assertEquals(" foo _bax a@b § ", + UrlCharSequenceNormalizer.getInstance().normalize(text)); + } + +} diff --git a/logd/src/logd/watcher.cpp b/logd/src/logd/watcher.cpp index 23d1580dbf4..c3752fcc3e8 100644 --- a/logd/src/logd/watcher.cpp +++ b/logd/src/logd/watcher.cpp @@ -366,7 +366,7 @@ Watcher::removeOldLogs(const char *prefix) } } } else if (globresult == GLOB_NOMATCH) { - LOG(info, "no old logfiles matching %s", pattern); + LOG(debug, "no old logfiles matching %s", pattern); } else { LOG(warning, "glob %s failed: %d", pattern, globresult); } diff --git a/logforwarder/src/apps/vespa-logforwarder-start/cf-handler.cpp b/logforwarder/src/apps/vespa-logforwarder-start/cf-handler.cpp index df4c6ee7f57..c76012d146a 100644 --- a/logforwarder/src/apps/vespa-logforwarder-start/cf-handler.cpp +++ b/logforwarder/src/apps/vespa-logforwarder-start/cf-handler.cpp @@ -84,7 +84,7 @@ CfHandler::doConfigure() getenv("VESPA_INSTANCE") != NULL && getenv("VESPA_ENVIRONMENT") != NULL && getenv("VESPA_REGION") != NULL) - { + { path = cfFilePath(config.splunkHome, "inputs.conf"); tmpPath = path + ".new"; fp = fopen(tmpPath.c_str(), "w"); @@ -99,9 +99,9 @@ CfHandler::doConfigure() if (config.clientName.size() == 0 || config.deploymentServer.size() == 0) { - childHandler.stopChild(config.splunkHome); + _childHandler.stopChild(); } else { - childHandler.startChild(config.splunkHome); + _childHandler.startChild(config.splunkHome); } } @@ -113,6 +113,10 @@ CfHandler::check() } } +void CfHandler::stop() { + _childHandler.stopChild(); +} + constexpr std::chrono::milliseconds CONFIG_TIMEOUT_MS(30 * 1000); void diff --git a/logforwarder/src/apps/vespa-logforwarder-start/cf-handler.h b/logforwarder/src/apps/vespa-logforwarder-start/cf-handler.h index f5bd4866930..8e7196fb034 100644 --- a/logforwarder/src/apps/vespa-logforwarder-start/cf-handler.h +++ b/logforwarder/src/apps/vespa-logforwarder-start/cf-handler.h @@ -9,7 +9,7 @@ using cloud::config::LogforwarderConfig; class CfHandler { private: - ChildHandler childHandler; + ChildHandler _childHandler; config::ConfigSubscriber _subscriber; config::ConfigHandle<LogforwarderConfig>::UP _handle; void subscribe(const std::string & configId, std::chrono::milliseconds timeout); @@ -18,5 +18,6 @@ public: CfHandler(); virtual ~CfHandler(); void start(const char *configId); + void stop(); void check(); }; diff --git a/logforwarder/src/apps/vespa-logforwarder-start/child-handler.cpp b/logforwarder/src/apps/vespa-logforwarder-start/child-handler.cpp index 810491404dd..a478e0bbd01 100644 --- a/logforwarder/src/apps/vespa-logforwarder-start/child-handler.cpp +++ b/logforwarder/src/apps/vespa-logforwarder-start/child-handler.cpp @@ -76,21 +76,26 @@ runSplunk(const vespalib::string &prefix, std::vector<const char *> args) void ChildHandler::startChild(const vespalib::string &prefix) { - if (! _childRunning) { - // it is possible that splunk was already running anyway, so - // make sure we restart it to get new config activated: - runSplunk(prefix, {"stop"}); + if (_childRunning && prefix == _runningPrefix) { + runSplunk(prefix, {"restart"}); + } else { + if (_childRunning) { + runSplunk(_runningPrefix, {"stop"}); + } else { + // it is possible that splunk was already running anyway, so + // make sure we restart it to get new config activated: + runSplunk(prefix, {"stop"}); + } sleep(1); runSplunk(prefix, {"start", "--answer-yes", "--no-prompt", "--accept-license"}); _childRunning = true; - } else { - runSplunk(prefix, {"restart"}); + _runningPrefix = prefix; } } void -ChildHandler::stopChild(const vespalib::string &prefix) +ChildHandler::stopChild() { - runSplunk(prefix, {"stop"}); + runSplunk(_runningPrefix, {"stop"}); _childRunning = false; } diff --git a/logforwarder/src/apps/vespa-logforwarder-start/child-handler.h b/logforwarder/src/apps/vespa-logforwarder-start/child-handler.h index 2eb50ee8944..22396a0e448 100644 --- a/logforwarder/src/apps/vespa-logforwarder-start/child-handler.h +++ b/logforwarder/src/apps/vespa-logforwarder-start/child-handler.h @@ -6,8 +6,9 @@ class ChildHandler { private: bool _childRunning; + vespalib::string _runningPrefix; public: void startChild(const vespalib::string &prefix); - void stopChild(const vespalib::string &prefix); + void stopChild(); ChildHandler(); }; diff --git a/logforwarder/src/apps/vespa-logforwarder-start/main.cpp b/logforwarder/src/apps/vespa-logforwarder-start/main.cpp index c6bb8f8946d..33fe4489811 100644 --- a/logforwarder/src/apps/vespa-logforwarder-start/main.cpp +++ b/logforwarder/src/apps/vespa-logforwarder-start/main.cpp @@ -21,6 +21,7 @@ public: handler.check(); usleep(12500); // Avoid busy looping; } + handler.stop(); } }; diff --git a/messagebus/pom.xml b/messagebus/pom.xml index d2ce9cb36d8..efc1997d2e5 100644 --- a/messagebus/pom.xml +++ b/messagebus/pom.xml @@ -53,6 +53,9 @@ <artifactId>maven-surefire-plugin</artifactId> <configuration> <forkCount>4</forkCount> + <systemPropertyVariables> + <java.util.logging.SimpleFormatter.format>%1$Ts.%1$TL [%4$s] %5$s {%3$s}%n</java.util.logging.SimpleFormatter.format> + </systemPropertyVariables> </configuration> </plugin> <plugin> diff --git a/messagebus/src/main/java/com/yahoo/messagebus/network/rpc/RPCNetwork.java b/messagebus/src/main/java/com/yahoo/messagebus/network/rpc/RPCNetwork.java index 7ed1b17a5b4..5ff9a4b0313 100644 --- a/messagebus/src/main/java/com/yahoo/messagebus/network/rpc/RPCNetwork.java +++ b/messagebus/src/main/java/com/yahoo/messagebus/network/rpc/RPCNetwork.java @@ -147,20 +147,27 @@ public class RPCNetwork implements Network, MethodHandler { @Override public boolean waitUntilReady(double seconds) { - for (int i = 0; i < seconds * 100; ++i) { + int millis = (int) seconds * 1000; + int i = 0; + do { if (mirror.ready()) { + if (i > 200) { + log.log(Level.INFO, "network became ready (at "+i+" ms)"); + } return true; } - if ((i % 100) == 0) { - log.log(Level.INFO, "waiting for network to become ready ("+(i/100)+" of "+((int)seconds)+" seconds)"); + if ((i == 200) || ((i > 200) && ((i % 1000) == 0))) { + log.log(Level.INFO, "waiting for network to become ready ("+i+" of "+millis+" ms)"); mirror.dumpState(); } try { + // could maybe have some back-off here, fixed at 10ms for now + i += 10; Thread.sleep(10); } catch (InterruptedException e) { // empty } - } + } while (i < millis); return false; } diff --git a/messagebus/src/main/java/com/yahoo/messagebus/network/rpc/test/TestServer.java b/messagebus/src/main/java/com/yahoo/messagebus/network/rpc/test/TestServer.java index c6dd14cf23b..92f8de93b03 100644 --- a/messagebus/src/main/java/com/yahoo/messagebus/network/rpc/test/TestServer.java +++ b/messagebus/src/main/java/com/yahoo/messagebus/network/rpc/test/TestServer.java @@ -6,7 +6,6 @@ import com.yahoo.component.Vtag; import com.yahoo.jrt.Spec; import com.yahoo.jrt.slobrok.api.Mirror; import com.yahoo.jrt.slobrok.server.Slobrok; -import java.util.logging.Level; import com.yahoo.messagebus.MessageBus; import com.yahoo.messagebus.MessageBusParams; import com.yahoo.messagebus.Protocol; @@ -21,6 +20,7 @@ import com.yahoo.messagebus.test.SimpleProtocol; import java.util.List; import java.util.concurrent.atomic.AtomicBoolean; import java.util.logging.Logger; +import java.util.logging.Level; /** * A simple test server implementation. @@ -29,6 +29,8 @@ import java.util.logging.Logger; */ public class TestServer { + private static Logger log = Logger.getLogger(TestServer.class.getName()); + private final AtomicBoolean destroyed = new AtomicBoolean(false); public final VersionedRPCNetwork net; public final MessageBus mb; @@ -45,6 +47,7 @@ public class TestServer { this(new MessageBusParams().addProtocol(new SimpleProtocol()), new RPCNetworkParams() .setIdentity(new Identity(name)) + .setNumNetworkThreads(1) .setSlobrokConfigId(getSlobrokConfig(slobrok))); if (protocol != null) { mb.putProtocol(protocol); @@ -52,6 +55,16 @@ public class TestServer { if (table != null) { setupRouting(table); } + log.log(Level.INFO, "Running testServer '"+name+"' at "+net.getConnectionSpec()+", location broker at "+slobrok.port()); + } + + /** Creates a new test server. */ + public TestServer(MessageBusParams mbusParams, Slobrok slobrok) { + this(mbusParams, + new RPCNetworkParams() + .setNumNetworkThreads(1) + .setSlobrokConfigId(getSlobrokConfig(slobrok))); + log.log(Level.INFO, "Running testServer <unnamed> at "+net.getConnectionSpec()+", location broker at "+slobrok.port()); } /** Creates a new test server. */ @@ -64,6 +77,7 @@ public class TestServer { public TestServer(MessageBusParams mbusParams) { mb = new MessageBus(new LocalNetwork(), mbusParams); net = null; + log.log(Level.INFO, "Running testServer without network"); } /** @@ -74,6 +88,11 @@ public class TestServer { */ public boolean destroy() { if (!destroyed.getAndSet(true)) { + if (net != null) { + log.log(Level.INFO, "Destroy testServer '"+net.getIdentity().getServicePrefix()+"' at "+net.getConnectionSpec()); + } else { + log.log(Level.INFO, "Destroy testServer without network"); + } mb.destroy(); if (net != null) net.destroy(); @@ -121,7 +140,8 @@ public class TestServer { * @return whether or not the required state was reached */ public boolean waitState(SlobrokState slobrokState) { - for (int i = 0; i < 6000 && !Thread.currentThread().isInterrupted(); ++i) { + int millis = 120 * 1000; + for (int i = 0; i < millis && !Thread.currentThread().isInterrupted(); ++i) { boolean done = true; for (String pattern : slobrokState.getPatterns()) { List<Mirror.Entry> res = net.getMirror().lookup(pattern); @@ -130,10 +150,16 @@ public class TestServer { } } if (done) { + if (i > 50) log.log(Level.INFO, "waitState OK after "+i+" ms"); return true; } + if ((i % 1000) == 50) { + log.log(Level.INFO, "waitState still waiting, "+i+" ms"); + var m = (Mirror) net.getMirror(); + m.dumpState(); + } try { - Thread.sleep(10); + Thread.sleep(1); } catch (InterruptedException e) { // ignore diff --git a/messagebus/src/test/java/com/yahoo/messagebus/network/rpc/SlobrokTestCase.java b/messagebus/src/test/java/com/yahoo/messagebus/network/rpc/SlobrokTestCase.java index 044ee87f4c4..931fed596e5 100644 --- a/messagebus/src/test/java/com/yahoo/messagebus/network/rpc/SlobrokTestCase.java +++ b/messagebus/src/test/java/com/yahoo/messagebus/network/rpc/SlobrokTestCase.java @@ -85,9 +85,9 @@ public class SlobrokTestCase { public void setUp() throws ListenFailedException { slobrok = new Slobrok(); String slobrokCfgId = "raw:slobrok[1]\nslobrok[0].connectionspec \"" + new Spec("localhost", slobrok.port()).toString() + "\"\n"; - net1 = new RPCNetwork(new RPCNetworkParams().setIdentity(new Identity("net/a")).setSlobrokConfigId(slobrokCfgId)); - net2 = new RPCNetwork(new RPCNetworkParams().setIdentity(new Identity("net/b")).setSlobrokConfigId(slobrokCfgId)); - net3 = new RPCNetwork(new RPCNetworkParams().setIdentity(new Identity("net/c")).setSlobrokConfigId(slobrokCfgId)); + net1 = new RPCNetwork(new RPCNetworkParams().setNumNetworkThreads(1).setIdentity(new Identity("net/a")).setSlobrokConfigId(slobrokCfgId)); + net2 = new RPCNetwork(new RPCNetworkParams().setNumNetworkThreads(1).setIdentity(new Identity("net/b")).setSlobrokConfigId(slobrokCfgId)); + net3 = new RPCNetwork(new RPCNetworkParams().setNumNetworkThreads(1).setIdentity(new Identity("net/c")).setSlobrokConfigId(slobrokCfgId)); port1 = net1.getPort(); port2 = net2.getPort(); port3 = net3.getPort(); diff --git a/messagebus/src/test/java/com/yahoo/messagebus/routing/RoutingTestCase.java b/messagebus/src/test/java/com/yahoo/messagebus/routing/RoutingTestCase.java index aec99306ff2..2c897814b8c 100644 --- a/messagebus/src/test/java/com/yahoo/messagebus/routing/RoutingTestCase.java +++ b/messagebus/src/test/java/com/yahoo/messagebus/routing/RoutingTestCase.java @@ -24,13 +24,17 @@ import com.yahoo.messagebus.test.Receptor; import com.yahoo.messagebus.test.SimpleMessage; import com.yahoo.messagebus.test.SimpleProtocol; import org.junit.After; +import org.junit.AfterClass; import org.junit.Before; +import org.junit.BeforeClass; import org.junit.Test; import java.net.UnknownHostException; import java.util.ArrayList; import java.util.Arrays; import java.util.List; +import java.util.logging.Logger; +import java.util.logging.Level; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; @@ -44,60 +48,86 @@ import static org.junit.Assert.assertTrue; */ public class RoutingTestCase { - Slobrok slobrok; - TestServer srcServer, dstServer; + static final Logger log = Logger.getLogger(RoutingTestCase.class.getName()); + + static Slobrok slobrok; + static RetryTransientErrorsPolicy retryPolicy; + static TestServer srcServer, dstServer; + SourceSession srcSession; DestinationSession dstSession; - RetryTransientErrorsPolicy retryPolicy; - @Before - public void setUp() throws ListenFailedException, UnknownHostException { + @BeforeClass + public static void commonSetup() throws ListenFailedException { slobrok = new Slobrok(); - dstServer = new TestServer(new MessageBusParams().addProtocol(new SimpleProtocol()), - new RPCNetworkParams().setIdentity(new Identity("dst")).setSlobrokConfigId( - TestServer.getSlobrokConfig(slobrok))); - dstSession = dstServer.mb.createDestinationSession( - new DestinationSessionParams().setName("session").setMessageHandler(new Receptor())); + dstServer = new TestServer("dst", null, slobrok, new SimpleProtocol()); retryPolicy = new RetryTransientErrorsPolicy(); retryPolicy.setBaseDelay(0); srcServer = new TestServer(new MessageBusParams().setRetryPolicy(retryPolicy).addProtocol(new SimpleProtocol()), - new RPCNetworkParams().setSlobrokConfigId(TestServer.getSlobrokConfig(slobrok))); + slobrok); + } + + @AfterClass + public static void commonTeardown() { + dstServer.destroy(); + srcServer.destroy(); + slobrok.stop(); + dstServer = null; + srcServer = null; + slobrok = null; + } + + @Before + public void setUp() throws ListenFailedException, UnknownHostException { + // reset some params: + retryPolicy.setEnabled(true); + retryPolicy.setBaseDelay(0); + srcServer.mb.putProtocol(new SimpleProtocol()); + srcServer.setupRouting(new RoutingTableSpec(SimpleProtocol.NAME)); + // create sessions: + dstSession = dstServer.mb.createDestinationSession( + new DestinationSessionParams().setName("session").setMessageHandler(new Receptor())); srcSession = srcServer.mb.createSourceSession( new SourceSessionParams().setTimeout(600.0).setThrottlePolicy(null).setReplyHandler(new Receptor())); + // wait for session register visible: assertTrue(srcServer.waitSlobrok("dst/session", 1)); } @After public void tearDown() { - slobrok.stop(); dstSession.destroy(); - dstServer.destroy(); srcSession.destroy(); - srcServer.destroy(); + // wait for session unregister visible: + assertTrue(srcServer.waitSlobrok("dst/session", 0)); } @Test public void requireThatNullRouteIsCaught() { + log.log(Level.INFO, "Starting: requireThatNullRouteIsCaught"); assertTrue(srcSession.send(createMessage("msg")).isAccepted()); Reply reply = ((Receptor)srcSession.getReplyHandler()).getReply(60); assertNotNull(reply); System.out.println(reply.getTrace()); assertEquals(1, reply.getNumErrors()); assertEquals(ErrorCode.ILLEGAL_ROUTE, reply.getError(0).getCode()); + log.log(Level.INFO, "Finished: requireThatNullRouteIsCaught"); } @Test public void requireThatEmptyRouteIsCaught() { + log.log(Level.INFO, "Starting: requireThatEmptyRouteIsCaught"); assertTrue(srcSession.send(createMessage("msg"), new Route()).isAccepted()); Reply reply = ((Receptor)srcSession.getReplyHandler()).getReply(60); assertNotNull(reply); System.out.println(reply.getTrace()); assertEquals(1, reply.getNumErrors()); assertEquals(ErrorCode.ILLEGAL_ROUTE, reply.getError(0).getCode()); + log.log(Level.INFO, "Finished: requireThatEmptyRouteIsCaught"); } @Test public void requireThatHopNameIsExpanded() { + log.log(Level.INFO, "Starting: requireThatHopNameIsExpanded"); srcServer.setupRouting(new RoutingTableSpec(SimpleProtocol.NAME) .addHop(new HopSpec("dst", "dst/session"))); assertTrue(srcSession.send(createMessage("msg"), Route.parse("dst")).isAccepted()); @@ -108,10 +138,12 @@ public class RoutingTestCase { assertNotNull(reply); System.out.println(reply.getTrace()); assertFalse(reply.hasErrors()); + log.log(Level.INFO, "Finished: requireThatHopNameIsExpanded"); } @Test public void requireThatRouteDirectiveWorks() { + log.log(Level.INFO, "Starting: requireThatRouteDirectiveWorks"); srcServer.setupRouting(new RoutingTableSpec(SimpleProtocol.NAME) .addRoute(new RouteSpec("dst").addHop("dst/session")) .addHop(new HopSpec("dir", "route:dst"))); @@ -123,10 +155,12 @@ public class RoutingTestCase { assertNotNull(reply); System.out.println(reply.getTrace()); assertFalse(reply.hasErrors()); + log.log(Level.INFO, "Finished: requireThatRouteDirectiveWorks"); } @Test public void requireThatRouteNameIsExpanded() { + log.log(Level.INFO, "Starting: requireThatRouteNameIsExpanded"); srcServer.setupRouting(new RoutingTableSpec(SimpleProtocol.NAME) .addRoute(new RouteSpec("dst").addHop("dst/session"))); assertTrue(srcSession.send(createMessage("msg"), Route.parse("dst")).isAccepted()); @@ -137,10 +171,12 @@ public class RoutingTestCase { assertNotNull(reply); System.out.println(reply.getTrace()); assertFalse(reply.hasErrors()); + log.log(Level.INFO, "Finished: requireThatRouteNameIsExpanded"); } @Test public void requireThatHopResolutionOverflowIsCaught() { + log.log(Level.INFO, "Starting: requireThatHopResolutionOverflowIsCaught"); srcServer.setupRouting(new RoutingTableSpec(SimpleProtocol.NAME) .addHop(new HopSpec("foo", "bar")) .addHop(new HopSpec("bar", "foo"))); @@ -150,10 +186,12 @@ public class RoutingTestCase { System.out.println(reply.getTrace()); assertEquals(1, reply.getNumErrors()); assertEquals(ErrorCode.ILLEGAL_ROUTE, reply.getError(0).getCode()); + log.log(Level.INFO, "Finished: requireThatHopResolutionOverflowIsCaught"); } @Test public void requireThatRouteResolutionOverflowIsCaught() { + log.log(Level.INFO, "Starting: requireThatRouteResolutionOverflowIsCaught"); srcServer.setupRouting(new RoutingTableSpec(SimpleProtocol.NAME) .addRoute(new RouteSpec("foo").addHop("route:foo"))); assertTrue(srcSession.send(createMessage("msg"), "foo").isAccepted()); @@ -162,10 +200,12 @@ public class RoutingTestCase { System.out.println(reply.getTrace()); assertEquals(1, reply.getNumErrors()); assertEquals(ErrorCode.ILLEGAL_ROUTE, reply.getError(0).getCode()); + log.log(Level.INFO, "Finished: requireThatRouteResolutionOverflowIsCaught"); } @Test public void requireThatRouteExpansionOnlyReplacesFirstHop() { + log.log(Level.INFO, "Starting: requireThatRouteExpansionOnlyReplacesFirstHop"); srcServer.setupRouting(new RoutingTableSpec(SimpleProtocol.NAME) .addRoute(new RouteSpec("foo").addHop("dst/session").addHop("bar"))); assertTrue(srcSession.send(createMessage("msg"), Route.parse("route:foo baz")).isAccepted()); @@ -179,10 +219,12 @@ public class RoutingTestCase { assertNotNull(reply); System.out.println(reply.getTrace()); assertFalse(reply.hasErrors()); + log.log(Level.INFO, "Finished: requireThatRouteExpansionOnlyReplacesFirstHop"); } @Test public void requireThatErrorDirectiveWorks() { + log.log(Level.INFO, "Starting: requireThatErrorDirectiveWorks"); Route route = Route.parse("foo/bar/baz"); route.getHop(0).setDirective(1, new ErrorDirective("err")); assertTrue(srcSession.send(createMessage("msg"), route).isAccepted()); @@ -192,10 +234,12 @@ public class RoutingTestCase { assertEquals(1, reply.getNumErrors()); assertEquals(ErrorCode.ILLEGAL_ROUTE, reply.getError(0).getCode()); assertEquals("err", reply.getError(0).getMessage()); + log.log(Level.INFO, "Finished: requireThatErrorDirectiveWorks"); } @Test public void requireThatIllegalSelectIsCaught() { + log.log(Level.INFO, "Starting: requireThatIllegalSelectIsCaught"); SimpleProtocol protocol = new SimpleProtocol(); protocol.addPolicyFactory("Custom", new CustomPolicyFactory()); srcServer.mb.putProtocol(protocol); @@ -207,10 +251,12 @@ public class RoutingTestCase { System.out.println(reply.getTrace()); assertEquals(1, reply.getNumErrors()); assertEquals(ErrorCode.NO_SERVICES_FOR_ROUTE, reply.getError(0).getCode()); + log.log(Level.INFO, "Finished: requireThatIllegalSelectIsCaught"); } @Test public void requireThatEmptySelectIsCaught() { + log.log(Level.INFO, "Starting: requireThatEmptySelectIsCaught"); SimpleProtocol protocol = new SimpleProtocol(); protocol.addPolicyFactory("Custom", new CustomPolicyFactory()); srcServer.mb.putProtocol(protocol); @@ -220,10 +266,12 @@ public class RoutingTestCase { System.out.println(reply.getTrace()); assertEquals(1, reply.getNumErrors()); assertEquals(ErrorCode.NO_SERVICES_FOR_ROUTE, reply.getError(0).getCode()); + log.log(Level.INFO, "Finished: requireThatEmptySelectIsCaught"); } @Test public void requireThatPolicySelectWorks() { + log.log(Level.INFO, "Starting: requireThatPolicySelectWorks"); SimpleProtocol protocol = new SimpleProtocol(); protocol.addPolicyFactory("Custom", new CustomPolicyFactory()); srcServer.mb.putProtocol(protocol); @@ -235,10 +283,12 @@ public class RoutingTestCase { assertNotNull(reply); System.out.println(reply.getTrace()); assertFalse(reply.hasErrors()); + log.log(Level.INFO, "Finished: requireThatPolicySelectWorks"); } @Test public void requireThatTransientErrorsAreRetried() { + log.log(Level.INFO, "Starting: requireThatTransientErrorsAreRetried"); assertTrue(srcSession.send(createMessage("msg"), Route.parse("dst/session")).isAccepted()); Message msg = ((Receptor)dstSession.getMessageHandler()).getMessage(60); assertNotNull(msg); @@ -261,10 +311,12 @@ public class RoutingTestCase { "[APP_TRANSIENT_ERROR @ localhost]: err2", "-[APP_TRANSIENT_ERROR @ localhost]: err2"), reply.getTrace()); + log.log(Level.INFO, "Finished: requireThatTransientErrorsAreRetried"); } @Test public void requireThatTransientErrorsAreRetriedWithPolicy() { + log.log(Level.INFO, "Starting: requireThatTransientErrorsAreRetriedWithPolicy"); SimpleProtocol protocol = new SimpleProtocol(); protocol.addPolicyFactory("Custom", new CustomPolicyFactory()); srcServer.mb.putProtocol(protocol); @@ -324,10 +376,12 @@ public class RoutingTestCase { "Merged [dst/session].", "Source session received reply. 0 message(s) now pending."), reply.getTrace()); + log.log(Level.INFO, "Finished: requireThatTransientErrorsAreRetriedWithPolicy"); } @Test public void requireThatRetryCanBeDisabled() { + log.log(Level.INFO, "Starting: requireThatRetryCanBeDisabled"); retryPolicy.setEnabled(false); assertTrue(srcSession.send(createMessage("msg"), Route.parse("dst/session")).isAccepted()); Message msg = ((Receptor)dstSession.getMessageHandler()).getMessage(60); @@ -340,10 +394,12 @@ public class RoutingTestCase { System.out.println(reply.getTrace()); assertEquals(1, reply.getNumErrors()); assertEquals(ErrorCode.APP_TRANSIENT_ERROR, reply.getError(0).getCode()); + log.log(Level.INFO, "Finished: requireThatRetryCanBeDisabled"); } @Test public void requireThatRetryCallsSelect() { + log.log(Level.INFO, "Starting: requireThatRetryCallsSelect"); SimpleProtocol protocol = new SimpleProtocol(); protocol.addPolicyFactory("Custom", new CustomPolicyFactory()); srcServer.mb.putProtocol(protocol); @@ -367,10 +423,12 @@ public class RoutingTestCase { "Sending reply", "Merged [dst/session]."), reply.getTrace()); + log.log(Level.INFO, "Finished: requireThatRetryCallsSelect"); } @Test public void requireThatPolicyCanDisableReselectOnRetry() { + log.log(Level.INFO, "Starting: requireThatPolicyCanDisableReselectOnRetry"); SimpleProtocol protocol = new SimpleProtocol(); protocol.addPolicyFactory("Custom", new CustomPolicyFactory(false)); srcServer.mb.putProtocol(protocol); @@ -394,10 +452,12 @@ public class RoutingTestCase { "Sending reply", "Merged [dst/session]."), reply.getTrace()); + log.log(Level.INFO, "Finished: requireThatPolicyCanDisableReselectOnRetry"); } @Test public void requireThatPolicyCanConsumeErrors() { + log.log(Level.INFO, "Starting: requireThatPolicyCanConsumeErrors"); SimpleProtocol protocol = new SimpleProtocol(); protocol.addPolicyFactory("Custom", new CustomPolicyFactory(true, ErrorCode.NO_ADDRESS_FOR_SERVICE)); srcServer.mb.putProtocol(protocol); @@ -416,10 +476,12 @@ public class RoutingTestCase { "Sending reply", "Merged [dst/session, dst/unknown]."), reply.getTrace()); + log.log(Level.INFO, "Finished: requireThatPolicyCanConsumeErrors"); } @Test public void requireThatPolicyOnlyConsumesDeclaredErrors() { + log.log(Level.INFO, "Starting: requireThatPolicyOnlyConsumesDeclaredErrors"); SimpleProtocol protocol = new SimpleProtocol(); protocol.addPolicyFactory("Custom", new CustomPolicyFactory()); srcServer.mb.putProtocol(protocol); @@ -434,10 +496,12 @@ public class RoutingTestCase { "[NO_ADDRESS_FOR_SERVICE @ localhost]", "Merged [dst/unknown]."), reply.getTrace()); + log.log(Level.INFO, "Finished: requireThatPolicyOnlyConsumesDeclaredErrors"); } @Test public void requireThatPolicyCanExpandToPolicy() { + log.log(Level.INFO, "Starting: requireThatPolicyCanExpandToPolicy"); SimpleProtocol protocol = new SimpleProtocol(); protocol.addPolicyFactory("Custom", new CustomPolicyFactory(true, ErrorCode.NO_ADDRESS_FOR_SERVICE)); srcServer.mb.putProtocol(protocol); @@ -452,10 +516,12 @@ public class RoutingTestCase { System.out.println(reply.getTrace()); assertEquals(1, reply.getNumErrors()); assertEquals(ErrorCode.NO_ADDRESS_FOR_SERVICE, reply.getError(0).getCode()); + log.log(Level.INFO, "Finished: requireThatPolicyCanExpandToPolicy"); } @Test public void requireThatReplyCanBeRemovedFromChildNodes() { + log.log(Level.INFO, "Starting: requireThatReplyCanBeRemovedFromChildNodes"); SimpleProtocol protocol = new SimpleProtocol(); protocol.addPolicyFactory("Custom", new SimpleProtocol.PolicyFactory() { @@ -483,10 +549,12 @@ public class RoutingTestCase { "Sending message", "-Sending message"), reply.getTrace()); + log.log(Level.INFO, "Finished: requireThatReplyCanBeRemovedFromChildNodes"); } @Test public void requireThatSetReplyWorks() { + log.log(Level.INFO, "Starting: requireThatSetReplyWorks"); SimpleProtocol protocol = new SimpleProtocol(); protocol.addPolicyFactory("Select", new CustomPolicyFactory(true, ErrorCode.APP_FATAL_ERROR)); protocol.addPolicyFactory("SetReply", new SimpleProtocol.PolicyFactory() { @@ -509,10 +577,12 @@ public class RoutingTestCase { assertEquals(1, reply.getNumErrors()); assertEquals(ErrorCode.APP_FATAL_ERROR, reply.getError(0).getCode()); assertEquals("foo", reply.getError(0).getMessage()); + log.log(Level.INFO, "Finished: requireThatSetReplyWorks"); } @Test public void requireThatReplyCanBeReusedOnRetry() { + log.log(Level.INFO, "Starting: requireThatReplyCanBeReusedOnRetry"); SimpleProtocol protocol = new SimpleProtocol(); protocol.addPolicyFactory("ReuseReply", new SimpleProtocol.PolicyFactory() { @@ -546,10 +616,12 @@ public class RoutingTestCase { assertNotNull(reply = ((Receptor)srcSession.getReplyHandler()).getReply(60)); System.out.println(reply.getTrace()); assertFalse(reply.hasErrors()); + log.log(Level.INFO, "Finished: requireThatReplyCanBeReusedOnRetry"); } @Test public void requireThatReplyCanBeRemovedAndRetried() { + log.log(Level.INFO, "Starting: requireThatReplyCanBeRemovedAndRetried"); SimpleProtocol protocol = new SimpleProtocol(); protocol.addPolicyFactory("RemoveReply", new SimpleProtocol.PolicyFactory() { @@ -588,10 +660,12 @@ public class RoutingTestCase { "Resolving 'dst/session'.", "Resolving '[SetReply:foo]'."), reply.getTrace()); + log.log(Level.INFO, "Finished: requireThatReplyCanBeRemovedAndRetried"); } @Test public void requireThatIgnoreResultWorks() { + log.log(Level.INFO, "Starting: requireThatIgnoreResultWorks"); assertTrue(srcSession.send(createMessage("msg"), Route.parse("?dst/session")).isAccepted()); Message msg = ((Receptor)dstSession.getMessageHandler()).getMessage(60); assertNotNull(msg); @@ -604,10 +678,12 @@ public class RoutingTestCase { assertFalse(reply.hasErrors()); assertTrace(Arrays.asList("Not waiting for a reply from 'dst/session'."), reply.getTrace()); + log.log(Level.INFO, "Finished: requireThatIgnoreResultWorks"); } @Test public void requireThatIgnoreResultCanBeSetInHopBlueprint() { + log.log(Level.INFO, "Starting: requireThatIgnoreResultCanBeSetInHopBlueprint"); srcServer.setupRouting(new RoutingTableSpec(SimpleProtocol.NAME) .addHop(new HopSpec("foo", "dst/session").setIgnoreResult(true))); assertTrue(srcSession.send(createMessage("msg"), Route.parse("foo")).isAccepted()); @@ -622,31 +698,39 @@ public class RoutingTestCase { assertFalse(reply.hasErrors()); assertTrace(Arrays.asList("Not waiting for a reply from 'dst/session'."), reply.getTrace()); + log.log(Level.INFO, "Finished: requireThatIgnoreResultCanBeSetInHopBlueprint"); } @Test public void requireThatIgnoreFlagPersistsThroughHopLookup() { + log.log(Level.INFO, "Starting: requireThatIgnoreFlagPersistsThroughHopLookup"); setupRouting(new RoutingTableSpec(SimpleProtocol.NAME).addHop(new HopSpec("foo", "dst/unknown"))); assertSend("?foo"); assertTrace("Ignoring errors in reply."); + log.log(Level.INFO, "Finished: requireThatIgnoreFlagPersistsThroughHopLookup"); } @Test public void requireThatIgnoreFlagPersistsThroughRouteLookup() { + log.log(Level.INFO, "Starting: requireThatIgnoreFlagPersistsThroughRouteLookup"); setupRouting(new RoutingTableSpec(SimpleProtocol.NAME).addRoute(new RouteSpec("foo").addHop("dst/unknown"))); assertSend("?foo"); assertTrace("Ignoring errors in reply."); + log.log(Level.INFO, "Finished: requireThatIgnoreFlagPersistsThroughRouteLookup"); } @Test public void requireThatIgnoreFlagPersistsThroughPolicySelect() { + log.log(Level.INFO, "Starting: requireThatIgnoreFlagPersistsThroughPolicySelect"); setupPolicy("Custom", MyPolicy.newSelectAndMerge("dst/unknown")); assertSend("?[Custom]"); assertTrace("Ignoring errors in reply."); + log.log(Level.INFO, "Finished: requireThatIgnoreFlagPersistsThroughPolicySelect"); } @Test public void requireThatIgnoreFlagIsSerializedWithMessage() { + log.log(Level.INFO, "Starting: requireThatIgnoreFlagIsSerializedWithMessage"); assertSend("dst/session foo ?bar"); Message msg = ((Receptor)dstSession.getMessageHandler()).getMessage(60); assertNotNull(msg); @@ -660,90 +744,114 @@ public class RoutingTestCase { assertTrue(hop.getIgnoreResult()); dstSession.acknowledge(msg); assertTrace("-Ignoring errors in reply."); + log.log(Level.INFO, "Finished: requireThatIgnoreFlagIsSerializedWithMessage"); } @Test public void requireThatIgnoreFlagDoesNotInterfere() { + log.log(Level.INFO, "Starting: requireThatIgnoreFlagDoesNotInterfere"); setupPolicy("Custom", MyPolicy.newSelectAndMerge("dst/session")); assertSend("?[Custom]"); assertTrace("-Ignoring errors in reply."); + log.log(Level.INFO, "Finished: requireThatIgnoreFlagDoesNotInterfere"); } @Test public void requireThatEmptySelectionCanBeIgnored() { + log.log(Level.INFO, "Starting: requireThatEmptySelectionCanBeIgnored"); setupPolicy("Custom", MyPolicy.newEmptySelection()); assertSend("?[Custom]"); assertTrace("Ignoring errors in reply."); + log.log(Level.INFO, "Finished: requireThatEmptySelectionCanBeIgnored"); } @Test public void requireThatSelectErrorCanBeIgnored() { + log.log(Level.INFO, "Starting: requireThatSelectErrorCanBeIgnored"); setupPolicy("Custom", MyPolicy.newSelectError(ErrorCode.APP_FATAL_ERROR, "foo")); assertSend("?[Custom]"); assertTrace("Ignoring errors in reply."); + log.log(Level.INFO, "Finished: requireThatSelectErrorCanBeIgnored"); } @Test public void requireThatSelectExceptionCanBeIgnored() { + log.log(Level.INFO, "Starting: requireThatSelectExceptionCanBeIgnored"); setupPolicy("Custom", MyPolicy.newSelectException(new RuntimeException())); assertSend("?[Custom]"); assertTrace("Ignoring errors in reply."); + log.log(Level.INFO, "Finished: requireThatSelectExceptionCanBeIgnored"); } @Test public void requireThatSelectAndThrowCanBeIgnored() { + log.log(Level.INFO, "Starting: requireThatSelectAndThrowCanBeIgnored"); setupPolicy("Custom", MyPolicy.newSelectAndThrow("dst/session", new RuntimeException())); assertSend("?[Custom]"); assertTrace("Ignoring errors in reply."); + log.log(Level.INFO, "Finished: requireThatSelectAndThrowCanBeIgnored"); } @Test public void requireThatEmptyMergeCanBeIgnored() { + log.log(Level.INFO, "Starting: requireThatEmptyMergeCanBeIgnored"); setupPolicy("Custom", MyPolicy.newEmptyMerge("dst/session")); assertSend("?[Custom]"); assertAcknowledge(); assertTrace("Ignoring errors in reply."); + log.log(Level.INFO, "Finished: requireThatEmptyMergeCanBeIgnored"); } @Test public void requireThatMergeErrorCanBeIgnored() { + log.log(Level.INFO, "Starting: requireThatMergeErrorCanBeIgnored"); setupPolicy("Custom", MyPolicy.newMergeError("dst/session", ErrorCode.APP_FATAL_ERROR, "foo")); assertSend("?[Custom]"); assertAcknowledge(); assertTrace("Ignoring errors in reply."); + log.log(Level.INFO, "Finished: requireThatMergeErrorCanBeIgnored"); } @Test public void requireThatMergeExceptionCanBeIgnored() { + log.log(Level.INFO, "Starting: requireThatMergeExceptionCanBeIgnored"); setupPolicy("Custom", MyPolicy.newMergeException("dst/session", new RuntimeException())); assertSend("?[Custom]"); assertAcknowledge(); assertTrace("Ignoring errors in reply."); + log.log(Level.INFO, "Finished: requireThatMergeExceptionCanBeIgnored"); } @Test public void requireThatMergeAndThrowCanBeIgnored() { + log.log(Level.INFO, "Starting: requireThatMergeAndThrowCanBeIgnored"); setupPolicy("Custom", MyPolicy.newMergeAndThrow("dst/session", new RuntimeException())); assertSend("?[Custom]"); assertAcknowledge(); assertTrace("Ignoring errors in reply."); + log.log(Level.INFO, "Finished: requireThatMergeAndThrowCanBeIgnored"); } @Test public void requireThatAllocServiceAddressCanBeIgnored() { + log.log(Level.INFO, "Starting: requireThatAllocServiceAddressCanBeIgnored"); assertSend("?dst/unknown"); assertTrace("Ignoring errors in reply."); + log.log(Level.INFO, "Finished: requireThatAllocServiceAddressCanBeIgnored"); } @Test public void requireThatDepthLimitCanBeIgnored() { + log.log(Level.INFO, "Starting: requireThatDepthLimitCanBeIgnored"); setupPolicy("Custom", MyPolicy.newSelectAndMerge("[Custom]")); assertSend("?[Custom]"); assertTrace("Ignoring errors in reply."); + log.log(Level.INFO, "Finished: requireThatDepthLimitCanBeIgnored"); } @Test public void requireThatRouteCanBeEmptyInDestination() { + log.log(Level.INFO, "Starting: requireThatRouteCanBeEmptyInDestination"); assertTrue(srcSession.send(createMessage("msg"), Route.parse("dst/session")).isAccepted()); Message msg = ((Receptor)dstSession.getMessageHandler()).getMessage(60); assertNotNull(msg); @@ -752,10 +860,12 @@ public class RoutingTestCase { Reply reply = ((Receptor)srcSession.getReplyHandler()).getReply(60); assertNotNull(reply); System.out.println(reply.getTrace()); + log.log(Level.INFO, "Finished: requireThatRouteCanBeEmptyInDestination"); } @Test public void requireThatOnlyActiveNodesAreAborted() { + log.log(Level.INFO, "Starting: requireThatOnlyActiveNodesAreAborted"); SimpleProtocol protocol = new SimpleProtocol(); protocol.addPolicyFactory("Custom", new CustomPolicyFactory(false)); protocol.addPolicyFactory("SetReply", new SimpleProtocol.PolicyFactory() { @@ -778,10 +888,12 @@ public class RoutingTestCase { assertEquals(2, reply.getNumErrors()); assertEquals(ErrorCode.APP_FATAL_ERROR, reply.getError(0).getCode()); assertEquals(ErrorCode.SEND_ABORTED, reply.getError(1).getCode()); + log.log(Level.INFO, "Finished: requireThatOnlyActiveNodesAreAborted"); } @Test public void requireThatTimeoutWorks() { + log.log(Level.INFO, "Starting: requireThatTimeoutWorks"); retryPolicy.setBaseDelay(0.01); srcSession.setTimeout(0.5); assertTrue(srcSession.send(createMessage("msg"), Route.parse("dst/unknown")).isAccepted()); @@ -791,16 +903,19 @@ public class RoutingTestCase { assertEquals(2, reply.getNumErrors()); assertEquals(ErrorCode.NO_ADDRESS_FOR_SERVICE, reply.getError(0).getCode()); assertEquals(ErrorCode.TIMEOUT, reply.getError(1).getCode()); + log.log(Level.INFO, "Finished: requireThatTimeoutWorks"); } @Test public void requireThatUnknownPolicyIsCaught() { + log.log(Level.INFO, "Starting: requireThatUnknownPolicyIsCaught"); assertTrue(srcSession.send(createMessage("msg"), Route.parse("[Unknown]")).isAccepted()); Reply reply = ((Receptor)srcSession.getReplyHandler()).getReply(60); assertNotNull(reply); System.out.println(reply.getTrace()); assertEquals(1, reply.getNumErrors()); assertEquals(ErrorCode.UNKNOWN_POLICY, reply.getError(0).getCode()); + log.log(Level.INFO, "Finished: requireThatUnknownPolicyIsCaught"); } private SimpleProtocol.PolicyFactory exceptionOnSelectThrowingMockFactory() { @@ -829,6 +944,7 @@ public class RoutingTestCase { @Test public void requireThatSelectExceptionIsCaught() { + log.log(Level.INFO, "Starting: requireThatSelectExceptionIsCaught"); SimpleProtocol protocol = new SimpleProtocol(); protocol.addPolicyFactory("Custom", exceptionOnSelectThrowingMockFactory()); srcServer.mb.putProtocol(protocol); @@ -839,6 +955,7 @@ public class RoutingTestCase { assertEquals(1, reply.getNumErrors()); assertEquals(ErrorCode.POLICY_ERROR, reply.getError(0).getCode()); assertTrue(reply.getError(0).getMessage().contains("69")); + log.log(Level.INFO, "Finished: requireThatSelectExceptionIsCaught"); } @Test @@ -857,6 +974,7 @@ public class RoutingTestCase { @Test public void requireThatMergeExceptionIsCaught() { + log.log(Level.INFO, "Starting: requireThatMergeExceptionIsCaught"); SimpleProtocol protocol = new SimpleProtocol(); protocol.addPolicyFactory("Custom", new SimpleProtocol.PolicyFactory() { @@ -892,6 +1010,7 @@ public class RoutingTestCase { assertEquals(1, reply.getNumErrors()); assertEquals(ErrorCode.POLICY_ERROR, reply.getError(0).getCode()); assertTrue(reply.getError(0).getMessage().contains("69")); + log.log(Level.INFO, "Finished: requireThatMergeExceptionIsCaught"); } //////////////////////////////////////////////////////////////////////////////// diff --git a/metrics-proxy/pom.xml b/metrics-proxy/pom.xml index 1633d78668b..76f7e92ad43 100644 --- a/metrics-proxy/pom.xml +++ b/metrics-proxy/pom.xml @@ -147,11 +147,6 @@ <artifactId>jetty-http</artifactId> <scope>test</scope> </dependency> - <dependency> - <groupId>org.hamcrest</groupId> - <artifactId>hamcrest-core</artifactId> - <scope>test</scope> - </dependency> </dependencies> <build> <plugins> diff --git a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/core/MetricsConsumers.java b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/core/MetricsConsumers.java index 457e27a5896..895bf344266 100644 --- a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/core/MetricsConsumers.java +++ b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/core/MetricsConsumers.java @@ -16,6 +16,7 @@ import java.util.Map; import java.util.Set; import java.util.function.Function; import java.util.stream.Collector; +import java.util.stream.Collectors; import static com.yahoo.stream.CustomCollectors.toLinkedMap; import static java.util.Collections.unmodifiableSet; @@ -31,6 +32,7 @@ public class MetricsConsumers { // All metrics for each consumer. private final Map<ConsumerId, List<ConfiguredMetric>> consumerMetrics; + private final Map<ConsumerId, Map<MetricId, ConfiguredMetric>> configuredMetricByMetricByConsumer; // All consumers for each metric (more useful than the opposite map). private final Map<ConfiguredMetric, Set<ConsumerId>> consumersByMetric; @@ -42,6 +44,10 @@ public class MetricsConsumers { consumerMetrics = config.consumer().stream().collect( toUnmodifiableLinkedMap(consumer -> ConsumerId.toConsumerId(consumer.name()), consumer -> convert(consumer.metric()))); + configuredMetricByMetricByConsumer = new HashMap<>(); + consumerMetrics.forEach((consumer, configuredList) -> + configuredMetricByMetricByConsumer.put(consumer, + configuredList.stream().collect(Collectors.toMap(ConfiguredMetric::id, Function.identity())))); consumersByMetric = createConsumersByMetric(consumerMetrics); consumersByMetricByMetricId = new HashMap<>(); consumersByMetric.forEach((configuredMetric, consumers) -> { @@ -67,6 +73,10 @@ public class MetricsConsumers { return consumersByMetricByMetricId.get(id); } + public Map<MetricId, ConfiguredMetric> getMetricsForConsumer(ConsumerId consumerId) { + return configuredMetricByMetricByConsumer.get(consumerId); + } + public Set<ConsumerId> getAllConsumers() { return unmodifiableSet(consumerMetrics.keySet()); } diff --git a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/core/MetricsManager.java b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/core/MetricsManager.java index de8d2c62880..f9ecaa29153 100644 --- a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/core/MetricsManager.java +++ b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/core/MetricsManager.java @@ -81,7 +81,16 @@ public class MetricsManager { * @return metrics for all matching services */ public List<MetricsPacket> getMetrics(List<VespaService> services, Instant startTime) { - MetricsPacket.Builder [] builderArray = getMetricsBuildersAsArray(services, startTime); + MetricsPacket.Builder [] builderArray = getMetricsBuildersAsArray(services, startTime, null); + List<MetricsPacket> metricsPackets = new ArrayList<>(builderArray.length); + for (int i = 0; i < builderArray.length; i++) { + metricsPackets.add(builderArray[i].build()); + builderArray[i] = null; // Set null to be able to GC the builder when packet has been created + } + return metricsPackets; + } + public List<MetricsPacket> getMetrics(List<VespaService> services, Instant startTime, ConsumerId consumerId) { + MetricsPacket.Builder [] builderArray = getMetricsBuildersAsArray(services, startTime, consumerId); List<MetricsPacket> metricsPackets = new ArrayList<>(builderArray.length); for (int i = 0; i < builderArray.length; i++) { metricsPackets.add(builderArray[i].build()); @@ -90,8 +99,8 @@ public class MetricsManager { return metricsPackets; } - public MetricsPacket.Builder [] getMetricsBuildersAsArray(List<VespaService> services, Instant startTime) { - List<MetricsPacket.Builder> builders = getMetricsAsBuilders(services, startTime); + private MetricsPacket.Builder [] getMetricsBuildersAsArray(List<VespaService> services, Instant startTime, ConsumerId consumerId) { + List<MetricsPacket.Builder> builders = getMetricsAsBuilders(services, startTime, consumerId); return builders.toArray(new MetricsPacket.Builder[builders.size()]); } @@ -99,13 +108,13 @@ public class MetricsManager { * Returns the metrics for the given services, in mutable state for further processing. * NOTE: Use {@link #getMetrics(List, Instant)} instead, unless further processing of the metrics is necessary. */ - public List<MetricsPacket.Builder> getMetricsAsBuilders(List<VespaService> services, Instant startTime) { + public List<MetricsPacket.Builder> getMetricsAsBuilders(List<VespaService> services, Instant startTime, ConsumerId consumerId) { if (services.isEmpty()) return Collections.emptyList(); log.log(FINE, () -> "Updating services prior to fetching metrics, number of services= " + services.size()); vespaServices.updateServices(services); - List<MetricsPacket.Builder> result = vespaMetrics.getMetrics(services); + List<MetricsPacket.Builder> result = vespaMetrics.getMetrics(services, consumerId); log.log(FINE, () -> "Got " + result.size() + " metrics packets for vespa services."); purgeStaleMetrics(); diff --git a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/core/VespaMetrics.java b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/core/VespaMetrics.java index 7a3f260cd34..ebb5d2fe8fb 100644 --- a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/core/VespaMetrics.java +++ b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/core/VespaMetrics.java @@ -15,6 +15,7 @@ import ai.vespa.metricsproxy.metric.model.MetricsPacket; import ai.vespa.metricsproxy.service.MetricsParser; import ai.vespa.metricsproxy.service.VespaService; +import java.time.Instant; import java.util.ArrayList; import java.util.Collections; import java.util.HashMap; @@ -65,7 +66,7 @@ public class VespaMetrics { * @param services the services to get metrics for * @return a list of metrics packet builders (to allow modification by the caller) */ - public List<MetricsPacket.Builder> getMetrics(List<VespaService> services) { + public List<MetricsPacket.Builder> getMetrics(List<VespaService> services, ConsumerId consumerId) { List<MetricsPacket.Builder> metricsPackets = new ArrayList<>(); for (VespaService service : services) { @@ -74,7 +75,9 @@ public class VespaMetrics { systemCheck.ifPresent(metricsPackets::add); MetricAggregator aggregator = new MetricAggregator(service.getDimensions()); - GetServiceMetricsConsumer metricsConsumer = new GetServiceMetricsConsumer(metricsConsumers, aggregator); + MetricsParser.Consumer metricsConsumer = (consumerId != null) + ? new GetServiceMetricsConsumer(metricsConsumers, aggregator, consumerId) + : new GetServiceMetricsConsumerForAll(metricsConsumers, aggregator); service.consumeMetrics(metricsConsumer); if (! aggregator.getAggregated().isEmpty()) { @@ -115,24 +118,14 @@ public class VespaMetrics { * In order to include a metric, it must exist in the given map of metric to consumers. * Each returned metric will contain a collection of consumers that it should be routed to. */ - private static class GetServiceMetricsConsumer implements MetricsParser.Consumer { - private final MetricAggregator aggregator; - private final MetricsConsumers metricsConsumers; - GetServiceMetricsConsumer(MetricsConsumers metricsConsumers, MetricAggregator aggregator) { - this.metricsConsumers = metricsConsumers; + private static abstract class GetServiceMetricsConsumerBase implements MetricsParser.Consumer { + protected final MetricAggregator aggregator; + + GetServiceMetricsConsumerBase(MetricAggregator aggregator) { this.aggregator = aggregator; } - @Override - public void consume(Metric candidate) { - Map<ConfiguredMetric, Set<ConsumerId>> consumersByMetric = metricsConsumers.getConsumersByMetric(candidate.getName()); - if (consumersByMetric != null) { - consumersByMetric.keySet().forEach( - configuredMetric -> aggregator.aggregate( - metricWithConfigProperties(candidate, configuredMetric, consumersByMetric.get(configuredMetric)))); - } - } - private static Metric metricWithConfigProperties(Metric candidate, + protected static Metric metricWithConfigProperties(Metric candidate, ConfiguredMetric configuredMetric, Set<ConsumerId> consumers) { Metric metric = candidate.clone(); @@ -153,11 +146,46 @@ public class VespaMetrics { } private static Set<ConsumerId> extractConsumers(Set<ConsumerId> configuredConsumers) { - Set<ConsumerId> consumers = Collections.emptySet(); - if (configuredConsumers != null) { - consumers = configuredConsumers; + return (configuredConsumers != null) ? configuredConsumers : Set.of(); + } + } + + private static class GetServiceMetricsConsumer extends GetServiceMetricsConsumerBase { + private final Map<MetricId, ConfiguredMetric> configuredMetrics; + private final Set<ConsumerId> consumerId; + + GetServiceMetricsConsumer(MetricsConsumers metricsConsumers, MetricAggregator aggregator, ConsumerId consumerId) { + super(aggregator); + this.consumerId = Set.of(consumerId); + this.configuredMetrics = metricsConsumers.getMetricsForConsumer(consumerId); + } + + @Override + public void consume(Metric candidate) { + ConfiguredMetric configuredMetric = configuredMetrics.get(candidate.getName()); + if (configuredMetric != null) { + aggregator.aggregate( + metricWithConfigProperties(candidate, configuredMetric, consumerId)); + } + } + } + + private static class GetServiceMetricsConsumerForAll extends GetServiceMetricsConsumerBase { + private final MetricsConsumers metricsConsumers; + + GetServiceMetricsConsumerForAll(MetricsConsumers metricsConsumers, MetricAggregator aggregator) { + super(aggregator); + this.metricsConsumers = metricsConsumers; + } + + @Override + public void consume(Metric candidate) { + Map<ConfiguredMetric, Set<ConsumerId>> consumersByMetric = metricsConsumers.getConsumersByMetric(candidate.getName()); + if (consumersByMetric != null) { + consumersByMetric.keySet().forEach( + configuredMetric -> aggregator.aggregate( + metricWithConfigProperties(candidate, configuredMetric, consumersByMetric.get(configuredMetric)))); } - return consumers; } } @@ -189,14 +217,7 @@ public class VespaMetrics { mergedDimensions.putAll(metric.getDimensions()); mergedDimensions.putAll(serviceDimensions); AggregationKey aggregationKey = new AggregationKey(mergedDimensions, metric.getConsumers()); - - if (aggregated.containsKey(aggregationKey)) { - aggregated.get(aggregationKey).add(metric); - } else { - List<Metric> ml = new ArrayList<>(); - ml.add(metric); - aggregated.put(aggregationKey, ml); - } + aggregated.computeIfAbsent(aggregationKey, key -> new ArrayList<>()).add(metric); } } @@ -207,8 +228,8 @@ public class VespaMetrics { return definitions == null ? Collections.emptyList() : definitions; } - private static void setMetaInfo(MetricsPacket.Builder builder, long timestamp) { - builder.timestamp(timestamp) + private static void setMetaInfo(MetricsPacket.Builder builder, Instant timestamp) { + builder.timestamp(timestamp.getEpochSecond()) .statusCode(0) .statusMessage("Data collected successfully"); } diff --git a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/http/ValuesFetcher.java b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/http/ValuesFetcher.java index 8f7f1c8a779..df670addcd8 100644 --- a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/http/ValuesFetcher.java +++ b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/http/ValuesFetcher.java @@ -41,7 +41,7 @@ public class ValuesFetcher { public List<MetricsPacket> fetch(String requestedConsumer) throws JsonRenderingException { ConsumerId consumer = getConsumerOrDefault(requestedConsumer, metricsConsumers); - return fetchAllMetrics() + return metricsManager.getMetrics(vespaServices.getVespaServices(), Instant.now(), consumer) .stream() .filter(metricsPacket -> metricsPacket.consumers().contains(consumer)) .collect(Collectors.toList()); @@ -50,7 +50,7 @@ public class ValuesFetcher { public MetricsPacket.Builder [] fetchMetricsAsBuilders(String requestedConsumer) throws JsonRenderingException { ConsumerId consumer = getConsumerOrDefault(requestedConsumer, metricsConsumers); - List<MetricsPacket.Builder> builders = metricsManager.getMetricsAsBuilders(vespaServices.getVespaServices(), Instant.now()) + List<MetricsPacket.Builder> builders = metricsManager.getMetricsAsBuilders(vespaServices.getVespaServices(), Instant.now(), consumer) .stream() .filter(builder -> builder.hasConsumer(consumer)) .collect(Collectors.toList()); diff --git a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/http/yamas/YamasHandler.java b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/http/yamas/YamasHandler.java index 822f4e0b5d5..10b3cc3077b 100644 --- a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/http/yamas/YamasHandler.java +++ b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/http/yamas/YamasHandler.java @@ -61,7 +61,7 @@ public class YamasHandler extends HttpHandlerBase { try { List<MetricsPacket> metrics = consumer == null ? valuesFetcher.fetchAllMetrics() : valuesFetcher.fetch(consumer); metrics.addAll(nodeMetricGatherer.gatherMetrics()); // TODO: Currently only add these metrics in this handler. Eventually should be included in all handlers - return new YamasResponse(OK, YamasJsonUtil.toYamasArray(metrics, true)); + return new YamasResponse(OK, metrics); } catch (JsonRenderingException e) { return new ErrorResponse(INTERNAL_SERVER_ERROR, e.getMessage()); } diff --git a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/http/yamas/YamasResponse.java b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/http/yamas/YamasResponse.java index 23b56e94ae4..49f5036b3fd 100644 --- a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/http/yamas/YamasResponse.java +++ b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/http/yamas/YamasResponse.java @@ -1,24 +1,25 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package ai.vespa.metricsproxy.http.yamas; -import ai.vespa.metricsproxy.metric.model.json.JacksonUtil; -import ai.vespa.metricsproxy.metric.model.json.YamasArrayJsonModel; -import com.fasterxml.jackson.databind.ObjectMapper; +import ai.vespa.metricsproxy.metric.model.MetricsPacket; +import ai.vespa.metricsproxy.metric.model.json.YamasJsonUtil; import com.yahoo.container.jdisc.HttpResponse; import java.io.IOException; import java.io.OutputStream; +import java.util.List; + /** * @author olaa */ public class YamasResponse extends HttpResponse { - private final YamasArrayJsonModel data; + private final List<MetricsPacket> metrics; - public YamasResponse(int code, YamasArrayJsonModel data) { + public YamasResponse(int code, List<MetricsPacket> metrics) { super(code); - this.data = data; + this.metrics = metrics; } @Override @@ -28,8 +29,7 @@ public class YamasResponse extends HttpResponse { @Override public void render(OutputStream outputStream) throws IOException { - ObjectMapper mapper = JacksonUtil.createObjectMapper(); - mapper.writeValue(outputStream, data); + YamasJsonUtil.toJson(metrics, outputStream, true); } } diff --git a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/Metric.java b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/Metric.java index a9e6e8e6e3c..d12685be97d 100644 --- a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/Metric.java +++ b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/Metric.java @@ -5,8 +5,7 @@ import ai.vespa.metricsproxy.metric.model.ConsumerId; import ai.vespa.metricsproxy.metric.model.DimensionId; import ai.vespa.metricsproxy.metric.model.MetricId; -import java.util.Collections; -import java.util.LinkedHashMap; +import java.time.Instant; import java.util.Map; import java.util.Objects; import java.util.Set; @@ -16,7 +15,7 @@ import java.util.Set; */ public class Metric { - private final long time; + private final Instant time; private final Number value; private final String description; private MetricId name; @@ -30,24 +29,28 @@ public class Metric { * @param value The numeric value * @param time The timestamp of this metric in seconds */ - public Metric(MetricId name, Number value, long time, Map<DimensionId, String> dimensions, String description) { + public Metric(MetricId name, Number value, Instant time, Map<DimensionId, String> dimensions, String description) { this.time = time; this.value = value; this.name = name; - this.dimensions = dimensions; + this.dimensions = Map.copyOf(dimensions); this.description = description; } public Metric(MetricId name, Number value, long timestamp) { - this(name, value, timestamp, Collections.emptyMap(), ""); + this(name, value, Instant.ofEpochSecond(timestamp), Map.of(), ""); + } + + public Metric(MetricId name, Number value, Instant timestamp) { + this(name, value, timestamp, Map.of(), ""); } public Metric(MetricId name, Number value) { - this(name, value, System.currentTimeMillis() / 1000); + this(name, value, Instant.now()); } public void setDimensions(Map<DimensionId, String> dimensions) { - this.dimensions = dimensions; + this.dimensions = Map.copyOf(dimensions); } /** @@ -88,7 +91,7 @@ public class Metric { /** * @return The UTC timestamp for when this metric was collected */ - public long getTimeStamp() { + public Instant getTimeStamp() { return this.time; } @@ -104,7 +107,7 @@ public class Metric { @Override public Metric clone() { - return new Metric(name, value, time, new LinkedHashMap<>(dimensions), getDescription()); + return new Metric(name, value, time, dimensions, getDescription()); } @Override @@ -114,7 +117,7 @@ public class Metric { return name.equals(rhs.name) && description.equals(rhs.description) && value.equals(rhs.value) - && (time == rhs.time) + && time.equals(rhs.time) && Objects.equals(dimensions, rhs.dimensions) && Objects.equals(consumers, rhs.consumers); } diff --git a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/Metrics.java b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/Metrics.java index 8450d9f6be7..ebc206c9245 100644 --- a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/Metrics.java +++ b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/Metrics.java @@ -1,6 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package ai.vespa.metricsproxy.metric; +import java.time.Instant; import java.util.ArrayList; import java.util.Collections; import java.util.List; @@ -13,14 +14,14 @@ import java.util.List; public class Metrics { private final List<Metric> metrics = new ArrayList<>(); - private long timestamp; + private Instant timestamp; private boolean isFrozen = false; public Metrics() { - this(System.currentTimeMillis() / 1000L); + this(Instant.now()); } - public Metrics(long timestamp) { + public Metrics(Instant timestamp) { this.timestamp = timestamp; } @@ -28,7 +29,7 @@ public class Metrics { if (isFrozen) throw new IllegalStateException("Frozen Metrics cannot be modified!"); } - public long getTimeStamp() { + public Instant getTimeStamp() { return this.timestamp; } @@ -37,7 +38,7 @@ public class Metrics { * * @param timestamp IN UTC seconds resolution */ - public void setTimeStamp(long timestamp) { + public void setTimeStamp(Instant timestamp) { ensureNotFrozen(); this.timestamp = timestamp; } diff --git a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/model/MetricsPacket.java b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/model/MetricsPacket.java index 54ffcd4ae0f..35761ebcf18 100644 --- a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/model/MetricsPacket.java +++ b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/model/MetricsPacket.java @@ -110,7 +110,7 @@ public class MetricsPacket { public Builder putMetrics(Collection<Metric> extraMetrics) { if (extraMetrics != null) - extraMetrics.forEach(metric -> metrics.put(metric.getName(), metric.getValue().doubleValue())); + extraMetrics.forEach(metric -> metrics.put(metric.getName(), metric.getValue())); return this; } diff --git a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/model/json/GenericJsonModel.java b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/model/json/GenericJsonModel.java index 9bc0dddb784..133d8fee81f 100644 --- a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/model/json/GenericJsonModel.java +++ b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/model/json/GenericJsonModel.java @@ -9,8 +9,6 @@ import com.fasterxml.jackson.databind.ObjectMapper; import java.io.IOException; import java.util.List; -import java.util.logging.Level; -import java.util.logging.Logger; import static com.fasterxml.jackson.annotation.JsonInclude.Include.NON_ABSENT; diff --git a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/model/json/JacksonUtil.java b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/model/json/JacksonUtil.java index 4ac2cc4dec8..d599e75b243 100644 --- a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/model/json/JacksonUtil.java +++ b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/model/json/JacksonUtil.java @@ -11,6 +11,7 @@ import com.fasterxml.jackson.databind.module.SimpleModule; import java.io.IOException; import java.text.DecimalFormat; import java.text.DecimalFormatSymbols; +import java.text.NumberFormat; import java.util.Locale; /** @@ -19,6 +20,18 @@ import java.util.Locale; */ public class JacksonUtil { + private static final ThreadLocal<DecimalFormat> withinLongRangeFormat = ThreadLocal.withInitial(JacksonUtil::createWithinLongRangeFormat); + private static final ThreadLocal<DecimalFormat> outsideLongRangeFormat = ThreadLocal.withInitial(JacksonUtil::createOutsideLongRangeFormat); + private static DecimalFormat createWithinLongRangeFormat() { + DecimalFormat df = new DecimalFormat("#.####", new DecimalFormatSymbols(Locale.ENGLISH)); + df.setMaximumFractionDigits(13); + return df; + } + private static DecimalFormat createOutsideLongRangeFormat() { + DecimalFormat df = new DecimalFormat("#.0###", new DecimalFormatSymbols(Locale.ENGLISH)); + df.setMaximumFractionDigits(13); + return df; + } /** * Returns an object mapper with a custom floating point serializer to avoid scientific notation */ @@ -30,14 +43,31 @@ public class JacksonUtil { mapper.registerModule(module); return mapper; } + /** + * Returns an object mapper with a custom floating point serializer to avoid scientific notation + */ + public static void writeDouble(JsonGenerator jgen, Double value) throws IOException { + jgen.writeNumber(format(value)); + } + + public static String format(Double value) { + return format(value, withinLongRangeFormat.get(), outsideLongRangeFormat.get()); + } + + private static String format(Double value, NumberFormat withinLongRange, NumberFormat outsideLongRange) { + if ((value <= Long.MAX_VALUE) && (value >= Long.MIN_VALUE)) { + return withinLongRange.format(value); + } else { + return outsideLongRange.format(value); + } + } public static class DoubleSerializer extends JsonSerializer<Double> { + private DecimalFormat withinLongRangeFormat = createWithinLongRangeFormat(); + private DecimalFormat outsideLongRangeFormat = createOutsideLongRangeFormat(); @Override - public void serialize(Double value, JsonGenerator jgen, - SerializerProvider provider) throws IOException { - DecimalFormat df = new DecimalFormat("#.####", new DecimalFormatSymbols(Locale.ENGLISH)); - df.setMaximumFractionDigits(13); - jgen.writeNumber(df.format(value)); + public void serialize(Double value, JsonGenerator jgen, SerializerProvider provider) throws IOException { + jgen.writeNumber(format(value, withinLongRangeFormat, outsideLongRangeFormat)); } } diff --git a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/model/json/YamasArrayJsonModel.java b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/model/json/YamasArrayJsonModel.java deleted file mode 100644 index 539e3b851ae..00000000000 --- a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/model/json/YamasArrayJsonModel.java +++ /dev/null @@ -1,54 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package ai.vespa.metricsproxy.metric.model.json; - -import com.fasterxml.jackson.annotation.JsonIgnoreProperties; -import com.fasterxml.jackson.annotation.JsonInclude; -import com.fasterxml.jackson.annotation.JsonProperty; -import com.fasterxml.jackson.core.JsonProcessingException; -import com.fasterxml.jackson.databind.ObjectMapper; - -import java.util.ArrayList; -import java.util.List; - -/** - * Datamodel for the metricsproxy representation of multiple yamas checks. - */ -@JsonIgnoreProperties(ignoreUnknown = true) -@JsonInclude(JsonInclude.Include.NON_NULL) -public class YamasArrayJsonModel { - @JsonProperty("metrics") - public final List<YamasJsonModel> metrics = new ArrayList<>(); - - public void add(List<YamasJsonModel> results) { - metrics.addAll(results); - } - - public void add(YamasJsonModel result) { - metrics.add(result); - } - - public void add(YamasArrayJsonModel array) { - metrics.addAll(array.metrics); - } - - /** - * Convenience method to serialize. - * <p> - * Custom floating point serializer to avoid scientifc notation - * - * @return Serialized json - */ - public String serialize() { - ObjectMapper mapper = JacksonUtil.createObjectMapper(); - - if (metrics.size() > 0) { - try { - return mapper.writeValueAsString(this); - } catch (JsonProcessingException e) { - e.printStackTrace(); - } - } - return "{}"; // Backwards compatibility - } - -} diff --git a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/model/json/YamasJsonModel.java b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/model/json/YamasJsonModel.java index d2be7be25d1..911c772c0e4 100644 --- a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/model/json/YamasJsonModel.java +++ b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/model/json/YamasJsonModel.java @@ -10,7 +10,6 @@ import com.fasterxml.jackson.annotation.JsonInclude; import com.fasterxml.jackson.annotation.JsonProperty; import com.fasterxml.jackson.annotation.JsonPropertyOrder; -import java.util.ArrayList; import java.util.HashMap; import java.util.LinkedHashMap; import java.util.LinkedHashSet; diff --git a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/model/json/YamasJsonUtil.java b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/model/json/YamasJsonUtil.java index 602879aa0b6..b5b8e30a218 100644 --- a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/model/json/YamasJsonUtil.java +++ b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/model/json/YamasJsonUtil.java @@ -5,21 +5,21 @@ import ai.vespa.metricsproxy.metric.model.ConsumerId; import ai.vespa.metricsproxy.metric.model.MetricsPacket; import ai.vespa.metricsproxy.metric.model.ServiceId; import com.fasterxml.jackson.core.JsonFactory; +import com.fasterxml.jackson.core.JsonGenerator; import com.fasterxml.jackson.core.JsonParser; +import com.fasterxml.jackson.core.StreamWriteFeature; import com.fasterxml.jackson.databind.ObjectMapper; -import com.yahoo.concurrent.CopyOnWriteHashMap; +import java.io.ByteArrayOutputStream; import java.io.IOException; +import java.io.OutputStream; import java.util.ArrayList; -import java.util.Collection; import java.util.List; -import java.util.Map; import java.util.Set; import java.util.logging.Logger; import java.util.stream.Collectors; import static ai.vespa.metricsproxy.http.ValuesFetcher.defaultMetricsConsumerId; -import static com.yahoo.stream.CustomCollectors.toLinkedMap; import static java.util.Collections.emptyList; import static java.util.logging.Level.WARNING; @@ -28,10 +28,12 @@ import static java.util.logging.Level.WARNING; */ public class YamasJsonUtil { private static final Logger log = Logger.getLogger(YamasJsonUtil.class.getName()); + private static final JsonFactory factory = JsonFactory.builder() + .enable(StreamWriteFeature.WRITE_BIGDECIMAL_AS_PLAIN) + .build(); static final String YAMAS_ROUTING = "yamas"; - private static final Map<Set<ConsumerId>, Map<String, YamasJsonModel.YamasJsonNamespace>> globalNameSpaces = new CopyOnWriteHashMap<>(); public static MetricsPacket.Builder toMetricsPacketBuilder(YamasJsonModel jsonModel) { if (jsonModel.application == null) throw new IllegalArgumentException("Service id cannot be null"); @@ -45,21 +47,6 @@ public class YamasJsonUtil { .addConsumers(jsonModel.getYamasConsumers()); } - public static YamasArrayJsonModel toYamasArray(Collection<MetricsPacket> metricsPackets) { - YamasArrayJsonModel yamasArray = toYamasArray(metricsPackets, false); - - // Add a single status object at the end - yamasArray.metrics.stream().findFirst().map(YamasJsonModel::getYamasConsumers) - .ifPresent(consumers -> yamasArray.add(getStatusYamasModel("Data collected successfully", 0, consumers))); - return yamasArray; - } - - public static YamasArrayJsonModel toYamasArray(Collection<MetricsPacket> metricsPackets, boolean addStatus) { - YamasArrayJsonModel yamasArray = new YamasArrayJsonModel(); - metricsPackets.forEach(packet -> yamasArray.add(toYamasModel(packet, addStatus))); - return yamasArray; - } - /** * Converts the given json formatted string to a list of metrics packet builders. * Note that this method returns an empty list if an IOException occurs, @@ -81,58 +68,91 @@ public class YamasJsonUtil { } } - private static YamasJsonModel getStatusYamasModel(String statusMessage, int statusCode, Set<ConsumerId> consumers) { - YamasJsonModel model = new YamasJsonModel(); - model.status_code = statusCode; - model.status_msg = statusMessage; - model.application = "yms_check_vespa"; - model.routing = computeIfAbsent(consumers); - return model; + public static List<MetricsPacket> appendOptionalStatusPacket(List<MetricsPacket> packets) { + if (packets.isEmpty()) return packets; + + Set<ConsumerId> consumers = extractSetForRouting(packets.get(0).consumers()); + if (consumers.isEmpty()) return packets; + List<MetricsPacket> withStatus = new ArrayList<>(packets); + withStatus.add(new MetricsPacket.Builder(ServiceId.toServiceId("yms_check_vespa")) + .statusCode(0) + .statusMessage("Data collected successfully") + .addConsumers(consumers).build()); + return withStatus; } - private static YamasJsonModel toYamasModel(MetricsPacket packet, boolean addStatus) { - YamasJsonModel model = new YamasJsonModel(); - - if (addStatus) { - model.status_code = packet.statusCode; - model.status_msg = packet.statusMessage; + public static String toJson(List<MetricsPacket> metrics, boolean addStatus) { + try (ByteArrayOutputStream output = new ByteArrayOutputStream()) { + toJson(metrics, output, addStatus); + output.flush(); + return output.toString(); + } catch (IOException e) { + return "{}"; } - - model.application = packet.service.id; - model.timestamp = (packet.timestamp == 0L) ? null : packet.timestamp; - - model.metrics = (packet.metrics().isEmpty()) - ? null - : packet.metrics().entrySet().stream().collect( - toLinkedMap(id2metric -> id2metric.getKey().id, - id2metric -> id2metric.getValue().doubleValue())); - - model.dimensions = (packet.dimensions().isEmpty()) - ? null - : packet.dimensions().entrySet() - .stream() - .filter(entry -> entry.getKey() != null && entry.getValue() != null) - .collect(toLinkedMap(id2dim -> id2dim.getKey().id, Map.Entry::getValue)); - - model.routing = computeIfAbsent(packet.consumers()); - - return model; } - - private static Map<String, YamasJsonModel.YamasJsonNamespace> computeIfAbsent(Set<ConsumerId> consumers) { - return globalNameSpaces.computeIfAbsent(consumers, YamasJsonUtil::createYamasJson); + private static Set<ConsumerId> extractSetForRouting(Set<ConsumerId> consumers) { + return consumers.stream() + .filter(consumerId -> consumerId != defaultMetricsConsumerId) + .collect(Collectors.toSet()); + } + public static void toJson(List<MetricsPacket> metrics, OutputStream outputStream, boolean addStatus) throws IOException { + JsonGenerator generator = factory.createGenerator(outputStream); + generator.writeStartObject(); + if (metrics.isEmpty()) { + generator.writeEndObject(); + return; + } + generator.writeArrayFieldStart("metrics"); + for (int i = 0; i < metrics.size() - 1; i++) { + toJson(metrics.get(i), generator, addStatus); + } + toJson(metrics.get(metrics.size() - 1), generator, true); + generator.writeEndArray(); + generator.writeEndObject(); + generator.close(); } - private static Map<String, YamasJsonModel.YamasJsonNamespace> createYamasJson(Set<ConsumerId> consumers) { - List<String> namespaces = consumers.stream() - .filter(consumerId -> consumerId != defaultMetricsConsumerId) - .map(consumer -> consumer.id) - .collect(Collectors.toList()); - if (namespaces.isEmpty()) return null; + private static void toJson(MetricsPacket metric, JsonGenerator generator, boolean addStatus) throws IOException { + generator.writeStartObject(); + if (addStatus) { + generator.writeNumberField("status_code", metric.statusCode); + } + if (metric.timestamp != 0) { + generator.writeNumberField("timestamp", metric.timestamp); + } + generator.writeStringField("application", metric.service.id); - YamasJsonModel.YamasJsonNamespace yamasJsonNamespace = new YamasJsonModel.YamasJsonNamespace(); - yamasJsonNamespace.namespaces = namespaces; - return Map.of(YAMAS_ROUTING, yamasJsonNamespace); - } + if ( ! metric.metrics().isEmpty()) { + generator.writeObjectFieldStart("metrics"); + for (var m : metric.metrics().entrySet()) { + generator.writeFieldName(m.getKey().id); + JacksonUtil.writeDouble(generator, m.getValue().doubleValue()); + } + generator.writeEndObject(); + } + if ( ! metric.dimensions().isEmpty()) { + generator.writeObjectFieldStart("dimensions"); + for (var m : metric.dimensions().entrySet()) { + generator.writeStringField(m.getKey().id, m.getValue()); + } + generator.writeEndObject(); + } + Set<ConsumerId> routing = extractSetForRouting(metric.consumers()); + if (!routing.isEmpty()) { + generator.writeObjectFieldStart("routing"); + generator.writeObjectFieldStart(YAMAS_ROUTING); + generator.writeArrayFieldStart("namespaces"); + for (ConsumerId consumer : routing) { + generator.writeString(consumer.id); + } + generator.writeEndArray(); + generator.writeEndObject(); + generator.writeEndObject(); + } + if (addStatus) { + generator.writeStringField("status_msg", metric.statusMessage); + } + generator.writeEndObject(); + } } diff --git a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/rpc/RpcServer.java b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/rpc/RpcServer.java index 3a0a9e9cf73..63672e7e600 100644 --- a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/rpc/RpcServer.java +++ b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/rpc/RpcServer.java @@ -4,6 +4,7 @@ package ai.vespa.metricsproxy.rpc; import ai.vespa.metricsproxy.core.MetricsManager; import ai.vespa.metricsproxy.metric.model.ConsumerId; import ai.vespa.metricsproxy.metric.model.MetricsPacket; +import ai.vespa.metricsproxy.metric.model.json.YamasJsonUtil; import ai.vespa.metricsproxy.service.VespaService; import ai.vespa.metricsproxy.service.VespaServices; import com.yahoo.jrt.ErrorCode; @@ -17,11 +18,10 @@ import java.util.logging.Level; import java.util.logging.Logger; import static ai.vespa.metricsproxy.metric.model.ConsumerId.toConsumerId; +import static ai.vespa.metricsproxy.metric.model.json.YamasJsonUtil.toJson; import static ai.vespa.metricsproxy.metric.model.json.YamasJsonUtil.toMetricsPackets; -import static ai.vespa.metricsproxy.metric.model.json.YamasJsonUtil.toYamasArray; import static com.yahoo.collections.CollectionUtil.mkString; import static java.util.logging.Level.FINE; -import static java.util.logging.Level.INFO; import static java.util.logging.Level.WARNING; /** @@ -125,9 +125,9 @@ public class RpcServer { log.log(FINE, () -> "Getting metrics for services: " + mkString(services, "[", ", ", "]")); if (services.isEmpty()) setNoServiceError(req, service); else withExceptionHandling(req, () -> { - List<MetricsPacket> packets = metricsManager.getMetrics(services, startTime); + List<MetricsPacket> packets = YamasJsonUtil.appendOptionalStatusPacket(metricsManager.getMetrics(services, startTime)); log.log(FINE,() -> "Returning metrics packets:\n" + mkString(packets, "\n")); - req.returnValues().add(new StringValue(toYamasArray(packets).serialize())); + req.returnValues().add(new StringValue(toJson(packets, false))); }); req.returnRequest(); } @@ -138,8 +138,8 @@ public class RpcServer { List<VespaService> services = vespaServices.getMonitoringServices(service); if (services.isEmpty()) setNoServiceError(req, service); else withExceptionHandling(req, () -> { - List<MetricsPacket> packets = metricsManager.getHealthMetrics(services); - req.returnValues().add(new StringValue(toYamasArray(packets, true).serialize())); + List<MetricsPacket> packets = YamasJsonUtil.appendOptionalStatusPacket(metricsManager.getHealthMetrics(services)); + req.returnValues().add(new StringValue(toJson(packets, true))); }); req.returnRequest(); } diff --git a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/service/HttpMetricFetcher.java b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/service/HttpMetricFetcher.java index 65c9bb1ff76..87bed1c79e3 100644 --- a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/service/HttpMetricFetcher.java +++ b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/service/HttpMetricFetcher.java @@ -77,15 +77,14 @@ public abstract class HttpMetricFetcher { } void handleException(Exception e, Object data, int timesFetched) { - logMessage("Unable to parse json '" + data + "' for service '" + service + "': " + - Exceptions.toMessageString(e), timesFetched); + logMessage("Unable to parse json '" + data + "' for service '" + service + "': ", e, timesFetched); } - private void logMessage(String message, int timesFetched) { + private void logMessage(String message, Exception e, int timesFetched) { if (service.isAlive() && timesFetched > 5) { - log.log(Level.INFO, message); + log.log(Level.INFO, message, e); } else { - log.log(Level.FINE, message); + log.log(Level.FINE, message, e); } } diff --git a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/service/MetricsParser.java b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/service/MetricsParser.java index 8157ecb72fd..bb21485e0e7 100644 --- a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/service/MetricsParser.java +++ b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/service/MetricsParser.java @@ -6,15 +6,16 @@ import ai.vespa.metricsproxy.metric.model.DimensionId; import ai.vespa.metricsproxy.metric.model.MetricId; import com.fasterxml.jackson.core.JsonParser; import com.fasterxml.jackson.core.JsonToken; -import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; import java.io.IOException; import java.io.InputStream; -import java.util.Collections; +import java.time.Instant; +import java.util.ArrayList; import java.util.HashMap; -import java.util.Iterator; +import java.util.List; import java.util.Map; +import java.util.stream.Collectors; import static ai.vespa.metricsproxy.metric.model.DimensionId.toDimensionId; @@ -30,7 +31,7 @@ public class MetricsParser { private static final ObjectMapper jsonMapper = new ObjectMapper(); - static void parse(String data, Consumer consumer) throws IOException { + public static void parse(String data, Consumer consumer) throws IOException { parse(jsonMapper.createParser(data), consumer); } @@ -54,21 +55,17 @@ public class MetricsParser { } } } - private static long secondsSince1970UTC() { - return System.currentTimeMillis() / 1000L; - } - static private long parseSnapshot(JsonParser parser) throws IOException { + static private Instant parseSnapshot(JsonParser parser) throws IOException { if (parser.getCurrentToken() != JsonToken.START_OBJECT) { throw new IOException("Expected start of 'snapshot' object, got " + parser.currentToken()); } - long timestamp = secondsSince1970UTC(); + Instant timestamp = Instant.now(); for (parser.nextToken(); parser.getCurrentToken() != JsonToken.END_OBJECT; parser.nextToken()) { String fieldName = parser.getCurrentName(); JsonToken token = parser.nextToken(); if (fieldName.equals("to")) { - timestamp = parser.getLongValue(); - long now = System.currentTimeMillis() / 1000; - timestamp = Metric.adjustTime(timestamp, now); + timestamp = Instant.ofEpochSecond(parser.getLongValue()); + timestamp = Instant.ofEpochSecond(Metric.adjustTime(timestamp.getEpochSecond(), Instant.now().getEpochSecond())); } else { if (token == JsonToken.START_OBJECT || token == JsonToken.START_ARRAY) { parser.skipChildren(); @@ -78,19 +75,14 @@ public class MetricsParser { return timestamp; } - static private void parseValues(JsonParser parser, long timestamp, Consumer consumer) throws IOException { + static private void parseMetricValues(JsonParser parser, Instant timestamp, Consumer consumer) throws IOException { if (parser.getCurrentToken() != JsonToken.START_ARRAY) { throw new IOException("Expected start of 'metrics:values' array, got " + parser.currentToken()); } - Map<String, Map<DimensionId, String>> uniqueDimensions = new HashMap<>(); + Map<Long, Map<DimensionId, String>> uniqueDimensions = new HashMap<>(); while (parser.nextToken() == JsonToken.START_OBJECT) { - // read everything from this START_OBJECT to the matching END_OBJECT - // and return it as a tree model ObjectNode - JsonNode value = jsonMapper.readTree(parser); - handleValue(value, timestamp, consumer, uniqueDimensions); - - // do whatever you need to do with this object + handleValue(parser, timestamp, consumer, uniqueDimensions); } } @@ -98,14 +90,14 @@ public class MetricsParser { if (parser.getCurrentToken() != JsonToken.START_OBJECT) { throw new IOException("Expected start of 'metrics' object, got " + parser.currentToken()); } - long timestamp = System.currentTimeMillis() / 1000L; + Instant timestamp = Instant.now(); for (parser.nextToken(); parser.getCurrentToken() != JsonToken.END_OBJECT; parser.nextToken()) { String fieldName = parser.getCurrentName(); JsonToken token = parser.nextToken(); if (fieldName.equals("snapshot")) { timestamp = parseSnapshot(parser); } else if (fieldName.equals("values")) { - parseValues(parser, timestamp, consumer); + parseMetricValues(parser, timestamp, consumer); } else { if (token == JsonToken.START_OBJECT || token == JsonToken.START_ARRAY) { parser.skipChildren(); @@ -114,50 +106,69 @@ public class MetricsParser { } } - static private void handleValue(JsonNode metric, long timestamp, Consumer consumer, - Map<String, Map<DimensionId, String>> uniqueDimensions) { - String name = metric.get("name").textValue(); - String description = ""; - - if (metric.has("description")) { - description = metric.get("description").textValue(); + private static Map<DimensionId, String> parseDimensions(JsonParser parser, + Map<Long, Map<DimensionId, String>> uniqueDimensions) throws IOException { + List<Map.Entry<String, String>> dims = new ArrayList<>(); + int keyHash = 0; + int valueHash = 0; + for (parser.nextToken(); parser.getCurrentToken() != JsonToken.END_OBJECT; parser.nextToken()) { + String fieldName = parser.getCurrentName(); + JsonToken token = parser.nextToken(); + if (token == JsonToken.VALUE_STRING){ + String value = parser.getValueAsString(); + dims.add(Map.entry(fieldName, value)); + keyHash ^= fieldName.hashCode(); + valueHash ^= value.hashCode(); + } else if (token == JsonToken.VALUE_NULL) { + // TODO Should log a warning if this happens + } else { + throw new IllegalArgumentException("Dimension '" + fieldName + "' must be a string"); + } } - - Map<DimensionId, String> dim = Collections.emptyMap(); - if (metric.has("dimensions")) { - JsonNode dimensions = metric.get("dimensions"); - StringBuilder sb = new StringBuilder(); - for (Iterator<?> it = dimensions.fieldNames(); it.hasNext(); ) { - String k = (String) it.next(); - String v = dimensions.get(k).asText(); - sb.append(toDimensionId(k)).append(v); + Long uniqueKey = (((long) keyHash) << 32) | (valueHash & 0xffffffffL); + return uniqueDimensions.computeIfAbsent(uniqueKey, key -> dims.stream().collect(Collectors.toUnmodifiableMap(e -> toDimensionId(e.getKey()), Map.Entry::getValue))); + } + private static List<Map.Entry<String, Number>> parseValues(String prefix, JsonParser parser) throws IOException { + List<Map.Entry<String, Number>> metrics = new ArrayList<>(); + for (parser.nextToken(); parser.getCurrentToken() != JsonToken.END_OBJECT; parser.nextToken()) { + String fieldName = parser.getCurrentName(); + JsonToken token = parser.nextToken(); + String metricName = prefix + fieldName; + if (token == JsonToken.VALUE_NUMBER_INT) { + metrics.add(Map.entry(metricName, parser.getLongValue())); + } else if (token == JsonToken.VALUE_NUMBER_FLOAT) { + metrics.add(Map.entry(metricName, parser.getValueAsDouble())); + } else { + throw new IllegalArgumentException("Value for aggregator '" + fieldName + "' is not a number"); } - if ( ! uniqueDimensions.containsKey(sb.toString())) { - dim = new HashMap<>(); - for (Iterator<?> it = dimensions.fieldNames(); it.hasNext(); ) { - String k = (String) it.next(); - String v = dimensions.get(k).textValue(); - dim.put(toDimensionId(k), v); + } + return metrics; + } + static private void handleValue(JsonParser parser, Instant timestamp, Consumer consumer, + Map<Long, Map<DimensionId, String>> uniqueDimensions) throws IOException { + String name = ""; + String description = ""; + Map<DimensionId, String> dim = Map.of(); + List<Map.Entry<String, Number>> values = List.of(); + for (parser.nextToken(); parser.getCurrentToken() != JsonToken.END_OBJECT; parser.nextToken()) { + String fieldName = parser.getCurrentName(); + JsonToken token = parser.nextToken(); + if (fieldName.equals("name")) { + name = parser.getText(); + } else if (fieldName.equals("description")) { + description = parser.getText(); + } else if (fieldName.equals("dimensions")) { + dim = parseDimensions(parser, uniqueDimensions); + } else if (fieldName.equals("values")) { + values = parseValues(name+".", parser); + } else { + if (token == JsonToken.START_OBJECT || token == JsonToken.START_ARRAY) { + parser.skipChildren(); } - uniqueDimensions.put(sb.toString(), Collections.unmodifiableMap(dim)); } - dim = uniqueDimensions.get(sb.toString()); } - - JsonNode aggregates = metric.get("values"); - String prefix = name + "."; - for (Iterator<?> it = aggregates.fieldNames(); it.hasNext(); ) { - String aggregator = (String) it.next(); - JsonNode aggregatorValue = aggregates.get(aggregator); - if (aggregatorValue == null) { - throw new IllegalArgumentException("Value for aggregator '" + aggregator + "' is missing"); - } - Number value = aggregatorValue.numberValue(); - if (value == null) { - throw new IllegalArgumentException("Value for aggregator '" + aggregator + "' is not a number"); - } - String metricName = prefix + aggregator; - consumer.consume(new Metric(MetricId.toMetricId(metricName), value, timestamp, dim, description)); + for (Map.Entry<String, Number> value : values) { + consumer.consume(new Metric(MetricId.toMetricId(value.getKey()), value.getValue(), timestamp, dim, description)); } } } diff --git a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/service/RemoteMetricsFetcher.java b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/service/RemoteMetricsFetcher.java index f375fd1586f..5ca9e6fd950 100644 --- a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/service/RemoteMetricsFetcher.java +++ b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/service/RemoteMetricsFetcher.java @@ -28,12 +28,8 @@ public class RemoteMetricsFetcher extends HttpMetricFetcher { } } - void createMetrics(String data, MetricsParser.Consumer consumer, int fetchCount) { - try { - MetricsParser.parse(data, consumer); - } catch (Exception e) { - handleException(e, data, fetchCount); - } + void createMetrics(String data, MetricsParser.Consumer consumer, int fetchCount) throws IOException { + MetricsParser.parse(data, consumer); } private void createMetrics(InputStream data, MetricsParser.Consumer consumer, int fetchCount) throws IOException { try { diff --git a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/service/SystemPoller.java b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/service/SystemPoller.java index 27f86b0d503..d159fdd3dab 100644 --- a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/service/SystemPoller.java +++ b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/service/SystemPoller.java @@ -5,6 +5,7 @@ import ai.vespa.metricsproxy.metric.Metric; import ai.vespa.metricsproxy.metric.Metrics; import ai.vespa.metricsproxy.metric.model.MetricId; +import java.io.Reader; import java.time.Duration; import java.time.Instant; import java.util.HashMap; @@ -106,7 +107,6 @@ public class SystemPoller { * @return array[0] = memoryResident, array[1] = memoryVirtual (kB units) */ static long[] getMemoryUsage(VespaService service) { - long[] size = new long[2]; BufferedReader br; int pid = service.getPid(); @@ -114,24 +114,33 @@ public class SystemPoller { br = new BufferedReader(new FileReader("/proc/" + pid + "/smaps")); } catch (FileNotFoundException ex) { service.setAlive(false); - return size; + return new long[2]; } - String line; try { - while ((line = br.readLine()) != null) { - String[] elems = line.split("\\s+"); - /* Memory size is given in kB - convert to bytes by multiply with 1024*/ - if (line.startsWith("Rss:")) { - size[memoryTypeResident] += Long.parseLong(elems[1]) * 1024; - } else if (line.startsWith("Size:")) { - size[memoryTypeVirtual] += Long.parseLong(elems[1]) * 1024; - } - } - - br.close(); + return getMemoryUsage(br); } catch (IOException ex) { log.log(Level.FINE, "Unable to read line from smaps file", ex); - return size; + return new long[2]; + } finally { + try { + br.close(); + } catch (IOException ex) { + log.log(Level.FINE, "Closing of smaps file failed", ex); + } + } + } + static long[] getMemoryUsage(BufferedReader br) throws IOException{ + String line; + long[] size = new long[2]; + while ((line = br.readLine()) != null) { + /* Memory size is given in kB - convert to bytes by multiply with 1024*/ + if (line.startsWith("Rss:")) { + String remain = line.substring(4).trim(); + size[memoryTypeResident] += Long.parseLong(remain.substring(0, remain.indexOf(' '))) * 1024; + } else if (line.startsWith("Size:")) { + String remain = line.substring(5).trim(); + size[memoryTypeVirtual] += Long.parseLong(remain.substring(0, remain.indexOf(' '))) * 1024; + } } return size; @@ -152,7 +161,7 @@ public class SystemPoller { log.log(Level.FINE, () -> "Monitoring system metrics for " + services.size() + " services"); boolean someAlive = services.stream().anyMatch(VespaService::isAlive); - lastTotalCpuJiffies = updateMetrics(lastTotalCpuJiffies, interval.getSeconds(), jiffiesInterface, services, lastCpuJiffiesMetrics); + lastTotalCpuJiffies = updateMetrics(lastTotalCpuJiffies, startTime, jiffiesInterface, services, lastCpuJiffiesMetrics); // If none of the services were alive, reschedule in a short time if (!someAlive) { @@ -162,7 +171,7 @@ public class SystemPoller { } } - static JiffiesAndCpus updateMetrics(JiffiesAndCpus prevTotalJiffies, long timeStamp, GetJiffies getJiffies, + static JiffiesAndCpus updateMetrics(JiffiesAndCpus prevTotalJiffies, Instant timeStamp, GetJiffies getJiffies, List<VespaService> services, Map<VespaService, Long> lastCpuJiffiesMetrics) { Map<VespaService, Long> currentServiceJiffies = new HashMap<>(); for (VespaService s : services) { diff --git a/metrics-proxy/src/test/java/ai/vespa/metricsproxy/core/MetricsManagerTest.java b/metrics-proxy/src/test/java/ai/vespa/metricsproxy/core/MetricsManagerTest.java index c02e76320f2..9e5a24ccd3c 100644 --- a/metrics-proxy/src/test/java/ai/vespa/metricsproxy/core/MetricsManagerTest.java +++ b/metrics-proxy/src/test/java/ai/vespa/metricsproxy/core/MetricsManagerTest.java @@ -34,11 +34,8 @@ import static ai.vespa.metricsproxy.metric.ExternalMetrics.ROLE_DIMENSION; import static ai.vespa.metricsproxy.metric.model.DimensionId.toDimensionId; import static ai.vespa.metricsproxy.metric.model.MetricId.toMetricId; import static ai.vespa.metricsproxy.metric.model.ServiceId.toServiceId; -import static org.hamcrest.CoreMatchers.containsString; -import static org.hamcrest.CoreMatchers.is; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNotNull; -import static org.junit.Assert.assertThat; import static org.junit.Assert.assertTrue; /** @@ -72,40 +69,40 @@ public class MetricsManagerTest { getMetricsConsumers(),getApplicationDimensions(), getNodeDimensions()); List<MetricsPacket> packets = metricsManager.getMetrics(testServices, Instant.EPOCH); - assertThat(packets.size(), is(1)); + assertEquals(1, packets.size()); assertTrue(packets.get(0).metrics().isEmpty()); - assertThat(packets.get(0).dimensions().get(toDimensionId("instance")), is(DownService.NAME)); - assertThat(packets.get(0).dimensions().get(toDimensionId("global")), is("value")); + assertEquals(DownService.NAME, packets.get(0).dimensions().get(toDimensionId("instance"))); + assertEquals("value", packets.get(0).dimensions().get(toDimensionId("global"))); } @Test public void each_service_gets_separate_metrics_packets() { List<MetricsPacket> packets = metricsManager.getMetrics(testServices, Instant.EPOCH); - assertThat(packets.size(), is(2)); + assertEquals(2, packets.size()); - assertThat(packets.get(0).dimensions().get(toDimensionId("instance")), is("dummy0")); - assertThat(packets.get(0).metrics().get(toMetricId("c.test")), is(1.0)); - assertThat(packets.get(0).metrics().get(toMetricId("val")), is(1.05)); + assertEquals("dummy0", packets.get(0).dimensions().get(toDimensionId("instance"))); + assertEquals(1, packets.get(0).metrics().get(toMetricId("c.test"))); + assertEquals(1.05, packets.get(0).metrics().get(toMetricId("val"))); - assertThat(packets.get(1).dimensions().get(toDimensionId("instance")), is("dummy1")); - assertThat(packets.get(1).metrics().get(toMetricId("c.test")), is(6.0)); - assertThat(packets.get(1).metrics().get(toMetricId("val")), is(2.35)); + assertEquals("dummy1", packets.get(1).dimensions().get(toDimensionId("instance"))); + assertEquals(6, packets.get(1).metrics().get(toMetricId("c.test"))); + assertEquals(2.35, packets.get(1).metrics().get(toMetricId("val"))); } @Test public void verify_expected_output_from_getMetricsById() { String dummy0Metrics = metricsManager.getMetricsByConfigId(SERVICE_0_ID); - assertThat(dummy0Metrics, containsString("'dummy.id.0'.val=1.050")); - assertThat(dummy0Metrics, containsString("'dummy.id.0'.c_test=1")); + assertTrue(dummy0Metrics.contains("'dummy.id.0'.val=1.050")); + assertTrue(dummy0Metrics.contains("'dummy.id.0'.c_test=1")); String dummy1Metrics = metricsManager.getMetricsByConfigId(SERVICE_1_ID); - assertThat(dummy1Metrics, containsString("'dummy.id.1'.val=2.350")); - assertThat(dummy1Metrics, containsString("'dummy.id.1'.c_test=6")); + assertTrue(dummy1Metrics.contains("'dummy.id.1'.val=2.350")); + assertTrue(dummy1Metrics.contains("'dummy.id.1'.c_test=6")); } @Test public void getServices_returns_service_types() { - assertThat(metricsManager.getAllVespaServices(), is("dummy")); + assertEquals("dummy", metricsManager.getAllVespaServices()); } @Test @@ -134,8 +131,8 @@ public class MetricsManagerTest { assertEquals(3, packets.size()); MetricsPacket systemPacket = packets.get(0); // system metrics are added before other metrics - assertThat(systemPacket.metrics().get(toMetricId("cpu")), is(1.0)); - assertThat(systemPacket.dimensions().get(toDimensionId("metrictype")), is("system")); + assertEquals(1, systemPacket.metrics().get(toMetricId("cpu"))); + assertEquals("system", systemPacket.dimensions().get(toDimensionId("metrictype"))); service0.setSystemMetrics(oldSystemMetrics); } @@ -148,7 +145,7 @@ public class MetricsManagerTest { .putMetrics(ImmutableList.of(new Metric(WHITELISTED_METRIC_ID, 0))))); List<MetricsPacket> packets = metricsManager.getMetrics(testServices, Instant.EPOCH); - assertThat(packets.size(), is(3)); + assertEquals(3, packets.size()); } @Test @@ -158,7 +155,7 @@ public class MetricsManagerTest { .putMetrics(ImmutableList.of(new Metric(toMetricId("not-whitelisted"), 0))))); List<MetricsPacket> packets = metricsManager.getMetrics(testServices, Instant.EPOCH); - assertThat(packets.size(), is(2)); + assertEquals(2, packets.size()); } @Test @@ -185,7 +182,7 @@ public class MetricsManagerTest { List<MetricsPacket> packets = metricsManager.getMetrics(testServices, Instant.EPOCH); for (MetricsPacket packet : packets) { - assertThat(packet.dimensions().get(ROLE_DIMENSION), is("role from extraMetrics")); + assertEquals("role from extraMetrics", packet.dimensions().get(ROLE_DIMENSION)); } } @@ -197,9 +194,9 @@ public class MetricsManagerTest { .putDimension(METRIC_TYPE_DIMENSION_ID, "from extraMetrics"))); List<MetricsPacket> packets = metricsManager.getMetrics(testServices, Instant.EPOCH); - assertThat(packets.get(0).dimensions().get(METRIC_TYPE_DIMENSION_ID), is("standard")); - assertThat(packets.get(1).dimensions().get(METRIC_TYPE_DIMENSION_ID), is("standard")); - assertThat(packets.get(2).dimensions().get(METRIC_TYPE_DIMENSION_ID), is("from extraMetrics")); + assertEquals("standard", packets.get(0).dimensions().get(METRIC_TYPE_DIMENSION_ID)); + assertEquals("standard", packets.get(1).dimensions().get(METRIC_TYPE_DIMENSION_ID)); + assertEquals("from extraMetrics", packets.get(2).dimensions().get(METRIC_TYPE_DIMENSION_ID)); } @Test diff --git a/metrics-proxy/src/test/java/ai/vespa/metricsproxy/metric/MetricsTest.java b/metrics-proxy/src/test/java/ai/vespa/metricsproxy/metric/MetricsTest.java index 1cd0281b7cd..46cc9b7b7fd 100644 --- a/metrics-proxy/src/test/java/ai/vespa/metricsproxy/metric/MetricsTest.java +++ b/metrics-proxy/src/test/java/ai/vespa/metricsproxy/metric/MetricsTest.java @@ -9,15 +9,15 @@ import org.junit.Test; import java.util.HashMap; import java.util.Map; import static ai.vespa.metricsproxy.metric.model.MetricId.toMetricId; - -import static org.hamcrest.CoreMatchers.is; -import static org.junit.Assert.assertThat; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; /** * @author Unknowm */ public class MetricsTest { - + private static final double EPSILON = 0.00000000001; @Test public void testIterator() { Metrics m = new Metrics(); @@ -33,36 +33,36 @@ public class MetricsTest { for (Metric metric: m.list()) { String k = metric.getName().id; - assertThat(map.containsKey(k), is(false)); + assertFalse(map.containsKey(k)); map.put(k, metric.getValue()); } - assertThat(map.get("a").intValue(), is(1)); - assertThat(map.get("b").doubleValue(), is(2.5)); + assertEquals(1, map.get("a").intValue()); + assertEquals(2.5, map.get("b").doubleValue(), EPSILON); } @Test public void testBasicMetric() { Metrics m = new Metrics(); m.add(new Metric(toMetricId("count"), 1, System.currentTimeMillis() / 1000)); - assertThat(m.list().size(), is(1)); - assertThat(m.list().get(0).getName(), is(toMetricId("count"))); + assertEquals(1, m.list().size()); + assertEquals(toMetricId("count"), m.list().get(0).getName()); } @Test public void testHealthMetric() { HealthMetric m = HealthMetric.get(null, null); - assertThat(m.isOk(), is(false)); + assertFalse(m.isOk()); m = HealthMetric.get("up", "test message"); - assertThat(m.isOk(), is(true)); - assertThat(m.getMessage(), is("test message")); + assertTrue(m.isOk()); + assertEquals("test message", m.getMessage()); m = HealthMetric.get("ok", "test message"); - assertThat(m.isOk(), is(true)); - assertThat(m.getMessage(), is("test message")); + assertTrue(m.isOk()); + assertEquals("test message", m.getMessage()); m = HealthMetric.get("bad", "test message"); - assertThat(m.isOk(), is(false)); - assertThat(m.getStatus(), is(StatusCode.UNKNOWN)); + assertFalse(m.isOk()); + assertEquals(StatusCode.UNKNOWN, m.getStatus()); } @Test @@ -70,30 +70,30 @@ public class MetricsTest { MetricsFormatter formatter = new MetricsFormatter(false, false); VespaService service = new DummyService(0, "config.id"); String data = formatter.format(service, "key", 1); - assertThat(data, is("'config.id'.key=1")); + assertEquals("'config.id'.key=1", data); formatter = new MetricsFormatter(true, false); data = formatter.format(service, "key", 1); - assertThat(data, is("dummy.'config.id'.key=1")); + assertEquals("dummy.'config.id'.key=1", data); formatter = new MetricsFormatter(true, true); data = formatter.format(service, "key", 1); - assertThat(data, is("dummy.config.'id'.key=1")); + assertEquals("dummy.config.'id'.key=1", data); formatter = new MetricsFormatter(false, true); data = formatter.format(service, "key", 1); - assertThat(data, is("config.'id'.key=1")); + assertEquals("config.'id'.key=1", data); } @Test public void testTimeAdjustment() { - assertThat(Metric.adjustTime(0L, 0L), is(0L)); - assertThat(Metric.adjustTime(59L, 59L), is(59L)); - assertThat(Metric.adjustTime(60L, 60L), is(60L)); - assertThat(Metric.adjustTime(59L, 60L), is(60L)); - assertThat(Metric.adjustTime(60L, 59L), is(60L)); - assertThat(Metric.adjustTime(59L, 61L), is(59L)); + assertEquals(0L, Metric.adjustTime(0L, 0L)); + assertEquals(59L, Metric.adjustTime(59L, 59L)); + assertEquals(60L, Metric.adjustTime(60L, 60L)); + assertEquals(60L, Metric.adjustTime(59L, 60L)); + assertEquals(60L, Metric.adjustTime(60L, 59L)); + assertEquals(59L, Metric.adjustTime(59L, 61L)); } } diff --git a/metrics-proxy/src/test/java/ai/vespa/metricsproxy/metric/model/json/GenericJsonModelTest.java b/metrics-proxy/src/test/java/ai/vespa/metricsproxy/metric/model/json/GenericJsonModelTest.java index 5a9bc9ee725..5ddd10fd03c 100644 --- a/metrics-proxy/src/test/java/ai/vespa/metricsproxy/metric/model/json/GenericJsonModelTest.java +++ b/metrics-proxy/src/test/java/ai/vespa/metricsproxy/metric/model/json/GenericJsonModelTest.java @@ -143,4 +143,21 @@ public class GenericJsonModelTest { return mapper.readValue(getFileContents(filename), GenericJsonModel.class); } + @Test + public void within_long_range_as_long_if_possible() { + assertEquals("7", JacksonUtil.format(7D)); + assertEquals("7.1", JacksonUtil.format(7.1)); + assertEquals("-7", JacksonUtil.format(-7D)); + assertEquals("-7.1", JacksonUtil.format(-7.1)); + } + + @Test + public void outside_long_range_as_decimal_if_possible() { + double within = Long.MAX_VALUE; + double outside = 3 * within; + assertEquals("9223372036854776000", JacksonUtil.format(within)); + assertEquals("-9223372036854776000", JacksonUtil.format(-within)); + assertEquals("27670116110564327000.0", JacksonUtil.format(outside)); + assertEquals("-27670116110564327000.0", JacksonUtil.format(-outside)); + } } diff --git a/metrics-proxy/src/test/java/ai/vespa/metricsproxy/metric/model/json/YamasJsonModelTest.java b/metrics-proxy/src/test/java/ai/vespa/metricsproxy/metric/model/json/YamasJsonModelTest.java index 540445fba5b..a5bc7a0877a 100644 --- a/metrics-proxy/src/test/java/ai/vespa/metricsproxy/metric/model/json/YamasJsonModelTest.java +++ b/metrics-proxy/src/test/java/ai/vespa/metricsproxy/metric/model/json/YamasJsonModelTest.java @@ -1,15 +1,17 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package ai.vespa.metricsproxy.metric.model.json; +import ai.vespa.metricsproxy.http.yamas.YamasResponse; import ai.vespa.metricsproxy.metric.model.MetricsPacket; import com.fasterxml.jackson.databind.ObjectMapper; import org.junit.Test; import java.io.BufferedReader; +import java.io.ByteArrayOutputStream; import java.io.IOException; import java.nio.file.Files; import java.nio.file.Paths; -import java.util.Collections; +import java.util.List; import static ai.vespa.metricsproxy.metric.model.ConsumerId.toConsumerId; import static ai.vespa.metricsproxy.metric.model.MetricId.toMetricId; @@ -32,10 +34,11 @@ public class YamasJsonModelTest { public void array_definition_creates_correct_json() throws IOException { YamasJsonModel jsonModel = getYamasJsonModel("yamas-array.json"); - YamasArrayJsonModel yamasData = new YamasArrayJsonModel(); - yamasData.add(jsonModel); - - assertEquals(EXPECTED_JSON, yamasData.serialize()); + try (ByteArrayOutputStream outputStream = new ByteArrayOutputStream()) { + YamasResponse response = new YamasResponse(200, List.of(YamasJsonUtil.toMetricsPacketBuilder(jsonModel).build())); + response.render(outputStream); + assertEquals(EXPECTED_JSON, outputStream.toString()); + } } @Test @@ -48,10 +51,11 @@ public class YamasJsonModelTest { assertEquals(5.555555555E9, jsonModel.metrics.get("memory_rss"), 0.1d); //Not using custom double renderer // Serialize and verify - YamasArrayJsonModel yamasArray = new YamasArrayJsonModel(); - yamasArray.add(jsonModel); - String string = yamasArray.serialize(); - assertEquals(EXPECTED_JSON, string); + try (ByteArrayOutputStream outputStream = new ByteArrayOutputStream()) { + YamasResponse response = new YamasResponse(200, List.of(YamasJsonUtil.toMetricsPacketBuilder(jsonModel).build())); + response.render(outputStream); + assertEquals(EXPECTED_JSON, outputStream.toString()); + } } @Test @@ -65,8 +69,7 @@ public class YamasJsonModelTest { assertEquals(5.555555555E9, metricsPacket.metrics().get(toMetricId("memory_rss")).doubleValue(), 0.1d); //Not using custom double rendrer // Serialize and verify - YamasArrayJsonModel yamasArray = YamasJsonUtil.toYamasArray(Collections.singleton(metricsPacket), true); - String string = yamasArray.serialize(); + String string = YamasJsonUtil.toJson(List.of(metricsPacket), true); assertEquals(EXPECTED_JSON, string); } diff --git a/metrics-proxy/src/test/java/ai/vespa/metricsproxy/metric/model/json/YamasJsonUtilTest.java b/metrics-proxy/src/test/java/ai/vespa/metricsproxy/metric/model/json/YamasJsonUtilTest.java index 1a0d5c3e149..ebd80b38a42 100644 --- a/metrics-proxy/src/test/java/ai/vespa/metricsproxy/metric/model/json/YamasJsonUtilTest.java +++ b/metrics-proxy/src/test/java/ai/vespa/metricsproxy/metric/model/json/YamasJsonUtilTest.java @@ -2,8 +2,12 @@ package ai.vespa.metricsproxy.metric.model.json; import ai.vespa.metricsproxy.metric.model.MetricsPacket; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.node.ArrayNode; import org.junit.Test; +import java.io.IOException; import java.util.List; import java.util.Set; @@ -12,69 +16,75 @@ import static ai.vespa.metricsproxy.http.ValuesFetcher.defaultMetricsConsumerId; import static ai.vespa.metricsproxy.metric.model.ServiceId.toServiceId; import static ai.vespa.metricsproxy.metric.model.json.YamasJsonUtil.YAMAS_ROUTING; import static ai.vespa.metricsproxy.metric.model.json.YamasJsonUtil.toMetricsPackets; -import static java.util.Collections.singleton; import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertNotNull; -import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; /** * @author gjoranv */ public class YamasJsonUtilTest { + private static final ObjectMapper jsonMapper = new ObjectMapper(); + private static JsonNode metrics(MetricsPacket packet, boolean addStatus) throws IOException { + return metrics(List.of(packet), addStatus).get(0); + } + private static ArrayNode metrics(List<MetricsPacket> packets, boolean addStatus) throws IOException { + return (ArrayNode) jsonMapper.readTree(YamasJsonUtil.toJson(packets, addStatus)).get("metrics"); + } @Test - public void json_model_gets_null_status_by_default() { - MetricsPacket packet = new MetricsPacket.Builder(toServiceId("foo")) - .build(); - YamasJsonModel jsonModel = YamasJsonUtil.toYamasArray(singleton(packet)).metrics.get(0); - assertNull(jsonModel.status_code); - assertNull(jsonModel.status_msg); + public void json_model_gets_null_status_by_default() throws IOException { + ArrayNode json = metrics(List.of(new MetricsPacket.Builder(toServiceId("foo")).build(), + new MetricsPacket.Builder(toServiceId("bar")).build()), false); + assertFalse(json.get(0).has("status_code")); + assertFalse(json.get(0).has("status_msg")); + assertTrue(json.get(1).has("status_code")); + assertTrue(json.get(1).has("status_msg")); } @Test - public void status_is_included_in_json_model_when_explicitly_asked_for() { - MetricsPacket packet = new MetricsPacket.Builder(toServiceId("foo")) - .build(); - YamasJsonModel jsonModel = YamasJsonUtil.toYamasArray(singleton(packet), true).metrics.get(0); - assertNotNull(jsonModel.status_code); - assertNotNull(jsonModel.status_msg); + public void status_is_included_in_json_model_when_explicitly_asked_for() throws IOException { + ArrayNode json = metrics(List.of(new MetricsPacket.Builder(toServiceId("foo")).build(), + new MetricsPacket.Builder(toServiceId("bar")).build()), true); + assertTrue(json.get(0).has("status_code")); + assertTrue(json.get(0).has("status_msg")); + assertTrue(json.get(1).has("status_code")); + assertTrue(json.get(1).has("status_msg")); } @Test - public void timestamp_0_in_packet_is_translated_to_null_in_json_model() { - MetricsPacket packet = new MetricsPacket.Builder(toServiceId("foo")) - .timestamp(0L) - .build(); - YamasJsonModel jsonModel = YamasJsonUtil.toYamasArray(singleton(packet)).metrics.get(0); - assertNull(jsonModel.timestamp); + public void timestamp_0_in_packet_is_translated_to_null_in_json_model() throws IOException { + MetricsPacket packet = new MetricsPacket.Builder(toServiceId("foo")).timestamp(0L).build(); + JsonNode json = metrics(packet, true); + assertFalse(json.has("timestamp")); } @Test - public void empty_consumers_is_translated_to_null_routing_in_json_model() { - MetricsPacket packet = new MetricsPacket.Builder(toServiceId("foo")) - .build(); - YamasJsonModel jsonModel = YamasJsonUtil.toYamasArray(singleton(packet)).metrics.get(0); - assertNull(jsonModel.routing); + public void empty_consumers_is_translated_to_null_routing_in_json_model() throws IOException { + MetricsPacket packet = new MetricsPacket.Builder(toServiceId("foo")).build(); + JsonNode json = metrics(packet, true); + assertFalse(json.has("routing")); } @Test - public void default_public_consumer_is_filtered_from_yamas_routing() { + public void default_public_consumer_is_filtered_from_yamas_routing() throws IOException { MetricsPacket packet = new MetricsPacket.Builder(toServiceId("foo")) .addConsumers(Set.of(vespaMetricsConsumerId, defaultMetricsConsumerId)) .build(); - YamasJsonModel jsonModel = YamasJsonUtil.toYamasArray(singleton(packet)).metrics.get(0); - List<String> namespaces = jsonModel.routing.get(YAMAS_ROUTING).namespaces; + JsonNode json = metrics(packet, false); + JsonNode routing = json.get("routing"); + JsonNode yamas = routing.get(YAMAS_ROUTING); + ArrayNode namespaces = (ArrayNode) yamas.get("namespaces"); assertEquals(1, namespaces.size()); - assertEquals(vespaMetricsConsumerId.id, namespaces.get(0)); + assertEquals(vespaMetricsConsumerId.id, namespaces.get(0).asText()); } @Test - public void only_default_public_consumer_yields_null_routing_in_json_model() { + public void only_default_public_consumer_yields_null_routing_in_json_model() throws IOException { MetricsPacket packet = new MetricsPacket.Builder(toServiceId("foo")) .addConsumers(Set.of(defaultMetricsConsumerId)) .build(); - YamasJsonModel jsonModel = YamasJsonUtil.toYamasArray(singleton(packet)).metrics.get(0); - assertNull(jsonModel.routing); + JsonNode json = metrics(packet, false); + assertFalse(json.has(YAMAS_ROUTING)); } @Test diff --git a/metrics-proxy/src/test/java/ai/vespa/metricsproxy/rpc/RpcHealthMetricsTest.java b/metrics-proxy/src/test/java/ai/vespa/metricsproxy/rpc/RpcHealthMetricsTest.java index 268eab9f5d3..98fa337fa78 100644 --- a/metrics-proxy/src/test/java/ai/vespa/metricsproxy/rpc/RpcHealthMetricsTest.java +++ b/metrics-proxy/src/test/java/ai/vespa/metricsproxy/rpc/RpcHealthMetricsTest.java @@ -16,9 +16,10 @@ import java.util.List; import static ai.vespa.metricsproxy.TestUtil.getFileContents; import static ai.vespa.metricsproxy.rpc.IntegrationTester.SERVICE_1_CONFIG_ID; -import static org.hamcrest.CoreMatchers.is; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertNotNull; -import static org.junit.Assert.assertThat; +import static org.junit.Assert.assertTrue; /** * @author jobergum @@ -40,21 +41,21 @@ public class RpcHealthMetricsTest { mockHttpServer.setResponse(HEALTH_OK_RESPONSE); List<VespaService> services = tester.vespaServices().getInstancesById(SERVICE_1_CONFIG_ID); - assertThat(services.size(), is(1)); + assertEquals(1, services.size()); VespaService qrserver = services.get(0); HealthMetric h = qrserver.getHealth(); assertNotNull("Health metric should never be null", h); - assertThat("Status failed, reason = " + h.getMessage(), h.isOk(), is(true)); - assertThat(h.getMessage(), is("WORKING")); + assertTrue("Status failed, reason = " + h.getMessage(), h.isOk()); + assertEquals("WORKING", h.getMessage()); mockHttpServer.setResponse(HEALTH_FAILED_RESPONSE); h = qrserver.getHealth(); assertNotNull("Health metric should never be null", h); - assertThat("Status should be failed" + h.getMessage(), h.isOk(), is(false)); - assertThat(h.getMessage(), is("SOMETHING FAILED")); + assertFalse("Status should be failed" + h.getMessage(), h.isOk()); + assertEquals("SOMETHING FAILED", h.getMessage()); String jsonRPCMessage = getHealthMetrics(tester, qrserver.getMonitoringName().id); - assertThat(jsonRPCMessage, is(WANTED_RPC_RESPONSE)); + assertEquals(WANTED_RPC_RESPONSE, jsonRPCMessage); } } @@ -62,7 +63,7 @@ public class RpcHealthMetricsTest { public void non_existent_service_name_returns_an_error_message() { try (IntegrationTester tester = new IntegrationTester()) { String jsonRPCMessage = getHealthMetrics(tester, "non-existing service"); - assertThat(jsonRPCMessage, is("105: No service with name 'non-existing service'")); + assertEquals("105: No service with name 'non-existing service'", jsonRPCMessage); } } diff --git a/metrics-proxy/src/test/java/ai/vespa/metricsproxy/rpc/RpcMetricsTest.java b/metrics-proxy/src/test/java/ai/vespa/metricsproxy/rpc/RpcMetricsTest.java index 214df36d6a1..beb0d3fce60 100644 --- a/metrics-proxy/src/test/java/ai/vespa/metricsproxy/rpc/RpcMetricsTest.java +++ b/metrics-proxy/src/test/java/ai/vespa/metricsproxy/rpc/RpcMetricsTest.java @@ -28,12 +28,9 @@ import static ai.vespa.metricsproxy.rpc.IntegrationTester.MONITORING_SYSTEM; import static ai.vespa.metricsproxy.rpc.IntegrationTester.SERVICE_1_CONFIG_ID; import static ai.vespa.metricsproxy.rpc.IntegrationTester.SERVICE_2_CONFIG_ID; import static ai.vespa.metricsproxy.service.VespaServices.ALL_SERVICES; -import static org.hamcrest.CoreMatchers.is; -import static org.hamcrest.CoreMatchers.notNullValue; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertNotNull; -import static org.junit.Assert.assertThat; import static org.junit.Assert.assertTrue; /** @@ -72,8 +69,8 @@ public class RpcMetricsTest { // Verify that application is used as serviceId, and that metric exists. JsonNode extraMetrics = findExtraMetricsObject(allServicesResponse); - assertThat(extraMetrics.get("metrics").get("foo.count").intValue(), is(3)); - assertThat(extraMetrics.get("dimensions").get("role").textValue(), is("extra-role")); + assertEquals(3, extraMetrics.get("metrics").get("foo.count").intValue()); + assertEquals("extra-role", extraMetrics.get("dimensions").get("role").textValue()); } } } @@ -101,13 +98,13 @@ public class RpcMetricsTest { tester.httpServer().setResponse(METRICS_RESPONSE); List<VespaService> services = tester.vespaServices().getInstancesById(SERVICE_1_CONFIG_ID); - assertThat("#Services should be 1 for config id " + SERVICE_1_CONFIG_ID, services.size(), is(1)); + assertEquals("#Services should be 1 for config id " + SERVICE_1_CONFIG_ID, 1, services.size()); VespaService qrserver = services.get(0); - assertThat(qrserver.getMonitoringName().id, is(MONITORING_SYSTEM + VespaService.SEPARATOR + "qrserver")); + assertEquals(MONITORING_SYSTEM + VespaService.SEPARATOR + "qrserver", qrserver.getMonitoringName().id); Metrics metrics = qrserver.getMetrics(); - assertThat("Fetched number of metrics is not correct", metrics.size(), is(2)); + assertEquals("Fetched number of metrics is not correct", 2, metrics.size()); Metric m = getMetric("foo.count", metrics); assertNotNull("Did not find expected metric with name 'foo.count'", m); Metric m2 = getMetric("bar.count", metrics); @@ -117,16 +114,16 @@ public class RpcMetricsTest { verifyMetricsFromRpcRequest(qrserver, rpcClient); services = tester.vespaServices().getInstancesById(SERVICE_2_CONFIG_ID); - assertThat("#Services should be 1 for config id " + SERVICE_2_CONFIG_ID, services.size(), is(1)); + assertEquals("#Services should be 1 for config id " + SERVICE_2_CONFIG_ID, 1, services.size()); VespaService storageService = services.get(0); verfiyMetricsFromServiceObject(storageService); String metricsById = getMetricsById(storageService.getConfigId(), rpcClient); - assertThat(metricsById, is("'storage.cluster.storage.storage.0'.foo_count=1 ")); + assertEquals("'storage.cluster.storage.storage.0'.foo_count=1 ", metricsById); String jsonResponse = getMetricsForYamas("non-existing", rpcClient).trim(); - assertThat(jsonResponse, is("105: No service with name 'non-existing'")); + assertEquals("105: No service with name 'non-existing'", jsonResponse); verifyMetricsFromRpcRequestForAllServices(rpcClient); @@ -145,21 +142,21 @@ public class RpcMetricsTest { private static void verifyMetricsFromRpcRequest(VespaService service, RpcClient client) throws IOException { String jsonResponse = getMetricsForYamas(service.getMonitoringName().id, client).trim(); ArrayNode metrics = (ArrayNode) jsonMapper.readTree(jsonResponse).get("metrics"); - assertThat("Expected 3 metric messages", metrics.size(), is(3)); + assertEquals("Expected 3 metric messages", 3, metrics.size()); for (int i = 0; i < metrics.size() - 1; i++) { // The last "metric message" contains only status code/message JsonNode jsonObject = metrics.get(i); assertFalse(jsonObject.has("status_code")); assertFalse(jsonObject.has("status_msg")); - assertThat(jsonObject.get("dimensions").get("foo").textValue(), is("bar")); - assertThat(jsonObject.get("dimensions").get("bar").textValue(), is("foo")); - assertThat(jsonObject.get("dimensions").get("serviceDim").textValue(), is("serviceDimValue")); - assertThat(jsonObject.get("routing").get("yamas").get("namespaces").size(), is(1)); + assertEquals("bar", jsonObject.get("dimensions").get("foo").textValue()); + assertEquals("foo", jsonObject.get("dimensions").get("bar").textValue()); + assertEquals("serviceDimValue", jsonObject.get("dimensions").get("serviceDim").textValue()); + assertEquals(1, jsonObject.get("routing").get("yamas").get("namespaces").size()); if (jsonObject.get("metrics").has("foo_count")) { - assertThat(jsonObject.get("metrics").get("foo_count").intValue(), is(1)); - assertThat(jsonObject.get("routing").get("yamas").get("namespaces").get(0).textValue(), is(vespaMetricsConsumerId.id)); + assertEquals(1, jsonObject.get("metrics").get("foo_count").intValue()); + assertEquals(vespaMetricsConsumerId.id, jsonObject.get("routing").get("yamas").get("namespaces").get(0).textValue()); } else { - assertThat(jsonObject.get("metrics").get("foo.count").intValue(), is(1)); - assertThat(jsonObject.get("routing").get("yamas").get("namespaces").get(0).textValue(), is(CUSTOM_CONSUMER_ID.id)); + assertEquals(1, jsonObject.get("metrics").get("foo.count").intValue()); + assertEquals(CUSTOM_CONSUMER_ID.id, jsonObject.get("routing").get("yamas").get("namespaces").get(0).textValue()); } } @@ -168,21 +165,21 @@ public class RpcMetricsTest { private void verfiyMetricsFromServiceObject(VespaService service) { Metrics storageMetrics = service.getMetrics(); - assertThat(storageMetrics.size(), is(2)); + assertEquals(2, storageMetrics.size()); Metric foo = getMetric("foo.count", storageMetrics); assertNotNull("Did not find expected metric with name 'foo.count'", foo); - assertThat("Expected 2 dimensions for metric foo", foo.getDimensions().size(), is(2)); - assertThat("Metric foo did not contain correct dimension mapping for key = foo.count", foo.getDimensions().containsKey(toDimensionId("foo")), is(true)); - assertThat("Metric foo did not contain correct dimension", foo.getDimensions().get(toDimensionId("foo")), is("bar")); - assertThat("Metric foo did not contain correct dimension", foo.getDimensions().containsKey(toDimensionId("bar")), is(true)); - assertThat("Metric foo did not contain correct dimension for key = bar", foo.getDimensions().get(toDimensionId("bar")), is("foo")); + assertEquals("Expected 2 dimensions for metric foo", 2, foo.getDimensions().size()); + assertTrue("Metric foo did not contain correct dimension mapping for key = foo.count", foo.getDimensions().containsKey(toDimensionId("foo"))); + assertEquals("Metric foo did not contain correct dimension", "bar", foo.getDimensions().get(toDimensionId("foo"))); + assertTrue("Metric foo did not contain correct dimension", foo.getDimensions().containsKey(toDimensionId("bar"))); + assertEquals("Metric foo did not contain correct dimension for key = bar", "foo", foo.getDimensions().get(toDimensionId("bar"))); } private void verifyMetricsFromRpcRequestForAllServices(RpcClient client) throws IOException { // Verify that metrics for all services can be retrieved in one request. String allServicesResponse = getMetricsForYamas(ALL_SERVICES, client).trim(); ArrayNode allServicesMetrics = (ArrayNode) jsonMapper.readTree(allServicesResponse).get("metrics"); - assertThat(allServicesMetrics.size(), is(5)); + assertEquals(5, allServicesMetrics.size()); } @Test @@ -192,9 +189,9 @@ public class RpcMetricsTest { tester.httpServer().setResponse(METRICS_RESPONSE); List<VespaService> services = tester.vespaServices().getInstancesById(SERVICE_1_CONFIG_ID); - assertThat(services.size(), is(1)); + assertEquals(1, services.size()); Metrics metrics = services.get(0).getMetrics(); - assertThat("Fetched number of metrics is not correct", metrics.size(), is(2)); + assertEquals("Fetched number of metrics is not correct", 2, metrics.size()); Metric m = getMetric("foo.count", metrics); assertNotNull("Did not find expected metric with name 'foo.count'", m); @@ -203,7 +200,7 @@ public class RpcMetricsTest { try (RpcClient rpcClient = new RpcClient(tester.rpcPort())) { String response = getAllMetricNamesForService(services.get(0).getMonitoringName().id, vespaMetricsConsumerId, rpcClient); - assertThat(response, is("foo.count=ON;output-name=foo_count,bar.count=OFF,")); + assertEquals("foo.count=ON;output-name=foo_count,bar.count=OFF,", response); } } } @@ -263,11 +260,11 @@ public class RpcMetricsTest { } private static void verifyStatusMessage(JsonNode jsonObject) { - assertThat(jsonObject.get("status_code").intValue(), is(0)); - assertThat(jsonObject.get("status_msg").textValue(), notNullValue()); - assertThat(jsonObject.get("application").textValue(), notNullValue()); - assertThat(jsonObject.get("routing"), notNullValue()); - assertThat(jsonObject.size(), is(4)); + assertEquals(0, jsonObject.get("status_code").intValue()); + assertNotNull(jsonObject.get("status_msg")); + assertNotNull(jsonObject.get("application")); + assertNotNull(jsonObject.get("routing")); + assertEquals(4, jsonObject.size()); } } diff --git a/metrics-proxy/src/test/java/ai/vespa/metricsproxy/service/ConfigSentinelClientTest.java b/metrics-proxy/src/test/java/ai/vespa/metricsproxy/service/ConfigSentinelClientTest.java index 6d009d2fd88..0ace697d545 100644 --- a/metrics-proxy/src/test/java/ai/vespa/metricsproxy/service/ConfigSentinelClientTest.java +++ b/metrics-proxy/src/test/java/ai/vespa/metricsproxy/service/ConfigSentinelClientTest.java @@ -6,8 +6,10 @@ import org.junit.Test; import java.util.ArrayList; import java.util.List; -import static org.hamcrest.CoreMatchers.is; -import static org.junit.Assert.assertThat; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.assertEquals; + /** * @author Unknown @@ -29,35 +31,35 @@ public class ConfigSentinelClientTest { try (MockConfigSentinelClient client = new MockConfigSentinelClient(configsentinel)) { client.updateServiceStatuses(services); - assertThat(qrserver.getPid(), is(6520)); - assertThat(qrserver.getState(), is("RUNNING")); - assertThat(qrserver.isAlive(), is(true)); - assertThat(searchnode4.getPid(), is(6534)); - assertThat(searchnode4.getState(), is("RUNNING")); - assertThat(searchnode4.isAlive(), is(true)); + assertEquals(6520, qrserver.getPid()); + assertEquals("RUNNING", qrserver.getState()); + assertTrue(qrserver.isAlive()); + assertEquals(6534, searchnode4.getPid()); + assertEquals("RUNNING", searchnode4.getState()); + assertTrue(searchnode4.isAlive()); - assertThat(docproc.getPid(), is(-1)); - assertThat(docproc.getState(), is("FINISHED")); - assertThat(docproc.isAlive(), is(false)); + assertEquals(-1, docproc.getPid()); + assertEquals("FINISHED", docproc.getState()); + assertFalse(docproc.isAlive()); configsentinel.reConfigure(); client.ping(docproc); - assertThat(docproc.getPid(), is(100)); - assertThat(docproc.getState(), is("RUNNING")); - assertThat(docproc.isAlive(), is(true)); + assertEquals(100, docproc.getPid()); + assertEquals("RUNNING", docproc.getState()); + assertTrue(docproc.isAlive()); //qrserver has yet not been checked - assertThat(qrserver.isAlive(), is(true)); + assertTrue(qrserver.isAlive()); client.updateServiceStatuses(services); - assertThat(docproc.getPid(), is(100)); - assertThat(docproc.getState(), is("RUNNING")); - assertThat(docproc.isAlive(), is(true)); + assertEquals(100, docproc.getPid()); + assertEquals("RUNNING", docproc.getState()); + assertTrue(docproc.isAlive()); //qrserver is no longer running on this node - so should be false - assertThat(qrserver.isAlive(), is(false)); + assertFalse(qrserver.isAlive()); } } @@ -90,13 +92,13 @@ public class ConfigSentinelClientTest { try (MockConfigSentinelClient client = new MockConfigSentinelClient(configsentinel)) { client.updateServiceStatuses(services); - assertThat(container.isAlive(),is(true)); - assertThat(container.getPid(),is(14338)); - assertThat(container.getState(),is("RUNNING")); + assertTrue(container.isAlive()); + assertEquals(14338, container.getPid()); + assertEquals("RUNNING", container.getState()); - assertThat(containerClusterController.isAlive(),is(true)); - assertThat(containerClusterController.getPid(),is(25020)); - assertThat(containerClusterController.getState(),is("RUNNING")); + assertTrue(containerClusterController.isAlive()); + assertEquals(25020, containerClusterController.getPid()); + assertEquals("RUNNING", containerClusterController.getState()); } } diff --git a/metrics-proxy/src/test/java/ai/vespa/metricsproxy/service/ContainerServiceTest.java b/metrics-proxy/src/test/java/ai/vespa/metricsproxy/service/ContainerServiceTest.java index 44f0e9608ac..79dab3338e9 100644 --- a/metrics-proxy/src/test/java/ai/vespa/metricsproxy/service/ContainerServiceTest.java +++ b/metrics-proxy/src/test/java/ai/vespa/metricsproxy/service/ContainerServiceTest.java @@ -11,8 +11,8 @@ import static ai.vespa.metricsproxy.TestUtil.getFileContents; import static ai.vespa.metricsproxy.metric.model.DimensionId.toDimensionId; import static ai.vespa.metricsproxy.metric.model.MetricId.toMetricId; import static ai.vespa.metricsproxy.service.RemoteMetricsFetcher.METRICS_PATH; -import static org.hamcrest.CoreMatchers.is; -import static org.junit.Assert.assertThat; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; /** * @author Unknown @@ -45,15 +45,15 @@ public class ContainerServiceTest { count++; System.out.println("Name: " + m.getName() + " value: " + m.getValue()); if (m.getDimensions().get(toDimensionId("chain")).equals("asvBlendingResult")) { - assertThat((Double)m.getValue(), is(26.4)); + assertEquals(26.4, m.getValue()); } else if (m.getDimensions().get(toDimensionId("chain")).equals("blendingResult")) { - assertThat((Double)m.getValue(), is(0.36666666666666664)); + assertEquals(0.36666666666666664, m.getValue()); } else { - assertThat("Unknown unknown chain", false, is(true)); + assertTrue("Unknown unknown chain", false); } } } - assertThat(count, is(2)); + assertEquals(2, count); } @After diff --git a/metrics-proxy/src/test/java/ai/vespa/metricsproxy/service/MetricsFetcherTest.java b/metrics-proxy/src/test/java/ai/vespa/metricsproxy/service/MetricsFetcherTest.java index 3af317ab36b..5d4512276d4 100644 --- a/metrics-proxy/src/test/java/ai/vespa/metricsproxy/service/MetricsFetcherTest.java +++ b/metrics-proxy/src/test/java/ai/vespa/metricsproxy/service/MetricsFetcherTest.java @@ -4,27 +4,31 @@ package ai.vespa.metricsproxy.service; import ai.vespa.metricsproxy.TestUtil; import ai.vespa.metricsproxy.metric.Metric; import ai.vespa.metricsproxy.metric.Metrics; -import ai.vespa.metricsproxy.metric.model.MetricId; import org.junit.Test; +import java.io.IOException; +import java.time.Instant; + import static ai.vespa.metricsproxy.metric.model.MetricId.toMetricId; -import static org.hamcrest.CoreMatchers.is; -import static org.junit.Assert.assertThat; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; /** */ public class MetricsFetcherTest { - private static int port = 9; //port number is not used in this test + private static final int port = 9; //port number is not used in this test - private class MetricsConsumer implements MetricsParser.Consumer { + private static class MetricsConsumer implements MetricsParser.Consumer { Metrics metrics = new Metrics(); @Override public void consume(Metric metric) { metrics.add(metric); } } - Metrics fetch(String data) { + Metrics fetch(String data) throws IOException { RemoteMetricsFetcher fetcher = new RemoteMetricsFetcher(new DummyService(0, "dummy/id/0"), port); MetricsConsumer consumer = new MetricsConsumer(); fetcher.createMetrics(data, consumer, 0); @@ -32,30 +36,59 @@ public class MetricsFetcherTest { } @Test - public void testStateFormatMetricsParse() { + public void testStateFormatMetricsParse() throws IOException { String jsonData = TestUtil.getFileContents("metrics-state.json"); Metrics metrics = fetch(jsonData); - assertThat(metrics.size(), is(10)); - assertThat(getMetric("query_hits.count", metrics).getValue().intValue(), is(28)); - assertThat(getMetric("queries.rate", metrics).getValue().doubleValue(), is(0.4667)); - assertThat(metrics.getTimeStamp(), is(1334134700L)); + assertEquals(10, metrics.size()); + assertEquals(28L, getMetric("query_hits.count", metrics).getValue()); + assertEquals(0.4667, getMetric("queries.rate", metrics).getValue()); + assertEquals(Instant.ofEpochSecond(1334134700L), metrics.getTimeStamp()); } @Test - public void testEmptyJson() { + public void testEmptyJson() throws IOException { String jsonData = "{}"; Metrics metrics = fetch(jsonData); - assertThat("Wrong number of metrics", metrics.size(), is(0)); + assertEquals(0, metrics.size()); + } + + @Test + public void testSkippingNullDimensions() throws IOException { + String jsonData = + "{\"status\" : {\"code\" : \"up\",\"message\" : \"Everything ok here\"}," + + "\"metrics\" : {\"snapshot\" : {\"from\" : 1334134640.089,\"to\" : 1334134700.088" + " }," + + "\"values\" : [" + + "{" + + " \"name\" : \"some.bogus.metric\"," + + " \"values\" : {" + + " \"count\" : 12," + + " \"rate\" : 0.2" + + " }," + + " \"dimensions\" : {" + + " \"version\" : null" + + " }" + + " }" + + "]}}"; + + Metrics metrics = fetch(jsonData); + assertEquals(2, metrics.size()); + assertTrue(metrics.list().get(0).getDimensions().isEmpty()); + assertTrue(metrics.list().get(1).getDimensions().isEmpty()); } @Test - public void testErrors() { + public void testErrors() throws IOException { String jsonData; - Metrics metrics; + Metrics metrics = null; jsonData = ""; - metrics = fetch(jsonData); - assertThat("Wrong number of metrics", metrics.size(), is(0)); + try { + metrics = fetch(jsonData); + fail("Should have an IOException instead"); + } catch (IOException e) { + assertEquals("Expected start of object, got null", e.getMessage()); + } + assertNull(metrics); jsonData = "{\n" + "\"status\" : {\n" + @@ -64,7 +97,7 @@ public class MetricsFetcherTest { "}\n" + "}"; metrics = fetch(jsonData); - assertThat("Wrong number of metrics", metrics.size(), is(0)); + assertEquals(0, metrics.size()); jsonData = "{\n" + "\"status\" : {\n" + @@ -92,8 +125,14 @@ public class MetricsFetcherTest { "}\n" + "}"; - metrics = fetch(jsonData); - assertThat("Wrong number of metrics", metrics.size(), is(0)); + metrics = null; + try { + metrics = fetch(jsonData); + fail("Should have an IOException instead"); + } catch (IllegalArgumentException e) { + assertEquals("Value for aggregator 'count' is not a number", e.getMessage()); + } + assertNull(metrics); } public Metric getMetric(String metric, Metrics metrics) { diff --git a/metrics-proxy/src/test/java/ai/vespa/metricsproxy/service/SystemPollerTest.java b/metrics-proxy/src/test/java/ai/vespa/metricsproxy/service/SystemPollerTest.java index f5ae0048275..30145746e79 100644 --- a/metrics-proxy/src/test/java/ai/vespa/metricsproxy/service/SystemPollerTest.java +++ b/metrics-proxy/src/test/java/ai/vespa/metricsproxy/service/SystemPollerTest.java @@ -4,18 +4,20 @@ package ai.vespa.metricsproxy.service; import ai.vespa.metricsproxy.metric.Metric; import ai.vespa.metricsproxy.metric.Metrics; import ai.vespa.metricsproxy.metric.model.MetricId; +import org.junit.Ignore; import org.junit.Test; import java.io.BufferedReader; +import java.io.IOException; import java.io.StringReader; +import java.time.Instant; import java.util.ArrayList; import java.util.HashMap; import java.util.List; import java.util.Map; -import static org.hamcrest.CoreMatchers.is; import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertThat; +import static org.junit.Assert.assertFalse; /** * @author Unknown @@ -71,18 +73,84 @@ public class SystemPollerTest { List<VespaService> services = new ArrayList<>(); services.add(s); - assertThat(s.isAlive(), is(false)); + assertFalse(s.isAlive()); long n = SystemPoller.getPidJiffies(s); - assertThat(n, is(0L)); + assertEquals(0L, n); long[] memusage = SystemPoller.getMemoryUsage(s); - assertThat(memusage[0], is(0L)); - assertThat(memusage[1], is(0L)); + assertEquals(0L, memusage[0]); + assertEquals(0L, memusage[1]); } + private static final String smaps = + "00400000-004de000 r-xp 00000000 fe:01 670312 /usr/bin/bash\n" + + "Size: 888 kB\n" + + "KernelPageSize: 4 kB\n" + + "MMUPageSize: 4 kB\n" + + "Rss: 824 kB\n" + + "Pss: 150 kB\n" + + "Shared_Clean: 824 kB\n" + + "Shared_Dirty: 0 kB\n" + + "Private_Clean: 0 kB\n" + + "Private_Dirty: 0 kB\n" + + "Referenced: 824 kB\n" + + "Anonymous: 0 kB\n" + + "LazyFree: 0 kB\n" + + "AnonHugePages: 0 kB\n" + + "ShmemPmdMapped: 0 kB\n" + + "FilePmdMapped: 0 kB\n" + + "Shared_Hugetlb: 0 kB\n" + + "Private_Hugetlb: 0 kB\n" + + "Swap: 0 kB\n" + + "SwapPss: 0 kB\n" + + "Locked: 0 kB\n" + + "THPeligible: 0\n" + + "VmFlags: rd ex mr mw me dw \n" + + "006dd000-006de000 r--p 000dd000 fe:01 670312 /usr/bin/bash\n" + + "Size: 4 kB\n" + + "KernelPageSize: 4 kB\n" + + "MMUPageSize: 4 kB\n" + + "Rss: 4 kB\n" + + "Pss: 4 kB\n" + + "Shared_Clean: 0 kB\n" + + "Shared_Dirty: 0 kB\n" + + "Private_Clean: 4 kB\n" + + "Private_Dirty: 0 kB\n" + + "Referenced: 4 kB\n" + + "Anonymous: 4 kB\n" + + "LazyFree: 0 kB\n" + + "AnonHugePages: 0 kB\n" + + "ShmemPmdMapped: 0 kB\n" + + "FilePmdMapped: 0 kB\n" + + "Shared_Hugetlb: 0 kB\n" + + "Private_Hugetlb: 0 kB\n" + + "Swap: 0 kB\n" + + "SwapPss: 0 kB\n" + + "Locked: 0 kB\n" + + "THPeligible: 0\n" + + "VmFlags: rd mr mw me dw ac \n"; + @Test - public - void testPerProcessJiffies() { + public void testSmapsParsing() throws IOException { + BufferedReader br = new BufferedReader(new StringReader(smaps)); + long[] memusage = SystemPoller.getMemoryUsage(br); + assertEquals(913408L, memusage[0]); + assertEquals(847872L, memusage[1]); + } + + @Ignore + @Test + public void benchmarkSmapsParsing() throws IOException { + for (int i=0; i < 100000; i++) { + BufferedReader br = new BufferedReader(new StringReader(smaps)); + long[] memusage = SystemPoller.getMemoryUsage(br); + assertEquals(913408L, memusage[0]); + assertEquals(847872L, memusage[1]); + } + } + + @Test + public void testPerProcessJiffies() { assertEquals(PER_PROC_JIFFIES[0], SystemPoller.getPidJiffies(new BufferedReader(new StringReader(perProcStats[0])))); assertEquals(PER_PROC_JIFFIES[1], SystemPoller.getPidJiffies(new BufferedReader(new StringReader(perProcStats[1])))); } @@ -109,8 +177,8 @@ public class SystemPollerTest { public void testCPUJiffies() { String line = "cpu1 102180864 789 56766899 12800020140 1654757 0 0"; CpuJiffies n = new CpuJiffies(line); - assertThat(n.getCpuId(), is(1)); - assertThat(n.getTotalJiffies(), is(12960623449L)); + assertEquals(1, n.getCpuId()); + assertEquals(12960623449L, n.getTotalJiffies()); } @Test @@ -121,7 +189,7 @@ public class SystemPollerTest { List<VespaService> services = List.of(s1); lastCpuJiffiesMetrics.put(s1, SystemPoller.getPidJiffies(new BufferedReader(new StringReader(perProcStats[0])))); - SystemPoller.JiffiesAndCpus next = SystemPoller.updateMetrics(prev, 1, + SystemPoller.JiffiesAndCpus next = SystemPoller.updateMetrics(prev, Instant.ofEpochSecond(1), new SystemPoller.GetJiffies() { @Override public SystemPoller.JiffiesAndCpus getTotalSystemJiffies() { diff --git a/metrics-proxy/src/test/java/ai/vespa/metricsproxy/service/VespaServiceTest.java b/metrics-proxy/src/test/java/ai/vespa/metricsproxy/service/VespaServiceTest.java index a54e050bac7..6c0e1865dcd 100644 --- a/metrics-proxy/src/test/java/ai/vespa/metricsproxy/service/VespaServiceTest.java +++ b/metrics-proxy/src/test/java/ai/vespa/metricsproxy/service/VespaServiceTest.java @@ -3,7 +3,6 @@ package ai.vespa.metricsproxy.service; import ai.vespa.metricsproxy.metric.Metric; import ai.vespa.metricsproxy.metric.Metrics; -import ai.vespa.metricsproxy.metric.model.MetricId; import org.junit.After; import org.junit.Before; import org.junit.Test; @@ -11,9 +10,7 @@ import org.junit.Test; import static ai.vespa.metricsproxy.TestUtil.getFileContents; import static ai.vespa.metricsproxy.metric.model.MetricId.toMetricId; import static ai.vespa.metricsproxy.service.RemoteMetricsFetcher.METRICS_PATH; -import static org.hamcrest.CoreMatchers.is; import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertThat; import static org.junit.Assert.assertTrue; /** @@ -40,17 +37,17 @@ public class VespaServiceTest { @Test public void testService() { VespaService service = new VespaService("qrserver", "container/qrserver.0"); - assertThat(service.getServiceName(), is("qrserver")); - assertThat(service.getInstanceName(), is("qrserver")); - assertThat(service.getPid(), is(-1)); - assertThat(service.getConfigId(), is("container/qrserver.0")); + assertEquals("qrserver", service.getServiceName()); + assertEquals("qrserver", service.getInstanceName()); + assertEquals(-1, service.getPid()); + assertEquals("container/qrserver.0", service.getConfigId()); service = VespaService.create("qrserver2", "container/qrserver.0", -1); - assertThat(service.getServiceName(), is("qrserver")); - assertThat(service.getInstanceName(), is("qrserver2")); - assertThat(service.getPid(), is(-1)); - assertThat(service.getConfigId(), is("container/qrserver.0")); + assertEquals("qrserver", service.getServiceName()); + assertEquals("qrserver2", service.getInstanceName()); + assertEquals(-1, service.getPid()); + assertEquals("container/qrserver.0", service.getConfigId()); } @Test diff --git a/metrics-proxy/src/test/java/ai/vespa/metricsproxy/service/VespaServicesTest.java b/metrics-proxy/src/test/java/ai/vespa/metricsproxy/service/VespaServicesTest.java index 264406a6fc6..dda8af8559b 100644 --- a/metrics-proxy/src/test/java/ai/vespa/metricsproxy/service/VespaServicesTest.java +++ b/metrics-proxy/src/test/java/ai/vespa/metricsproxy/service/VespaServicesTest.java @@ -7,8 +7,7 @@ import org.junit.Test; import java.util.List; import static ai.vespa.metricsproxy.service.VespaServices.ALL_SERVICES; -import static org.hamcrest.CoreMatchers.is; -import static org.junit.Assert.assertThat; +import static org.junit.Assert.assertEquals; /** * TODO: add more tests @@ -24,7 +23,7 @@ public class VespaServicesTest { new DummyService(1, "dummy/id/1")); VespaServices services = new VespaServices(dummyServices); - assertThat(services.getMonitoringServices("vespa.dummy").size(), is(2)); + assertEquals(2, services.getMonitoringServices("vespa.dummy").size()); } @Test @@ -33,7 +32,7 @@ public class VespaServicesTest { new DummyService(0, "dummy/id/0")); VespaServices services = new VespaServices(dummyServices); - assertThat(services.getMonitoringServices(ALL_SERVICES).size(), is(1)); + assertEquals(1, services.getMonitoringServices(ALL_SERVICES).size()); } } diff --git a/node-admin/pom.xml b/node-admin/pom.xml index 8fe5b2aecd1..b5808ebc699 100644 --- a/node-admin/pom.xml +++ b/node-admin/pom.xml @@ -86,12 +86,6 @@ <!-- Test --> <dependency> - <groupId>org.hamcrest</groupId> - <artifactId>hamcrest-junit</artifactId> - <version>2.0.0.0</version> - <scope>test</scope> - </dependency> - <dependency> <groupId>org.mockito</groupId> <artifactId>mockito-core</artifactId> <scope>test</scope> diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/UnixPath.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/UnixPath.java index 6949f648865..cc780e277ad 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/UnixPath.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/UnixPath.java @@ -263,9 +263,8 @@ public class UnixPath { return uncheck(() -> Files.deleteIfExists(path)); } - public UnixPath deleteIfExists() { - uncheck(() -> Files.deleteIfExists(path)); - return this; + public boolean deleteIfExists() { + return uncheck(() -> Files.deleteIfExists(path)); } /** @return false path does not exist, is not a directory, or has at least one entry. */ diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/configserver/ConfigServerApiImplTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/configserver/ConfigServerApiImplTest.java index 64406fcfafa..866b7d2877a 100644 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/configserver/ConfigServerApiImplTest.java +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/configserver/ConfigServerApiImplTest.java @@ -21,9 +21,6 @@ import java.nio.charset.StandardCharsets; import java.time.Duration; import java.util.List; -import static org.hamcrest.CoreMatchers.is; -import static org.hamcrest.Matchers.arrayContainingInAnyOrder; -import static org.hamcrest.junit.MatcherAssert.assertThat; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertTrue; @@ -88,7 +85,7 @@ public class ConfigServerApiImplTest { @Test public void testBasicParsingSingleServer() { TestPojo answer = configServerApi.get("/path", TestPojo.class); - assertThat(answer.foo, is("bar")); + assertEquals(answer.foo, "bar"); assertLogStringContainsGETForAHost(); } @@ -139,8 +136,8 @@ public class ConfigServerApiImplTest { // ignore } - String[] log = mockLog.toString().split(" "); - assertThat(log, arrayContainingInAnyOrder("GET http://host1:666/path", "GET http://host2:666/path")); + List<String> log = List.of(mockLog.toString().split(" ")); + assertTrue(log.containsAll(List.of("GET http://host1:666/path", "GET http://host2:666/path"))); } @Test diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/configserver/flags/RealFlagRepositoryTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/configserver/flags/RealFlagRepositoryTest.java index 29581fe8b4d..8c914530122 100644 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/configserver/flags/RealFlagRepositoryTest.java +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/configserver/flags/RealFlagRepositoryTest.java @@ -6,14 +6,13 @@ import com.yahoo.vespa.flags.json.FlagData; import com.yahoo.vespa.flags.json.wire.WireFlagData; import com.yahoo.vespa.flags.json.wire.WireFlagDataList; import com.yahoo.vespa.hosted.node.admin.configserver.ConfigServerApi; -import org.hamcrest.collection.IsMapContaining; -import org.hamcrest.collection.IsMapWithSize; import org.junit.Test; import java.util.ArrayList; import java.util.Map; -import static org.hamcrest.MatcherAssert.assertThat; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.eq; import static org.mockito.Mockito.mock; @@ -35,7 +34,7 @@ public class RealFlagRepositoryTest { when(configServerApi.get(any(), eq(WireFlagDataList.class))).thenReturn(list); Map<FlagId, FlagData> allFlagData = repository.getAllFlagData(); - assertThat(allFlagData, IsMapWithSize.aMapWithSize(1)); - assertThat(allFlagData, IsMapContaining.hasKey(new FlagId("id1"))); + assertEquals(1, allFlagData.size()); + assertTrue(allFlagData.containsKey(new FlagId("id1"))); } }
\ No newline at end of file diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/RealNodeRepositoryTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/RealNodeRepositoryTest.java index 6a21814d5da..5d15d4353e2 100644 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/RealNodeRepositoryTest.java +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/RealNodeRepositoryTest.java @@ -23,10 +23,8 @@ import java.util.List; import java.util.Optional; import java.util.Set; -import static org.hamcrest.Matchers.is; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertThat; import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; @@ -103,13 +101,13 @@ public class RealNodeRepositoryTest { String dockerHostHostname = "dockerhost1.yahoo.com"; List<NodeSpec> containersToRun = nodeRepositoryApi.getNodes(dockerHostHostname); - assertThat(containersToRun.size(), is(1)); + assertEquals(1, containersToRun.size()); NodeSpec node = containersToRun.get(0); - assertThat(node.hostname(), is("host4.yahoo.com")); - assertThat(node.wantedDockerImage().get(), is(DockerImage.fromString("docker-registry.domain.tld:8080/dist/vespa:6.42.0"))); - assertThat(node.state(), is(NodeState.active)); - assertThat(node.wantedRestartGeneration().get(), is(0L)); - assertThat(node.currentRestartGeneration().get(), is(0L)); + assertEquals("host4.yahoo.com", node.hostname()); + assertEquals(DockerImage.fromString("docker-registry.domain.tld:8080/dist/vespa:6.42.0"), node.wantedDockerImage().get()); + assertEquals(NodeState.active, node.state()); + assertEquals(Long.valueOf(0), node.wantedRestartGeneration().get()); + assertEquals(Long.valueOf(0), node.currentRestartGeneration().get()); assertEquals(1, node.vcpu(), delta); assertEquals(4, node.memoryGb(), delta); assertEquals(100, node.diskGb(), delta); diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/systemd/SystemCtlTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/systemd/SystemCtlTest.java index 163c3410c4a..eea6c744bab 100644 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/systemd/SystemCtlTest.java +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/systemd/SystemCtlTest.java @@ -6,9 +6,7 @@ import com.yahoo.vespa.hosted.node.admin.task.util.process.ChildProcessFailureEx import com.yahoo.vespa.hosted.node.admin.task.util.process.TestTerminal; import org.junit.Test; -import static org.hamcrest.CoreMatchers.containsString; import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertThat; import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; import static org.mockito.Mockito.mock; @@ -139,7 +137,7 @@ public class SystemCtlTest { systemCtl.serviceExists(taskContext, "foo"); fail(); } catch (Exception e) { - assertThat(e.getMessage(), containsString("garbage")); + assertTrue(e.getMessage().contains("garbage")); } } diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/yum/YumPackageNameTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/yum/YumPackageNameTest.java index 45e1dc73521..0ff98e2b5fe 100644 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/yum/YumPackageNameTest.java +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/yum/YumPackageNameTest.java @@ -5,10 +5,8 @@ import org.junit.Test; import java.util.Optional; -import static org.hamcrest.CoreMatchers.containsStringIgnoringCase; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertThat; import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; @@ -174,7 +172,7 @@ public class YumPackageNameTest { YumPackageName.fromString("epoch:docker-engine-selinux-1.12.6-1.el7.x86_64"); fail(); } catch (IllegalArgumentException e) { - assertThat(e.getMessage(), containsStringIgnoringCase("epoch")); + assertTrue(e.getMessage().toLowerCase().contains("epoch")); } } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/ClusterNodesTimeseries.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/ClusterNodesTimeseries.java index 5ad4ef2e263..dd35eb7b54d 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/ClusterNodesTimeseries.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/ClusterNodesTimeseries.java @@ -31,7 +31,7 @@ public class ClusterNodesTimeseries { // If none can be detected we assume the node is new/was down. // If either this is the case, or there is a generation change, we ignore // the first warmupWindow metrics - var timeseries = db.getNodeTimeseries(period.plus(warmupDuration.multipliedBy(4)), clusterNodes); + var timeseries = db.getNodeTimeseries(period.plus(warmupDuration.multipliedBy(8)), clusterNodes); if (cluster.lastScalingEvent().isPresent()) { long currentGeneration = cluster.lastScalingEvent().get().generation(); timeseries = keepCurrentGenerationAfterWarmup(timeseries, currentGeneration); diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/NodeTimeseries.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/NodeTimeseries.java index 4a5f8972e11..c58b08cb3b5 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/NodeTimeseries.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/NodeTimeseries.java @@ -85,7 +85,7 @@ public class NodeTimeseries { if (snapshot.generation() < 0) return true; // Content nodes do not yet send generation if (snapshot.generation() < currentGeneration) return false; if (generationChange.isEmpty()) return true; - return ! snapshot.at().isBefore(generationChange.get().plus(warmupDuration)); + return ! snapshot.at().isBefore(generationChange.get().plus(warmupDuration.multipliedBy(2))); } } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/LoadBalancerProvisioner.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/LoadBalancerProvisioner.java index 1c2f6664258..5ff78c53f8a 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/LoadBalancerProvisioner.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/LoadBalancerProvisioner.java @@ -10,6 +10,9 @@ import com.yahoo.config.provision.NodeType; import com.yahoo.config.provision.TenantName; import com.yahoo.config.provision.exception.LoadBalancerServiceException; import com.yahoo.transaction.NestedTransaction; +import com.yahoo.vespa.flags.BooleanFlag; +import com.yahoo.vespa.flags.FetchVector; +import com.yahoo.vespa.flags.PermanentFlags; import com.yahoo.vespa.hosted.provision.Node; import com.yahoo.vespa.hosted.provision.NodeList; import com.yahoo.vespa.hosted.provision.NodeRepository; @@ -51,11 +54,13 @@ public class LoadBalancerProvisioner { private final NodeRepository nodeRepository; private final CuratorDatabaseClient db; private final LoadBalancerService service; + private final BooleanFlag deactivateRouting; public LoadBalancerProvisioner(NodeRepository nodeRepository, LoadBalancerService service) { this.nodeRepository = nodeRepository; this.db = nodeRepository.database(); this.service = service; + this.deactivateRouting = PermanentFlags.DEACTIVATE_ROUTING.bindTo(nodeRepository.flagSource()); // Read and write all load balancers to make sure they are stored in the latest version of the serialization format for (var id : db.readLoadBalancerIds()) { try (var lock = db.lock(id.application())) { @@ -204,12 +209,20 @@ public class LoadBalancerProvisioner { /** Provision or reconfigure a load balancer instance, if necessary */ private Optional<LoadBalancerInstance> provisionInstance(LoadBalancerId id, NodeList nodes, Optional<LoadBalancer> currentLoadBalancer) { - Set<Real> reals = realsOf(nodes); + boolean shouldDeactivateRouting = deactivateRouting.with(FetchVector.Dimension.APPLICATION_ID, + id.application().serializedForm()) + .value(); + Set<Real> reals; + if (shouldDeactivateRouting) { + reals = Set.of(); + } else { + reals = realsOf(nodes); + } if (hasReals(currentLoadBalancer, reals)) return currentLoadBalancer.get().instance(); log.log(Level.INFO, () -> "Provisioning instance for " + id + ", targeting: " + reals); try { return Optional.of(service.create(new LoadBalancerSpec(id.application(), id.cluster(), reals), - allowEmptyReals(currentLoadBalancer))); + shouldDeactivateRouting || allowEmptyReals(currentLoadBalancer))); } catch (Exception e) { log.log(Level.WARNING, e, () -> "Could not (re)configure " + id + ", targeting: " + reals + ". The operation will be retried on next deployment"); @@ -237,7 +250,7 @@ public class LoadBalancerProvisioner { /** Returns real servers for given nodes */ private Set<Real> realsOf(NodeList nodes) { - Set<Real> reals = new LinkedHashSet<Real>(); + Set<Real> reals = new LinkedHashSet<>(); for (var node : nodes) { for (var ip : reachableIpAddresses(node)) { reals.add(new Real(HostName.from(node.hostname()), ip)); diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/AutoscalingMaintainerTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/AutoscalingMaintainerTest.java index d9037181f59..9052ee30506 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/AutoscalingMaintainerTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/AutoscalingMaintainerTest.java @@ -211,6 +211,7 @@ public class AutoscalingMaintainerTest { tester.deploy(app1, cluster1, capacity); // fast completion tester.addMeasurements(1.0f, 0.3f, 0.3f, 0, 1, app1); + tester.clock().advance(Duration.ofSeconds(150)); tester.addMeasurements(1.0f, 0.3f, 0.3f, 0, 1, app1); tester.maintainer().maintain(); assertEquals("Scale up: " + tester.cluster(app1, cluster1).autoscalingStatus(), @@ -219,6 +220,7 @@ public class AutoscalingMaintainerTest { // fast completion, with initially overloaded cpu tester.addMeasurements(3.0f, 0.3f, 0.3f, 1, 1, app1); + tester.clock().advance(Duration.ofSeconds(150)); tester.addMeasurements(0.2f, 0.3f, 0.3f, 1, 1, app1); tester.maintainer().maintain(); assertEquals("No autoscaling since we ignore the (first) data point in the warup period", diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/LoadBalancerProvisionerTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/LoadBalancerProvisionerTest.java index b1f2efaef02..614817bce84 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/LoadBalancerProvisionerTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/LoadBalancerProvisionerTest.java @@ -13,6 +13,7 @@ import com.yahoo.config.provision.NodeType; import com.yahoo.config.provision.exception.LoadBalancerServiceException; import com.yahoo.transaction.NestedTransaction; import com.yahoo.vespa.flags.InMemoryFlagSource; +import com.yahoo.vespa.flags.PermanentFlags; import com.yahoo.vespa.hosted.provision.Node; import com.yahoo.vespa.hosted.provision.NodeList; import com.yahoo.vespa.hosted.provision.lb.LoadBalancer; @@ -160,6 +161,18 @@ public class LoadBalancerProvisionerTest { tester.activate(app1, prepare(app1, clusterRequest(ClusterSpec.Type.container, containerCluster1), clusterRequest(ClusterSpec.Type.content, contentCluster))); + + // Routing is disabled through feature flag. Reals are removed on next deployment + tester.loadBalancerService().throwOnCreate(false); + flagSource.withBooleanFlag(PermanentFlags.DEACTIVATE_ROUTING.id(), true); + tester.activate(app1, prepare(app1, + clusterRequest(ClusterSpec.Type.container, containerCluster1), + clusterRequest(ClusterSpec.Type.content, contentCluster))); + List<LoadBalancer> activeLoadBalancers = lbApp1.get().stream() + .filter(lb -> lb.state() == LoadBalancer.State.active) + .collect(Collectors.toList()); + assertEquals(1, activeLoadBalancers.size()); + assertEquals(Set.of(), activeLoadBalancers.get(0).instance().get().reals()); } @Test diff --git a/parent/pom.xml b/parent/pom.xml index 2cebaf21833..1c24316a80b 100644 --- a/parent/pom.xml +++ b/parent/pom.xml @@ -496,11 +496,6 @@ <version>${onnxruntime.version}</version> </dependency> <dependency> - <groupId>com.optimaize.languagedetector</groupId> - <artifactId>language-detector</artifactId> - <version>0.6</version> - </dependency> - <dependency> <groupId>com.yahoo.athenz</groupId> <artifactId>athenz-zms-java-client</artifactId> <version>${athenz.version}</version> @@ -704,7 +699,7 @@ <dependency> <groupId>org.apache.opennlp</groupId> <artifactId>opennlp-tools</artifactId> - <version>1.8.4</version> + <version>1.9.3</version> </dependency> <dependency> <groupId>org.apache.velocity</groupId> @@ -870,7 +865,7 @@ <dependency> <groupId>xerces</groupId> <artifactId>xercesImpl</artifactId> - <version>2.12.0</version> + <version>2.12.1</version> </dependency> </dependencies> </dependencyManagement> diff --git a/searchlib/src/test/java/com/yahoo/searchlib/expression/FloatBucketResultNodeTestCase.java b/searchlib/src/test/java/com/yahoo/searchlib/expression/FloatBucketResultNodeTestCase.java index 1acfb3cdc7e..78db838cefe 100644 --- a/searchlib/src/test/java/com/yahoo/searchlib/expression/FloatBucketResultNodeTestCase.java +++ b/searchlib/src/test/java/com/yahoo/searchlib/expression/FloatBucketResultNodeTestCase.java @@ -3,8 +3,9 @@ package com.yahoo.searchlib.expression; import org.junit.Test; -import static org.hamcrest.core.Is.is; -import static org.junit.Assert.*; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; /** * @author Ulf Lilleengen diff --git a/searchlib/src/test/java/com/yahoo/searchlib/expression/IntegerBucketResultNodeTestCase.java b/searchlib/src/test/java/com/yahoo/searchlib/expression/IntegerBucketResultNodeTestCase.java index c038a207ba2..00c0739ed9a 100644 --- a/searchlib/src/test/java/com/yahoo/searchlib/expression/IntegerBucketResultNodeTestCase.java +++ b/searchlib/src/test/java/com/yahoo/searchlib/expression/IntegerBucketResultNodeTestCase.java @@ -1,12 +1,10 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.searchlib.expression; -import com.yahoo.vespa.objects.BufferSerializer; import org.junit.Test; -import static org.hamcrest.core.Is.is; +import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertThat; import static org.junit.Assert.assertTrue; /** @@ -25,8 +23,8 @@ public class IntegerBucketResultNodeTestCase extends ResultNodeTest { @Test public void testRange() { IntegerBucketResultNode bucket = new IntegerBucketResultNode(4, 10); - assertThat(bucket.getFrom(), is(4l)); - assertThat(bucket.getTo(), is(10l)); + assertEquals(4, bucket.getFrom()); + assertEquals(10, bucket.getTo()); assertFalse(bucket.empty()); assertTrue(dumpNode(bucket).contains("from: 4")); assertTrue(dumpNode(bucket).contains("to: 10")); diff --git a/searchlib/src/test/java/com/yahoo/searchlib/expression/IntegerResultNodeTestCase.java b/searchlib/src/test/java/com/yahoo/searchlib/expression/IntegerResultNodeTestCase.java index b081506dca8..744df472fb2 100644 --- a/searchlib/src/test/java/com/yahoo/searchlib/expression/IntegerResultNodeTestCase.java +++ b/searchlib/src/test/java/com/yahoo/searchlib/expression/IntegerResultNodeTestCase.java @@ -1,17 +1,14 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.searchlib.expression; -import com.yahoo.vespa.objects.BufferSerializer; -import com.yahoo.vespa.objects.ObjectDumper; import org.junit.Test; import java.lang.reflect.InvocationTargetException; import java.util.Arrays; import java.util.List; -import static org.hamcrest.core.Is.is; +import static org.junit.Assert.assertArrayEquals; import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertThat; import static org.junit.Assert.assertTrue; /** @@ -29,52 +26,52 @@ public class IntegerResultNodeTestCase extends ResultNodeTest { @Test public void testClassId() { - assertThat(new Int8ResultNode().getClassId(), is(Int8ResultNode.classId)); - assertThat(new Int16ResultNode().getClassId(), is(Int16ResultNode.classId)); - assertThat(new Int32ResultNode().getClassId(), is(Int32ResultNode.classId)); - assertThat(new IntegerResultNode().getClassId(), is(IntegerResultNode.classId)); - assertThat(new BoolResultNode().getClassId(), is(BoolResultNode.classId)); + assertEquals(Int8ResultNode.classId, new Int8ResultNode().getClassId()); + assertEquals(Int16ResultNode.classId, new Int16ResultNode().getClassId()); + assertEquals(Int32ResultNode.classId, new Int32ResultNode().getClassId()); + assertEquals(IntegerResultNode.classId, new IntegerResultNode().getClassId()); + assertEquals(BoolResultNode.classId, new BoolResultNode().getClassId()); } @Test public void testTypeConversion() { for (NumericResultNode node : getResultNodes(3)) { - assertThat(node.getInteger(), is(3l)); - assertEquals(node.getFloat(), 3.0, 0.01); - assertThat(node.getRaw(), is(new byte[]{0, 0, 0, 0, 0, 0, 0, (byte) 3})); - assertThat(node.getString(), is("3")); - assertThat(node.getNumber().toString(), is("3")); + assertEquals(3, node.getInteger()); + assertEquals(3.0, node.getFloat(), 0.01); + assertArrayEquals(new byte[]{0, 0, 0, 0, 0, 0, 0, (byte) 3}, node.getRaw()); + assertEquals("3", node.getString()); + assertEquals("3", node.getNumber().toString()); } } @Test public void testMath() { for (NumericResultNode node : getResultNodes(5)) { - assertThat(node.getInteger(), is(5l)); + assertEquals(5, node.getInteger()); node.negate(); - assertThat(node.getInteger(), is(-5l)); + assertEquals(-5, node.getInteger()); node.multiply(new Int32ResultNode(3)); - assertThat(node.getInteger(), is(-15l)); + assertEquals(-15, node.getInteger()); node.add(new Int32ResultNode(1)); - assertThat(node.getInteger(), is(-14l)); + assertEquals(-14, node.getInteger()); node.divide(new Int32ResultNode(2)); - assertThat(node.getInteger(), is(-7l)); + assertEquals(-7, node.getInteger()); node.modulo(new Int32ResultNode(3)); - assertThat(node.getInteger(), is(-1l)); + assertEquals(-1, node.getInteger()); node.min(new Int32ResultNode(2)); - assertThat(node.getInteger(), is(-1l)); + assertEquals(-1, node.getInteger()); node.min(new Int32ResultNode(-2)); - assertThat(node.getInteger(), is(-2l)); + assertEquals(-2, node.getInteger()); node.max(new Int32ResultNode(-4)); - assertThat(node.getInteger(), is(-2l)); + assertEquals(-2, node.getInteger()); node.max(new Int32ResultNode(4)); - assertThat(node.getInteger(), is(4l)); - assertThat(node.onCmp(new Int32ResultNode(3)), is(1)); - assertThat(node.onCmp(new Int32ResultNode(4)), is(0)); - assertThat(node.onCmp(new Int32ResultNode(5)), is(-1)); + assertEquals(4, node.getInteger()); + assertEquals(1, node.onCmp(new Int32ResultNode(3))); + assertEquals(0, node.onCmp(new Int32ResultNode(4))); + assertEquals(-1, node.onCmp(new Int32ResultNode(5))); node.set(new Int32ResultNode(8)); - assertThat(node.getInteger(), is(8l)); - assertThat(node.hashCode(), is((int)(8 + node.getClassId()))); + assertEquals(8, node.getInteger()); + assertEquals(8 + node.getClassId(), node.hashCode()); assertTrue(dumpNode(node).contains("value: 8")); } } @@ -95,37 +92,37 @@ public class IntegerResultNodeTestCase extends ResultNodeTest { public void testInt8() { Int8ResultNode node = new Int8ResultNode(); node.setValue((byte) 5); - assertThat(node.getInteger(), is(5l)); + assertEquals(5, node.getInteger()); } @Test public void testInt16() { Int16ResultNode node = new Int16ResultNode(); node.setValue((short)5); - assertThat(node.getInteger(), is(5l)); + assertEquals(5, node.getInteger()); } @Test public void testInt32() { Int32ResultNode node = new Int32ResultNode(); node.setValue(5); - assertThat(node.getInteger(), is(5l)); + assertEquals(5, node.getInteger()); } @Test public void testLong() { IntegerResultNode node = new IntegerResultNode(); node.setValue(5); - assertThat(node.getInteger(), is(5l)); + assertEquals(5, node.getInteger()); } @Test public void testSerialization() throws IllegalAccessException, InstantiationException, NoSuchMethodException, InvocationTargetException { for (NumericResultNode node : getResultNodes(8)) { - assertThat(node.getInteger(), is(8L)); + assertEquals(8, node.getInteger()); NumericResultNode out = node.getClass().getConstructor().newInstance(); assertCorrectSerialization(node, out); - assertThat(out.getInteger(), is(node.getInteger())); + assertEquals(out.getInteger(), node.getInteger()); } } } diff --git a/searchlib/src/test/java/com/yahoo/searchlib/expression/NullResultNodeTestCase.java b/searchlib/src/test/java/com/yahoo/searchlib/expression/NullResultNodeTestCase.java index ff2af418ad4..64c30769f60 100644 --- a/searchlib/src/test/java/com/yahoo/searchlib/expression/NullResultNodeTestCase.java +++ b/searchlib/src/test/java/com/yahoo/searchlib/expression/NullResultNodeTestCase.java @@ -3,13 +3,8 @@ package com.yahoo.searchlib.expression; import com.yahoo.vespa.objects.ObjectDumper; import org.junit.Test; - -import java.util.regex.Pattern; - -import static org.hamcrest.core.Is.is; -import static org.hamcrest.core.IsNot.not; import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertThat; +import static org.junit.Assert.assertNotEquals; import static org.junit.Assert.assertTrue; /** @@ -20,17 +15,17 @@ public class NullResultNodeTestCase { @Test public void testNullResultNode() { NullResultNode nullRes = new NullResultNode(); - assertThat(nullRes.onGetClassId(), is(NullResultNode.classId)); - assertThat(nullRes.getInteger(), is(0l)); - assertThat(nullRes.getString(), is("")); - assertThat(nullRes.getRaw(), is(new byte[0])); - assertEquals(nullRes.getFloat(), 0.0, 0.01); - assertThat(nullRes.onCmp(new NullResultNode()), is(0)); - assertThat(nullRes.onCmp(new IntegerResultNode(0)), is(not(0))); + assertEquals(NullResultNode.classId, nullRes.onGetClassId()); + assertEquals(0, nullRes.getInteger()); + assertTrue(nullRes.getString().isEmpty()); + assertEquals(0, nullRes.getRaw().length); + assertEquals(0.0, nullRes.getFloat(), 0.01); + assertEquals(0, nullRes.onCmp(new NullResultNode())); + assertNotEquals(0, nullRes.onCmp(new IntegerResultNode(0))); ObjectDumper dumper = new ObjectDumper(); nullRes.visitMembers(dumper); assertTrue(dumper.toString().contains("result: <NULL>")); nullRes.set(new IntegerResultNode(3)); - assertThat(nullRes.onCmp(new IntegerResultNode(3)), is(not(0))); + assertNotEquals(0, nullRes.onCmp(new IntegerResultNode(3))); } } diff --git a/searchlib/src/test/java/com/yahoo/searchlib/expression/RawBucketResultNodeTestCase.java b/searchlib/src/test/java/com/yahoo/searchlib/expression/RawBucketResultNodeTestCase.java index eef4f225778..ab9fb2e2e50 100644 --- a/searchlib/src/test/java/com/yahoo/searchlib/expression/RawBucketResultNodeTestCase.java +++ b/searchlib/src/test/java/com/yahoo/searchlib/expression/RawBucketResultNodeTestCase.java @@ -3,9 +3,8 @@ package com.yahoo.searchlib.expression; import org.junit.Test; -import static org.hamcrest.core.Is.is; +import static org.junit.Assert.assertArrayEquals; import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertThat; import static org.junit.Assert.assertTrue; /** @@ -24,8 +23,8 @@ public class RawBucketResultNodeTestCase extends ResultNodeTest { public void testRange() { RawBucketResultNode bucket = new RawBucketResultNode(new RawResultNode(new byte[]{6, 9}), new RawResultNode(new byte[]{9, 6})); assertFalse(bucket.empty()); - assertThat(bucket.getFrom(), is(new byte[]{6, 9})); - assertThat(bucket.getTo(), is(new byte[]{9, 6})); + assertArrayEquals(new byte[]{6, 9}, bucket.getFrom()); + assertArrayEquals(new byte[]{9, 6}, bucket.getTo()); assertCorrectSerialization(bucket, new RawBucketResultNode()); assertTrue(dumpNode(bucket).contains("value: RawData(data = [6, 9])")); assertTrue(dumpNode(bucket).contains("value: RawData(data = [9, 6])")); diff --git a/searchlib/src/test/java/com/yahoo/searchlib/expression/ResultNodeTest.java b/searchlib/src/test/java/com/yahoo/searchlib/expression/ResultNodeTest.java index fb033801352..52be4d422d5 100644 --- a/searchlib/src/test/java/com/yahoo/searchlib/expression/ResultNodeTest.java +++ b/searchlib/src/test/java/com/yahoo/searchlib/expression/ResultNodeTest.java @@ -4,8 +4,7 @@ package com.yahoo.searchlib.expression; import com.yahoo.vespa.objects.BufferSerializer; import com.yahoo.vespa.objects.ObjectDumper; -import static org.hamcrest.core.Is.is; -import static org.junit.Assert.assertThat; +import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; /** @@ -24,7 +23,7 @@ public class ResultNodeTest { from.serialize(buffer); buffer.flip(); to.deserialize(buffer); - assertThat(from.onCmp(to), is(0)); + assertEquals(0, from.onCmp(to)); } public void assertOrder(ResultNode a, ResultNode b, ResultNode c) { diff --git a/searchlib/src/test/java/com/yahoo/searchlib/expression/ResultNodeVectorTestCase.java b/searchlib/src/test/java/com/yahoo/searchlib/expression/ResultNodeVectorTestCase.java index 54f4744897a..614e6bb608a 100644 --- a/searchlib/src/test/java/com/yahoo/searchlib/expression/ResultNodeVectorTestCase.java +++ b/searchlib/src/test/java/com/yahoo/searchlib/expression/ResultNodeVectorTestCase.java @@ -1,14 +1,9 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.searchlib.expression; -import com.yahoo.vespa.objects.BufferSerializer; import org.junit.Test; - -import java.util.List; - -import static org.hamcrest.core.Is.is; -import static org.hamcrest.core.IsNot.not; -import static org.junit.Assert.assertThat; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotEquals; import static org.junit.Assert.assertTrue; /** @@ -18,12 +13,12 @@ import static org.junit.Assert.assertTrue; public class ResultNodeVectorTestCase extends ResultNodeTest { @Test public void testClassId() { - assertThat(new IntegerResultNodeVector().getClassId(), is(IntegerResultNodeVector.classId)); - assertThat(new Int32ResultNodeVector().getClassId(), is(Int32ResultNodeVector.classId)); - assertThat(new Int16ResultNodeVector().getClassId(), is(Int16ResultNodeVector.classId)); - assertThat(new Int8ResultNodeVector().getClassId(), is(Int8ResultNodeVector.classId)); - assertThat(new FloatResultNodeVector().getClassId(), is(FloatResultNodeVector.classId)); - assertThat(new BoolResultNodeVector().getClassId(), is(BoolResultNodeVector.classId)); + assertEquals(IntegerResultNodeVector.classId, new IntegerResultNodeVector().getClassId()); + assertEquals(Int32ResultNodeVector.classId, new Int32ResultNodeVector().getClassId()); + assertEquals(Int16ResultNodeVector.classId, new Int16ResultNodeVector().getClassId()); + assertEquals(Int8ResultNodeVector.classId, new Int8ResultNodeVector().getClassId()); + assertEquals(FloatResultNodeVector.classId, new FloatResultNodeVector().getClassId()); + assertEquals(BoolResultNodeVector.classId, new BoolResultNodeVector().getClassId()); } @Test @@ -32,37 +27,37 @@ public class ResultNodeVectorTestCase extends ResultNodeTest { b.add(new BoolResultNode(true)); b.add(new BoolResultNode(false)); b.add((ResultNode)new BoolResultNode(false)); - assertThat(b.getVector().size(), is(3)); + assertEquals(3, b.getVector().size()); Int8ResultNodeVector i8 = new Int8ResultNodeVector(); i8.add(new Int8ResultNode((byte)9)); i8.add(new Int8ResultNode((byte)2)); i8.add((ResultNode)new Int8ResultNode((byte)5)); - assertThat(i8.getVector().size(), is(3)); + assertEquals(3, i8.getVector().size()); Int16ResultNodeVector i16 = new Int16ResultNodeVector(); i16.add(new Int16ResultNode((short)9)); i16.add(new Int16ResultNode((short)2)); i16.add((ResultNode)new Int16ResultNode((short)5)); - assertThat(i16.getVector().size(), is(3)); + assertEquals(3, i16.getVector().size()); Int32ResultNodeVector i32 = new Int32ResultNodeVector(); i32.add(new Int32ResultNode(9)); i32.add(new Int32ResultNode(2)); i32.add((ResultNode)new Int32ResultNode(5)); - assertThat(i32.getVector().size(), is(3)); + assertEquals(3, i32.getVector().size()); IntegerResultNodeVector ieger = new IntegerResultNodeVector(); ieger.add(new IntegerResultNode(9)); ieger.add(new IntegerResultNode(2)); ieger.add((ResultNode)new IntegerResultNode(5)); - assertThat(ieger.getVector().size(), is(3)); + assertEquals(3, ieger.getVector().size()); FloatResultNodeVector floatvec = new FloatResultNodeVector(); floatvec.add(new FloatResultNode(3.3)); floatvec.add(new FloatResultNode(3.4)); floatvec.add((ResultNode)new FloatResultNode(3.5)); - assertThat(floatvec.getVector().size(), is(3)); + assertEquals(3, floatvec.getVector().size()); } @Test @@ -155,11 +150,11 @@ public class ResultNodeVectorTestCase extends ResultNodeTest { } private void assertVecEqual(ResultNodeVector vec1, ResultNodeVector vec2) { - assertThat(vec1.onCmp(vec2), is(0)); + assertEquals(0, vec1.onCmp(vec2)); } private void assertClassCmp(ResultNodeVector add) { - assertThat(add.onCmp(new NullResultNode()), is(not(0))); + assertNotEquals(0, add.onCmp(new NullResultNode())); } @Test diff --git a/searchlib/src/test/java/com/yahoo/searchlib/expression/StringBucketResultNodeTestCase.java b/searchlib/src/test/java/com/yahoo/searchlib/expression/StringBucketResultNodeTestCase.java index 15c89b4beb3..fee60c9df7b 100644 --- a/searchlib/src/test/java/com/yahoo/searchlib/expression/StringBucketResultNodeTestCase.java +++ b/searchlib/src/test/java/com/yahoo/searchlib/expression/StringBucketResultNodeTestCase.java @@ -2,9 +2,7 @@ package com.yahoo.searchlib.expression; import org.junit.Test; - -import static org.hamcrest.core.Is.is; -import static org.junit.Assert.assertThat; +import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; /** @@ -22,8 +20,8 @@ public class StringBucketResultNodeTestCase extends ResultNodeTest { @Test public void testRange() { StringBucketResultNode bucket = new StringBucketResultNode("a", "d"); - assertThat(bucket.getFrom(), is("a")); - assertThat(bucket.getTo(), is("d")); + assertEquals("a", bucket.getFrom()); + assertEquals("d", bucket.getTo()); assertTrue(dumpNode(bucket).contains("value: 'a'")); assertTrue(dumpNode(bucket).contains("value: 'd'")); assertCorrectSerialization(bucket, new StringBucketResultNode()); diff --git a/searchlib/src/vespa/searchlib/attribute/changevector.h b/searchlib/src/vespa/searchlib/attribute/changevector.h index f1fb58eb9d0..271497398a8 100644 --- a/searchlib/src/vespa/searchlib/attribute/changevector.h +++ b/searchlib/src/vespa/searchlib/attribute/changevector.h @@ -25,32 +25,33 @@ struct ChangeBase { DIV, CLEARDOC }; - enum {UNSET_ENUM = 0xffffffffu}; + enum {UNSET_ENTRY_REF = 0xffffffffu}; ChangeBase() : _type(NOOP), _doc(0), _weight(1), - _enumScratchPad(UNSET_ENUM) + _cached_entry_ref(UNSET_ENTRY_REF) { } ChangeBase(Type type, uint32_t d, int32_t w = 1) : _type(type), _doc(d), _weight(w), - _enumScratchPad(UNSET_ENUM) + _cached_entry_ref(UNSET_ENTRY_REF) { } int cmp(const ChangeBase &b) const { int diff(_doc - b._doc); return diff; } bool operator <(const ChangeBase & b) const { return cmp(b) < 0; } - uint32_t getEnum() const { return _enumScratchPad; } - void setEnum(uint32_t value) const { _enumScratchPad = value; } - bool isEnumValid() const { return _enumScratchPad != UNSET_ENUM; } + uint32_t get_entry_ref() const { return _cached_entry_ref; } + void set_entry_ref(uint32_t entry_ref) const { _cached_entry_ref = entry_ref; } + bool has_entry_ref() const { return _cached_entry_ref != UNSET_ENTRY_REF; } + void clear_entry_ref() const { _cached_entry_ref = UNSET_ENTRY_REF; } Type _type; uint32_t _doc; int32_t _weight; - mutable uint32_t _enumScratchPad; + mutable uint32_t _cached_entry_ref; }; template <typename T> diff --git a/searchlib/src/vespa/searchlib/attribute/enumattribute.h b/searchlib/src/vespa/searchlib/attribute/enumattribute.h index 8136e654152..f0ff23a06b4 100644 --- a/searchlib/src/vespa/searchlib/attribute/enumattribute.h +++ b/searchlib/src/vespa/searchlib/attribute/enumattribute.h @@ -56,6 +56,7 @@ protected: virtual void considerAttributeChange(const Change & c, EnumStoreBatchUpdater & inserter) = 0; vespalib::MemoryUsage getEnumStoreValuesMemoryUsage() const override; void populate_address_space_usage(AddressSpaceUsage& usage) const override; + void cache_change_data_entry_ref(const Change& c) const; public: EnumAttribute(const vespalib::string & baseFileName, const AttributeVector::Config & cfg); ~EnumAttribute(); diff --git a/searchlib/src/vespa/searchlib/attribute/enumattribute.hpp b/searchlib/src/vespa/searchlib/attribute/enumattribute.hpp index 3e578856c2b..c0680fd9238 100644 --- a/searchlib/src/vespa/searchlib/attribute/enumattribute.hpp +++ b/searchlib/src/vespa/searchlib/attribute/enumattribute.hpp @@ -84,6 +84,15 @@ EnumAttribute<B>::populate_address_space_usage(AddressSpaceUsage& usage) const usage.set(AddressSpaceComponents::enum_store, _enumStore.get_values_address_space_usage()); } +template <typename B> +void +EnumAttribute<B>::cache_change_data_entry_ref(const Change& c) const +{ + EnumIndex new_idx; + _enumStore.find_index(c._data.raw(), new_idx); + c.set_entry_ref(new_idx.ref()); +} + } // namespace search diff --git a/searchlib/src/vespa/searchlib/attribute/multienumattribute.hpp b/searchlib/src/vespa/searchlib/attribute/multienumattribute.hpp index 251bbd7c8a7..7df30b895d2 100644 --- a/searchlib/src/vespa/searchlib/attribute/multienumattribute.hpp +++ b/searchlib/src/vespa/searchlib/attribute/multienumattribute.hpp @@ -25,10 +25,10 @@ template <typename B, typename M> bool MultiValueEnumAttribute<B, M>::extractChangeData(const Change & c, EnumIndex & idx) { - if ( ! c.isEnumValid() ) { + if ( ! c.has_entry_ref() ) { return this->_enumStore.find_index(c._data.raw(), idx); } - idx = EnumIndex(vespalib::datastore::EntryRef(c.getEnum())); + idx = EnumIndex(vespalib::datastore::EntryRef(c.get_entry_ref())); return true; } @@ -42,9 +42,9 @@ MultiValueEnumAttribute<B, M>::considerAttributeChange(const Change & c, EnumSto { EnumIndex idx; if (!this->_enumStore.find_index(c._data.raw(), idx)) { - c.setEnum(inserter.insert(c._data.raw()).ref()); + c.set_entry_ref(inserter.insert(c._data.raw()).ref()); } else { - c.setEnum(idx.ref()); + c.set_entry_ref(idx.ref()); } } } diff --git a/searchlib/src/vespa/searchlib/attribute/singleenumattribute.hpp b/searchlib/src/vespa/searchlib/attribute/singleenumattribute.hpp index dde853cbc90..a51c9804cf2 100644 --- a/searchlib/src/vespa/searchlib/attribute/singleenumattribute.hpp +++ b/searchlib/src/vespa/searchlib/attribute/singleenumattribute.hpp @@ -140,9 +140,9 @@ SingleValueEnumAttribute<B>::considerUpdateAttributeChange(const Change & c, Enu { EnumIndex idx; if (!this->_enumStore.find_index(c._data.raw(), idx)) { - c.setEnum(inserter.insert(c._data.raw()).ref()); + c.set_entry_ref(inserter.insert(c._data.raw()).ref()); } else { - c.setEnum(idx.ref()); + c.set_entry_ref(idx.ref()); } considerUpdateAttributeChange(c); // for numeric } @@ -168,8 +168,8 @@ SingleValueEnumAttribute<B>::applyUpdateValueChange(const Change& c, EnumStoreBa { EnumIndex oldIdx = _enumIndices[c._doc]; EnumIndex newIdx; - if (c.isEnumValid()) { - newIdx = EnumIndex(vespalib::datastore::EntryRef(c.getEnum())); + if (c.has_entry_ref()) { + newIdx = EnumIndex(vespalib::datastore::EntryRef(c.get_entry_ref())); } else { this->_enumStore.find_index(c._data.raw(), newIdx); } @@ -181,6 +181,8 @@ void SingleValueEnumAttribute<B>::applyValueChanges(EnumStoreBatchUpdater& updater) { ValueModifier valueGuard(this->getValueModifier()); + // This avoids searching for the defaultValue in the enum store for each CLEARDOC in the change vector. + this->cache_change_data_entry_ref(this->_defaultValue); for (const auto& change : this->_changes.getInsertOrder()) { if (change._type == ChangeBase::UPDATE) { applyUpdateValueChange(change, updater); @@ -192,6 +194,8 @@ SingleValueEnumAttribute<B>::applyValueChanges(EnumStoreBatchUpdater& updater) applyUpdateValueChange(clearDoc, updater); } } + // We must clear the cached entry ref as the defaultValue might be located in another data buffer on later invocations. + this->_defaultValue.clear_entry_ref(); } template <typename B> diff --git a/searchlib/src/vespa/searchlib/attribute/singlenumericenumattribute.hpp b/searchlib/src/vespa/searchlib/attribute/singlenumericenumattribute.hpp index d6622f7c880..c6b720ba14e 100644 --- a/searchlib/src/vespa/searchlib/attribute/singlenumericenumattribute.hpp +++ b/searchlib/src/vespa/searchlib/attribute/singlenumericenumattribute.hpp @@ -38,9 +38,9 @@ SingleValueNumericEnumAttribute<B>::considerArithmeticAttributeChange(const Chan EnumIndex idx; if (!this->_enumStore.find_index(newValue, idx)) { - c.setEnum(inserter.insert(newValue).ref()); + c.set_entry_ref(inserter.insert(newValue).ref()); } else { - c.setEnum(idx.ref()); + c.set_entry_ref(idx.ref()); } _currDocValues[c._doc] = newValue; diff --git a/searchlib/src/vespa/searchlib/attribute/singlenumericpostattribute.hpp b/searchlib/src/vespa/searchlib/attribute/singlenumericpostattribute.hpp index 1083d0f4cb8..4d37171e151 100644 --- a/searchlib/src/vespa/searchlib/attribute/singlenumericpostattribute.hpp +++ b/searchlib/src/vespa/searchlib/attribute/singlenumericpostattribute.hpp @@ -47,8 +47,8 @@ SingleValueNumericPostingAttribute<B>::applyUpdateValueChange(const Change & c, std::map<DocId, EnumIndex> & currEnumIndices) { EnumIndex newIdx; - if (c.isEnumValid()) { - newIdx = EnumIndex(vespalib::datastore::EntryRef(c.getEnum())); + if (c.has_entry_ref()) { + newIdx = EnumIndex(vespalib::datastore::EntryRef(c.get_entry_ref())); } else { enumStore.find_index(c._data.raw(), newIdx); } @@ -89,6 +89,8 @@ SingleValueNumericPostingAttribute<B>::applyValueChanges(EnumStoreBatchUpdater& // used to make sure several arithmetic operations on the same document in a single commit works std::map<DocId, EnumIndex> currEnumIndices; + // This avoids searching for the defaultValue in the enum store for each CLEARDOC in the change vector. + this->cache_change_data_entry_ref(this->_defaultValue); for (const auto& change : this->_changes.getInsertOrder()) { auto enumIter = currEnumIndices.find(change._doc); EnumIndex oldIdx; @@ -114,6 +116,8 @@ SingleValueNumericPostingAttribute<B>::applyValueChanges(EnumStoreBatchUpdater& applyUpdateValueChange(clearDoc, enumStore, currEnumIndices); } } + // We must clear the cached entry ref as the defaultValue might be located in another data buffer on later invocations. + this->_defaultValue.clear_entry_ref(); makePostingChange(enumStore.get_comparator(), currEnumIndices, changePost); diff --git a/searchlib/src/vespa/searchlib/attribute/singlestringpostattribute.hpp b/searchlib/src/vespa/searchlib/attribute/singlestringpostattribute.hpp index e77c59e915d..4670ee075fe 100644 --- a/searchlib/src/vespa/searchlib/attribute/singlestringpostattribute.hpp +++ b/searchlib/src/vespa/searchlib/attribute/singlestringpostattribute.hpp @@ -45,8 +45,8 @@ SingleValueStringPostingAttributeT<B>::applyUpdateValueChange(const Change & c, std::map<DocId, EnumIndex> &currEnumIndices) { EnumIndex newIdx; - if (c.isEnumValid()) { - newIdx = EnumIndex(vespalib::datastore::EntryRef(c.getEnum())); + if (c.has_entry_ref()) { + newIdx = EnumIndex(vespalib::datastore::EntryRef(c.get_entry_ref())); } else { enumStore.find_index(c._data.raw(), newIdx); } @@ -90,6 +90,8 @@ SingleValueStringPostingAttributeT<B>::applyValueChanges(EnumStoreBatchUpdater& // used to make sure several arithmetic operations on the same document in a single commit works std::map<DocId, EnumIndex> currEnumIndices; + // This avoids searching for the defaultValue in the enum store for each CLEARDOC in the change vector. + this->cache_change_data_entry_ref(this->_defaultValue); for (const auto& change : this->_changes.getInsertOrder()) { auto enumIter = currEnumIndices.find(change._doc); EnumIndex oldIdx; @@ -105,6 +107,8 @@ SingleValueStringPostingAttributeT<B>::applyValueChanges(EnumStoreBatchUpdater& applyUpdateValueChange(this->_defaultValue, enumStore, currEnumIndices); } } + // We must clear the cached entry ref as the defaultValue might be located in another data buffer on later invocations. + this->_defaultValue.clear_entry_ref(); makePostingChange(enumStore.get_folded_comparator(), dictionary, currEnumIndices, changePost); diff --git a/security-utils/pom.xml b/security-utils/pom.xml index 39a52fb12db..55b4b190f97 100644 --- a/security-utils/pom.xml +++ b/security-utils/pom.xml @@ -40,11 +40,6 @@ <scope>test</scope> </dependency> <dependency> - <groupId>org.hamcrest</groupId> - <artifactId>hamcrest-library</artifactId> - <scope>test</scope> - </dependency> - <dependency> <groupId>com.yahoo.vespa</groupId> <artifactId>testutil</artifactId> <version>${project.version}</version> diff --git a/security-utils/src/test/java/com/yahoo/security/KeyUtilsTest.java b/security-utils/src/test/java/com/yahoo/security/KeyUtilsTest.java index b0d587bb19a..f6f48d8b1b8 100644 --- a/security-utils/src/test/java/com/yahoo/security/KeyUtilsTest.java +++ b/security-utils/src/test/java/com/yahoo/security/KeyUtilsTest.java @@ -7,10 +7,9 @@ import java.security.KeyPair; import java.security.PrivateKey; import java.security.PublicKey; -import static org.hamcrest.CoreMatchers.containsString; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNotNull; -import static org.junit.Assert.assertThat; +import static org.junit.Assert.assertTrue; /** * @author bjorncs @@ -55,8 +54,8 @@ public class KeyUtilsTest { public void can_serialize_and_deserialize_rsa_publickey_using_pem_format() { KeyPair keyPair = KeyUtils.generateKeypair(KeyAlgorithm.RSA); String pem = KeyUtils.toPem(keyPair.getPublic()); - assertThat(pem, containsString("BEGIN PUBLIC KEY")); - assertThat(pem, containsString("END PUBLIC KEY")); + assertTrue(pem.contains("BEGIN PUBLIC KEY")); + assertTrue(pem.contains("END PUBLIC KEY")); PublicKey deserializedKey = KeyUtils.fromPemEncodedPublicKey(pem); assertEquals(keyPair.getPublic(), deserializedKey); assertEquals(KeyAlgorithm.RSA.getAlgorithmName(), deserializedKey.getAlgorithm()); @@ -66,8 +65,8 @@ public class KeyUtilsTest { public void can_serialize_and_deserialize_ec_publickey_using_pem_format() { KeyPair keyPair = KeyUtils.generateKeypair(KeyAlgorithm.EC); String pem = KeyUtils.toPem(keyPair.getPublic()); - assertThat(pem, containsString("BEGIN PUBLIC KEY")); - assertThat(pem, containsString("END PUBLIC KEY")); + assertTrue(pem.contains("BEGIN PUBLIC KEY")); + assertTrue(pem.contains("END PUBLIC KEY")); PublicKey deserializedKey = KeyUtils.fromPemEncodedPublicKey(pem); assertEquals(keyPair.getPublic(), deserializedKey); assertEquals(KeyAlgorithm.EC.getAlgorithmName(), deserializedKey.getAlgorithm()); @@ -76,8 +75,8 @@ public class KeyUtilsTest { private static void testPrivateKeySerialization(KeyAlgorithm keyAlgorithm, KeyFormat keyFormat, String pemLabel) { KeyPair keyPair = KeyUtils.generateKeypair(keyAlgorithm); String pem = KeyUtils.toPem(keyPair.getPrivate(), keyFormat); - assertThat(pem, containsString("BEGIN " + pemLabel)); - assertThat(pem, containsString("END " + pemLabel)); + assertTrue(pem.contains("BEGIN " + pemLabel)); + assertTrue(pem.contains("END " + pemLabel)); PrivateKey deserializedKey = KeyUtils.fromPemEncodedPrivateKey(pem); assertEquals(keyPair.getPrivate(), deserializedKey); assertEquals(keyAlgorithm.getAlgorithmName(), deserializedKey.getAlgorithm()); diff --git a/security-utils/src/test/java/com/yahoo/security/Pkcs10CsrUtilsTest.java b/security-utils/src/test/java/com/yahoo/security/Pkcs10CsrUtilsTest.java index ff0076b9cc4..32b8dfc5bcd 100644 --- a/security-utils/src/test/java/com/yahoo/security/Pkcs10CsrUtilsTest.java +++ b/security-utils/src/test/java/com/yahoo/security/Pkcs10CsrUtilsTest.java @@ -6,9 +6,8 @@ import org.junit.Test; import javax.security.auth.x500.X500Principal; import java.security.KeyPair; -import static org.hamcrest.CoreMatchers.containsString; import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertThat; +import static org.junit.Assert.assertTrue; /** * @author bjorncs @@ -22,8 +21,8 @@ public class Pkcs10CsrUtilsTest { Pkcs10Csr csr = Pkcs10CsrBuilder.fromKeypair(subject, keypair, SignatureAlgorithm.SHA512_WITH_ECDSA).build(); String pem = Pkcs10CsrUtils.toPem(csr); Pkcs10Csr deserializedCsr = Pkcs10CsrUtils.fromPem(pem); - assertThat(pem, containsString("BEGIN CERTIFICATE REQUEST")); - assertThat(pem, containsString("END CERTIFICATE REQUEST")); + assertTrue(pem.contains("BEGIN CERTIFICATE REQUEST")); + assertTrue(pem.contains("END CERTIFICATE REQUEST")); assertEquals(subject, deserializedCsr.getSubject()); } diff --git a/security-utils/src/test/java/com/yahoo/security/X509CertificateUtilsTest.java b/security-utils/src/test/java/com/yahoo/security/X509CertificateUtilsTest.java index 7075179234e..b2e800542b8 100644 --- a/security-utils/src/test/java/com/yahoo/security/X509CertificateUtilsTest.java +++ b/security-utils/src/test/java/com/yahoo/security/X509CertificateUtilsTest.java @@ -13,12 +13,9 @@ import java.util.Arrays; import java.util.List; import static com.yahoo.security.SubjectAlternativeName.Type.DNS_NAME; -import static org.hamcrest.CoreMatchers.containsString; -import static org.hamcrest.CoreMatchers.equalTo; -import static org.hamcrest.Matchers.is; + import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertThat; import static org.junit.Assert.assertTrue; /** @@ -32,8 +29,8 @@ public class X509CertificateUtilsTest { X509Certificate cert = TestUtils.createCertificate(keypair, subject); assertEquals(subject, cert.getSubjectX500Principal()); String pem = X509CertificateUtils.toPem(cert); - assertThat(pem, containsString("BEGIN CERTIFICATE")); - assertThat(pem, containsString("END CERTIFICATE")); + assertTrue(pem.contains("BEGIN CERTIFICATE")); + assertTrue(pem.contains("END CERTIFICATE")); X509Certificate deserializedCert = X509CertificateUtils.fromPem(pem); assertEquals(subject, deserializedCert.getSubjectX500Principal()); } @@ -70,8 +67,8 @@ public class X509CertificateUtilsTest { .build(); List<SubjectAlternativeName> sans = X509CertificateUtils.getSubjectAlternativeNames(cert); - assertThat(sans.size(), is(1)); - assertThat(sans.get(0), equalTo(san)); + assertEquals(1, sans.size()); + assertEquals(san, sans.get(0)); } @Test diff --git a/storage/src/tests/distributor/twophaseupdateoperationtest.cpp b/storage/src/tests/distributor/twophaseupdateoperationtest.cpp index 5aa2a3e5662..509f417baf1 100644 --- a/storage/src/tests/distributor/twophaseupdateoperationtest.cpp +++ b/storage/src/tests/distributor/twophaseupdateoperationtest.cpp @@ -351,6 +351,9 @@ TEST_F(TwoPhaseUpdateOperationTest, simple) { EXPECT_EQ("UpdateReply(id:ns:testdoctype1::1, BucketId(0x0000000000000000), " "timestamp 0, timestamp of updated doc: 90) ReturnCode(NONE)", _sender.getLastReply(true)); + + EXPECT_EQ(metrics().updates.failures.notfound.getValue(), 0); + EXPECT_EQ(metrics().updates.failures.test_and_set_failed.getValue(), 0); } TEST_F(TwoPhaseUpdateOperationTest, non_existing) { @@ -361,6 +364,8 @@ TEST_F(TwoPhaseUpdateOperationTest, non_existing) { EXPECT_EQ("UpdateReply(id:ns:testdoctype1::1, BucketId(0x0000000000000000), " "timestamp 0, timestamp of updated doc: 0) ReturnCode(NONE)", _sender.getLastReply(true)); + + EXPECT_EQ(metrics().updates.failures.notfound.getValue(), 1); } TEST_F(TwoPhaseUpdateOperationTest, update_failed) { @@ -627,6 +632,8 @@ TEST_F(TwoPhaseUpdateOperationTest, safe_path_updates_newest_received_document) "timestamp 0, timestamp of updated doc: 70) " "ReturnCode(NONE)", _sender.getLastReply(true)); + + EXPECT_EQ(metrics().updates.ok.getValue(), 1); } TEST_F(TwoPhaseUpdateOperationTest, create_if_non_existent_creates_document_if_all_empty_gets) { @@ -656,6 +663,8 @@ TEST_F(TwoPhaseUpdateOperationTest, create_if_non_existent_creates_document_if_a "timestamp 0, timestamp of updated doc: 200000000) " "ReturnCode(NONE)", _sender.getLastReply(true)); + + EXPECT_EQ(metrics().updates.ok.getValue(), 1); } TEST_F(TwoPhaseUpdateOperationTest, update_fails_if_safe_path_has_failed_put) { @@ -680,6 +689,9 @@ TEST_F(TwoPhaseUpdateOperationTest, update_fails_if_safe_path_has_failed_put) { "timestamp 0, timestamp of updated doc: 200000000) " "ReturnCode(IO_FAILURE)", _sender.getLastReply(true)); + + EXPECT_EQ(metrics().updates.ok.getValue(), 0); + EXPECT_EQ(metrics().updates.failures.storagefailure.getValue(), 1); } TEST_F(TwoPhaseUpdateOperationTest, update_fails_if_safe_path_gets_fail) { @@ -696,6 +708,9 @@ TEST_F(TwoPhaseUpdateOperationTest, update_fails_if_safe_path_gets_fail) { "timestamp 0, timestamp of updated doc: 0) " "ReturnCode(IO_FAILURE)", _sender.getLastReply(true)); + + EXPECT_EQ(metrics().updates.ok.getValue(), 0); + EXPECT_EQ(metrics().updates.failures.storagefailure.getValue(), 1); } TEST_F(TwoPhaseUpdateOperationTest, update_fails_if_apply_throws_exception) { @@ -741,6 +756,10 @@ TEST_F(TwoPhaseUpdateOperationTest, non_existing_with_auto_create) { "timestamp 0, timestamp of updated doc: 200000000) " "ReturnCode(NONE)", _sender.getLastReply(true)); + + EXPECT_EQ(metrics().updates.ok.getValue(), 1); + // "Not found" failure not counted when create: true is set, since the update itself isn't failed. + EXPECT_EQ(metrics().updates.failures.notfound.getValue(), 0); } TEST_F(TwoPhaseUpdateOperationTest, safe_path_fails_update_when_mismatching_timestamp_constraint) { @@ -759,6 +778,9 @@ TEST_F(TwoPhaseUpdateOperationTest, safe_path_fails_update_when_mismatching_time "ReturnCode(NONE, No document with requested " "timestamp found)", _sender.getLastReply(true)); + + EXPECT_EQ(metrics().updates.ok.getValue(), 0); + EXPECT_EQ(metrics().updates.failures.notfound.getValue(), 1); } TEST_F(TwoPhaseUpdateOperationTest, safe_path_update_propagates_message_settings_to_gets_and_puts) { @@ -863,6 +885,9 @@ TEST_F(TwoPhaseUpdateOperationTest, safe_path_condition_mismatch_fails_with_tas_ "ReturnCode(TEST_AND_SET_CONDITION_FAILED, " "Condition did not match document)", _sender.getLastReply(true)); + + EXPECT_EQ(metrics().updates.failures.notfound.getValue(), 0); + EXPECT_EQ(metrics().updates.failures.test_and_set_failed.getValue(), 1); } TEST_F(TwoPhaseUpdateOperationTest, safe_path_condition_match_sends_puts_with_updated_doc) { @@ -928,6 +953,9 @@ TEST_F(TwoPhaseUpdateOperationTest, safe_path_condition_with_missing_doc_and_no_ "ReturnCode(TEST_AND_SET_CONDITION_FAILED, " "Document did not exist)", _sender.getLastReply(true)); + + EXPECT_EQ(metrics().updates.failures.notfound.getValue(), 0); // Not counted as "not found" failure when TaS is present + EXPECT_EQ(metrics().updates.failures.test_and_set_failed.getValue(), 1); } TEST_F(TwoPhaseUpdateOperationTest, safe_path_condition_with_missing_doc_and_auto_create_sends_puts) { @@ -937,9 +965,22 @@ TEST_F(TwoPhaseUpdateOperationTest, safe_path_condition_with_missing_doc_and_aut .createIfNonExistent(true)); cb->start(_sender, framework::MilliSecTime(0)); - replyToGet(*cb, _sender, 0, 100, false); - replyToGet(*cb, _sender, 1, 110, false); + replyToGet(*cb, _sender, 0, 0, false); + replyToGet(*cb, _sender, 1, 0, false); ASSERT_EQ("Put => 1,Put => 0", _sender.getCommands(true, false, 2)); + + replyToPut(*cb, _sender, 2); + replyToPut(*cb, _sender, 3); + + EXPECT_EQ("UpdateReply(id:ns:testdoctype1::1, " + "BucketId(0x0000000000000000), " + "timestamp 0, timestamp of updated doc: 200000000) " + "ReturnCode(NONE)", + _sender.getLastReply(true)); + + EXPECT_EQ(metrics().updates.failures.notfound.getValue(), 0); // Not counted as "not found" failure when we auto create + EXPECT_EQ(metrics().updates.failures.test_and_set_failed.getValue(), 0); + EXPECT_EQ(metrics().updates.ok.getValue(), 1); } void diff --git a/storage/src/vespa/storage/distributor/operations/external/twophaseupdateoperation.cpp b/storage/src/vespa/storage/distributor/operations/external/twophaseupdateoperation.cpp index 563c2d573cd..af3b727e07b 100644 --- a/storage/src/vespa/storage/distributor/operations/external/twophaseupdateoperation.cpp +++ b/storage/src/vespa/storage/distributor/operations/external/twophaseupdateoperation.cpp @@ -138,15 +138,16 @@ TwoPhaseUpdateOperation::transitionTo(SendState newState) void TwoPhaseUpdateOperation::ensureUpdateReplyCreated() { - if (!_updateReply.get()) { - _updateReply = _updateCmd->makeReply(); + if (!_updateReply) { + _updateReply = std::dynamic_pointer_cast<api::UpdateReply>(std::shared_ptr<api::StorageReply>(_updateCmd->makeReply())); + assert(_updateReply); } } void TwoPhaseUpdateOperation::sendReply( DistributorStripeMessageSender& sender, - std::shared_ptr<api::StorageReply>& reply) + std::shared_ptr<api::UpdateReply> reply) { assert(!_replySent); reply->getTrace().addChild(std::move(_trace)); @@ -154,12 +155,20 @@ TwoPhaseUpdateOperation::sendReply( _replySent = true; } +// This particular method is called when we synthesize our own UpdateReply, +// not when we take over an already produced one from an UpdateOperation. +// The latter will already increment _updateMetric fields implicitly. void TwoPhaseUpdateOperation::sendReplyWithResult( DistributorStripeMessageSender& sender, const api::ReturnCode& result) { ensureUpdateReplyCreated(); + // Don't bump metrics if document not found but otherwise OK. + // Already counted in metrics prior to calling this method. + if (!(result.success() && (_updateReply->getOldTimestamp() == 0))) { + _updateMetric.updateFromResult(result); + } // else: `notfound` metric already incremented. _updateReply->setResult(result); sendReply(sender, _updateReply); } @@ -195,7 +204,7 @@ TwoPhaseUpdateOperation::startFastPathUpdate(DistributorStripeMessageSender& sen transitionTo(SendState::UPDATES_SENT); if (intermediate._reply.get()) { - sendReply(sender, intermediate._reply); + sendReply(sender, std::dynamic_pointer_cast<api::UpdateReply>(intermediate._reply)); } } @@ -367,18 +376,26 @@ TwoPhaseUpdateOperation::handleFastPathReceive(DistributorStripeMessageSender& s if (intermediate._reply.get()) { assert(_sendState == SendState::UPDATES_SENT); addTraceFromReply(*intermediate._reply); - UpdateOperation& cb = static_cast<UpdateOperation&> (callbackOp); + auto& cb = dynamic_cast<UpdateOperation&>(callbackOp); std::pair<document::BucketId, uint16_t> bestNode = cb.getNewestTimestampLocation(); - - if (!intermediate._reply->getResult().success() || - bestNode.first == document::BucketId(0)) { + auto intermediate_update_reply = std::dynamic_pointer_cast<api::UpdateReply>(intermediate._reply); + assert(intermediate_update_reply); + + if (!intermediate_update_reply->getResult().success() || + bestNode.first == document::BucketId(0)) + { + if (intermediate_update_reply->getResult().success() && + (intermediate_update_reply->getOldTimestamp() == 0)) + { + _updateMetric.failures.notfound.inc(); + } // Failed or was consistent - sendReply(sender, intermediate._reply); + sendReply(sender, std::move(intermediate_update_reply)); } else { LOG(debug, "Update(%s) fast path: was inconsistent!", update_doc_id().c_str()); - _updateReply = intermediate._reply; + _updateReply = std::move(intermediate_update_reply); _fast_path_repair_source_node = bestNode.second; document::Bucket bucket(_updateCmd->getBucket().getBucketSpace(), bestNode.first); auto cmd = std::make_shared<api::GetCommand>(bucket, _updateCmd->getDocumentId(), document::AllFields::NAME); @@ -535,9 +552,10 @@ TwoPhaseUpdateOperation::handleSafePathReceivedGet(DistributorStripeMessageSende document::Document::SP docToUpdate; api::Timestamp putTimestamp = _op_ctx.generate_unique_timestamp(); - if (reply.getDocument().get()) { + if (reply.getDocument()) { api::Timestamp receivedTimestamp = reply.getLastModifiedTimestamp(); if (!satisfiesUpdateTimestampConstraint(receivedTimestamp)) { + _updateMetric.failures.notfound.inc(); sendReplyWithResult(sender, api::ReturnCode(api::ReturnCode::OK, "No document with requested timestamp found")); return; @@ -556,6 +574,7 @@ TwoPhaseUpdateOperation::handleSafePathReceivedGet(DistributorStripeMessageSende docToUpdate = createBlankDocument(); setUpdatedForTimestamp(putTimestamp); } else { + _updateMetric.failures.notfound.inc(); sendReplyWithResult(sender, reply.getResult()); return; } @@ -644,7 +663,7 @@ void TwoPhaseUpdateOperation::setUpdatedForTimestamp(api::Timestamp ts) { ensureUpdateReplyCreated(); - static_cast<api::UpdateReply&>(*_updateReply).setOldTimestamp(ts); + _updateReply->setOldTimestamp(ts); } std::shared_ptr<document::Document> @@ -700,7 +719,7 @@ TwoPhaseUpdateOperation::onClose(DistributorStripeMessageSender& sender) { auto candidateReply = std::move(intermediate._reply); if (candidateReply && candidateReply->getType() == api::MessageType::UPDATE_REPLY) { assert(_mode == Mode::FAST_PATH); - sendReply(sender, candidateReply); // Sets _replySent + sendReply(sender, std::dynamic_pointer_cast<api::UpdateReply>(candidateReply)); // Sets _replySent } } else { break; diff --git a/storage/src/vespa/storage/distributor/operations/external/twophaseupdateoperation.h b/storage/src/vespa/storage/distributor/operations/external/twophaseupdateoperation.h index aa59a457eb7..45dec86268f 100644 --- a/storage/src/vespa/storage/distributor/operations/external/twophaseupdateoperation.h +++ b/storage/src/vespa/storage/distributor/operations/external/twophaseupdateoperation.h @@ -16,6 +16,7 @@ namespace storage { namespace api { class UpdateCommand; +class UpdateReply; class CreateBucketReply; class ReturnCode; } @@ -94,7 +95,7 @@ private: static const char* stateToString(SendState); void sendReply(DistributorStripeMessageSender&, - std::shared_ptr<api::StorageReply>&); + std::shared_ptr<api::UpdateReply>); void sendReplyWithResult(DistributorStripeMessageSender&, const api::ReturnCode&); void ensureUpdateReplyCreated(); @@ -144,7 +145,7 @@ private: PersistenceOperationMetricSet& _getMetric; PersistenceOperationMetricSet& _metadata_get_metrics; std::shared_ptr<api::UpdateCommand> _updateCmd; - std::shared_ptr<api::StorageReply> _updateReply; + std::shared_ptr<api::UpdateReply> _updateReply; const DistributorNodeContext& _node_ctx; DistributorStripeOperationContext& _op_ctx; const DocumentSelectionParser& _parser; diff --git a/storage/src/vespa/storage/distributor/persistence_operation_metric_set.cpp b/storage/src/vespa/storage/distributor/persistence_operation_metric_set.cpp index 9a7e2aba5e4..0e42a505567 100644 --- a/storage/src/vespa/storage/distributor/persistence_operation_metric_set.cpp +++ b/storage/src/vespa/storage/distributor/persistence_operation_metric_set.cpp @@ -39,9 +39,14 @@ PersistenceFailuresMetricSet::PersistenceFailuresMetricSet(MetricSet* owner) sum.addMetricToSum(timeout); sum.addMetricToSum(busy); sum.addMetricToSum(inconsistent_bucket); - sum.addMetricToSum(notfound); - // TaS/concurrent mutation failures not added to the main failure metric, as they're not "failures" as per se. - // TODO introduce separate aggregate for such metrics + // We don't consider the following as explicit failures (even though they're in the failure set) + // and therefore don't count them as part of the aggregate sum: + // + // - Test-and-set mismatches + // - Concurrent mutation failures + // - Document to be updated not found + // + // TODO introduce separate aggregate for such metrics. Presumably when we deprecate legacy metric paths. } PersistenceFailuresMetricSet::~PersistenceFailuresMetricSet() = default; diff --git a/vespa-feed-client/src/test/java/ai/vespa/feed/client/impl/HttpRequestStrategyTest.java b/vespa-feed-client/src/test/java/ai/vespa/feed/client/impl/HttpRequestStrategyTest.java index d7be4ead078..13dc7ad4624 100644 --- a/vespa-feed-client/src/test/java/ai/vespa/feed/client/impl/HttpRequestStrategyTest.java +++ b/vespa-feed-client/src/test/java/ai/vespa/feed/client/impl/HttpRequestStrategyTest.java @@ -205,7 +205,9 @@ class HttpRequestStrategyTest { DocumentId id2 = DocumentId.of("ns", "type", "2"); DocumentId id3 = DocumentId.of("ns", "type", "3"); DocumentId id4 = DocumentId.of("ns", "type", "4"); + DocumentId id5 = DocumentId.of("ns", "type", "5"); HttpRequest failing = new HttpRequest("POST", "/", null, null, null); + HttpRequest partial = new HttpRequest("POST", "/", null, null, null); HttpRequest request = new HttpRequest("POST", "/", null, null, null); HttpRequest blocking = new HttpRequest("POST", "/", null, null, null); @@ -218,22 +220,28 @@ class HttpRequestStrategyTest { blocker.arriveAndAwaitAdvance(); // ... block dispatch thread, so we get something in the queue. throw new RuntimeException("armageddon"); // Dispatch thread should die, tearing down everything. } - else if (req == failing) { + else if (req == partial) { phaser.arriveAndAwaitAdvance(); // Let test thread enqueue more ops before failing (and retrying) this. vessel.completeExceptionally(new IOException("failed")); - phaser.arriveAndAwaitAdvance(); // Ensure a retry is scheduled before test thread is allowed to continue. } - else phaser.arriveAndAwaitAdvance(); // Don't complete from mock cluster, but require destruction to do this. + else if (req == failing) { + System.err.println("failing"); + vessel.completeExceptionally(new RuntimeException("fatal")); + } }); + // inflight completes dispatch, but causes no response. CompletableFuture<HttpResponse> inflight = strategy.enqueue(id1, request); + // serialised 1 and 2 are waiting for the above inflight to complete. CompletableFuture<HttpResponse> serialised1 = strategy.enqueue(id1, request); CompletableFuture<HttpResponse> serialised2 = strategy.enqueue(id1, request); - CompletableFuture<HttpResponse> failed = strategy.enqueue(id2, failing); - CompletableFuture<HttpResponse> blocked = strategy.enqueue(id3, blocking); - CompletableFuture<HttpResponse> delayed = strategy.enqueue(id4, request); - phaser.arriveAndAwaitAdvance(); // inflight completes dispatch, but causes no response. - phaser.arriveAndAwaitAdvance(); // failed is allowed to dispatch ... - phaser.arriveAndAwaitAdvance(); // ... and a retry is enqueued. + CompletableFuture<HttpResponse> retried = strategy.enqueue(id2, partial); + CompletableFuture<HttpResponse> failed = strategy.enqueue(id3, failing); + CompletableFuture<HttpResponse> blocked = strategy.enqueue(id4, blocking); + CompletableFuture<HttpResponse> delayed = strategy.enqueue(id5, request); + phaser.arriveAndAwaitAdvance(); // retried is allowed to dispatch, and will be retried async. + // failed immediately fails, and lets us assert the above retry is indeed enqueued. + assertEquals("ai.vespa.feed.client.FeedException: java.lang.RuntimeException: fatal", + assertThrows(ExecutionException.class, failed::get).getMessage()); phaser.arriveAndAwaitAdvance(); // blocked starts dispatch, and hangs, blocking dispatch thread. // Current state: inflight is "inflight to cluster", serialised1/2 are waiting completion of it; @@ -242,7 +250,8 @@ class HttpRequestStrategyTest { assertFalse(inflight.isDone()); assertFalse(serialised1.isDone()); assertFalse(serialised2.isDone()); - assertFalse(failed.isDone()); + assertTrue(failed.isDone()); + assertFalse(retried.isDone()); assertFalse(blocked.isDone()); assertFalse(delayed.isDone()); @@ -259,7 +268,7 @@ class HttpRequestStrategyTest { assertEquals("ai.vespa.feed.client.FeedException: Operation aborted", assertThrows(ExecutionException.class, delayed::get).getMessage()); assertEquals("ai.vespa.feed.client.FeedException: java.io.IOException: failed", - assertThrows(ExecutionException.class, failed::get).getMessage()); + assertThrows(ExecutionException.class, retried::get).getMessage()); assertEquals("ai.vespa.feed.client.FeedException: Operation aborted", assertThrows(ExecutionException.class, strategy.enqueue(id1, request)::get).getMessage()); } diff --git a/vespa-hadoop/pom.xml b/vespa-hadoop/pom.xml index 9c3ec0d5e07..d6d9d0e2141 100644 --- a/vespa-hadoop/pom.xml +++ b/vespa-hadoop/pom.xml @@ -28,12 +28,32 @@ <artifactId>hadoop-common</artifactId> <version>${hadoop.version}</version> <scope>provided</scope> + <exclusions> + <exclusion> + <groupId>log4j</groupId> + <artifactId>log4j</artifactId> + </exclusion> + <exclusion> + <groupId>org.slf4j</groupId> + <artifactId>slf4j-log4j12</artifactId> + </exclusion> + </exclusions> </dependency> <dependency> <groupId>org.apache.hadoop</groupId> <artifactId>hadoop-client</artifactId> <version>${hadoop.version}</version> <scope>provided</scope> + <exclusions> + <exclusion> + <groupId>log4j</groupId> + <artifactId>log4j</artifactId> + </exclusion> + <exclusion> + <groupId>org.slf4j</groupId> + <artifactId>slf4j-log4j12</artifactId> + </exclusion> + </exclusions> </dependency> <dependency> <groupId>org.apache.pig</groupId> @@ -73,8 +93,31 @@ <artifactId>hadoop-minicluster</artifactId> <version>${hadoop.version}</version> <scope>test</scope> + <exclusions> + <exclusion> + <groupId>log4j</groupId> + <artifactId>log4j</artifactId> + </exclusion> + <exclusion> + <groupId>org.slf4j</groupId> + <artifactId>slf4j-log4j12</artifactId> + </exclusion> + </exclusions> </dependency> <dependency> + <!-- Use slf4j's replacement for log4j to avoid log4j in the dependency tree --> + <groupId>org.slf4j</groupId> + <artifactId>log4j-over-slf4j</artifactId> + <scope>test</scope> + </dependency> + <dependency> + <!-- Bind to java.util.logging --> + <groupId>org.slf4j</groupId> + <artifactId>slf4j-jdk14</artifactId> + <scope>test</scope> + </dependency> + + <dependency> <groupId>org.junit.jupiter</groupId> <artifactId>junit-jupiter</artifactId> <scope>test</scope> diff --git a/vespaclient-container-plugin/src/test/java/com/yahoo/vespa/http/server/FeedHandlerCompressionTest.java b/vespaclient-container-plugin/src/test/java/com/yahoo/vespa/http/server/FeedHandlerCompressionTest.java index 01784226c9e..6f1b5eebcc4 100644 --- a/vespaclient-container-plugin/src/test/java/com/yahoo/vespa/http/server/FeedHandlerCompressionTest.java +++ b/vespaclient-container-plugin/src/test/java/com/yahoo/vespa/http/server/FeedHandlerCompressionTest.java @@ -10,8 +10,7 @@ import java.io.IOException; import java.io.InputStream; import java.util.zip.GZIPOutputStream; -import static org.hamcrest.core.Is.is; -import static org.junit.Assert.*; +import static org.junit.Assert.assertEquals; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; @@ -42,7 +41,7 @@ public class FeedHandlerCompressionTest { } processedInput.append((char)readValue); } - assertThat(processedInput.toString(), is(testData)); + assertEquals(processedInput.toString(), testData); } /** @@ -64,7 +63,7 @@ public class FeedHandlerCompressionTest { } processedInput.append((char)readValue); } - assertThat(processedInput.toString(), is(testData)); + assertEquals(processedInput.toString(), testData); } } diff --git a/vespaclient-container-plugin/src/test/java/com/yahoo/vespa/http/server/FeedHandlerV3Test.java b/vespaclient-container-plugin/src/test/java/com/yahoo/vespa/http/server/FeedHandlerV3Test.java index 1fd6425f945..2f4afb0c2a5 100644 --- a/vespaclient-container-plugin/src/test/java/com/yahoo/vespa/http/server/FeedHandlerV3Test.java +++ b/vespaclient-container-plugin/src/test/java/com/yahoo/vespa/http/server/FeedHandlerV3Test.java @@ -30,9 +30,8 @@ import java.io.InputStream; import java.util.concurrent.Executor; import java.util.concurrent.Executors; -import static org.hamcrest.CoreMatchers.is; -import static org.hamcrest.CoreMatchers.startsWith; -import static org.junit.Assert.assertThat; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; import static org.mockito.ArgumentMatchers.any; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; @@ -47,8 +46,8 @@ public class FeedHandlerV3Test { HttpResponse httpResponse = feedHandlerV3.handle(createRequest(1)); ByteArrayOutputStream outStream = new ByteArrayOutputStream(); httpResponse.render(outStream); - assertThat(httpResponse.getContentType(), is("text/plain")); - assertThat(Utf8.toString(outStream.toByteArray()), is("1230 OK message trace\n")); + assertEquals(httpResponse.getContentType(), "text/plain"); + assertEquals(Utf8.toString(outStream.toByteArray()), "1230 OK message trace\n"); } @Test @@ -57,9 +56,9 @@ public class FeedHandlerV3Test { HttpResponse httpResponse = feedHandlerV3.handle(createBrokenRequest()); ByteArrayOutputStream outStream = new ByteArrayOutputStream(); httpResponse.render(outStream); - assertThat(httpResponse.getContentType(), is("text/plain")); - assertThat(Utf8.toString(outStream.toByteArray()), startsWith("1230 ERROR ")); - assertThat(metric.get(MetricNames.PARSE_ERROR), is(1L)); + assertEquals(httpResponse.getContentType(), "text/plain"); + assertTrue(Utf8.toString(outStream.toByteArray()).startsWith("1230 ERROR ")); + assertEquals(1L, metric.get(MetricNames.PARSE_ERROR)); } @Test @@ -68,9 +67,9 @@ public class FeedHandlerV3Test { HttpResponse httpResponse = feedHandlerV3.handle(createRequest(100)); ByteArrayOutputStream outStream = new ByteArrayOutputStream(); httpResponse.render(outStream); - assertThat(httpResponse.getContentType(), is("text/plain")); + assertEquals(httpResponse.getContentType(), "text/plain"); String result = Utf8.toString(outStream.toByteArray()); - assertThat(Splitter.on("\n").splitToList(result).size(), is(101)); + assertEquals(101, Splitter.on("\n").splitToList(result).size()); } private static DocumentTypeManager createDoctypeManager() { diff --git a/vespaclient-container-plugin/src/test/java/com/yahoo/vespa/http/server/VersionsTestCase.java b/vespaclient-container-plugin/src/test/java/com/yahoo/vespa/http/server/VersionsTestCase.java index a13f4f79ca9..6858c4bede3 100644 --- a/vespaclient-container-plugin/src/test/java/com/yahoo/vespa/http/server/VersionsTestCase.java +++ b/vespaclient-container-plugin/src/test/java/com/yahoo/vespa/http/server/VersionsTestCase.java @@ -10,10 +10,9 @@ import java.util.Arrays; import java.util.Collections; import java.util.List; -import static org.hamcrest.CoreMatchers.instanceOf; -import static org.hamcrest.CoreMatchers.nullValue; -import static org.hamcrest.core.Is.is; -import static org.junit.Assert.assertThat; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertTrue; /** * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a> @@ -34,86 +33,86 @@ public class VersionsTestCase { private static final List<String> GARBAGE = Collections.singletonList("garbage"); @Test - public void testEmpty() throws Exception { + public void testEmpty() { Tuple2<HttpResponse, Integer> v = FeedHandler.doCheckProtocolVersion(EMPTY); - assertThat(v.first, instanceOf(ErrorHttpResponse.class)); - assertThat(v.second, is(-1)); + assertTrue(v.first instanceof ErrorHttpResponse); + assertEquals(Integer.valueOf(-1), v.second); } @Test - public void testOneTwo() throws Exception { + public void testOneTwo() { Tuple2<HttpResponse, Integer> v = FeedHandler.doCheckProtocolVersion(ONE_TWO); - assertThat(v.first, instanceOf(ErrorHttpResponse.class)); - assertThat(v.second, is(-1)); + assertTrue(v.first instanceof ErrorHttpResponse); + assertEquals(Integer.valueOf(-1), v.second); } @Test - public void testOneThree() throws Exception { + public void testOneThree() { Tuple2<HttpResponse, Integer> v = FeedHandler.doCheckProtocolVersion(ONE_THREE); - assertThat(v.first, nullValue()); - assertThat(v.second, is(3)); + assertNull(v.first); + assertEquals(Integer.valueOf(3), v.second); } @Test - public void testTwoThree() throws Exception { + public void testTwoThree() { Tuple2<HttpResponse, Integer> v = FeedHandler.doCheckProtocolVersion(TWO_THREE); - assertThat(v.first, nullValue()); - assertThat(v.second, is(3)); + assertNull(v.first); + assertEquals(Integer.valueOf(3), v.second); } @Test - public void testOneNullThree() throws Exception { + public void testOneNullThree() { Tuple2<HttpResponse, Integer> v = FeedHandler.doCheckProtocolVersion(ONE_NULL_THREE); - assertThat(v.first, nullValue()); - assertThat(v.second, is(3)); + assertNull(v.first); + assertEquals(Integer.valueOf(3), v.second); } @Test - public void testOneCommaThree() throws Exception { + public void testOneCommaThree() { Tuple2<HttpResponse, Integer> v = FeedHandler.doCheckProtocolVersion(ONE_COMMA_THREE); - assertThat(v.first, nullValue()); - assertThat(v.second, is(3)); + assertNull(v.first); + assertEquals(Integer.valueOf(3), v.second); } @Test - public void testOneEmptyThree() throws Exception { + public void testOneEmptyThree() { Tuple2<HttpResponse, Integer> v = FeedHandler.doCheckProtocolVersion(ONE_EMPTY_THREE); - assertThat(v.first, nullValue()); - assertThat(v.second, is(3)); + assertNull(v.first); + assertEquals(Integer.valueOf(3), v.second); } @Test public void testTooLarge() throws Exception { Tuple2<HttpResponse, Integer> v = FeedHandler.doCheckProtocolVersion(TOO_LARGE_NUMBER); - assertThat(v.first, instanceOf(ErrorHttpResponse.class)); + assertTrue(v.first instanceof ErrorHttpResponse); ByteArrayOutputStream errorMsg = new ByteArrayOutputStream(); ErrorHttpResponse errorResponse = (ErrorHttpResponse) v.first; errorResponse.render(errorMsg); - assertThat(errorMsg.toString(), - is("Could not parse X-Yahoo-Feed-Protocol-Versionheader of request (values: [1000000000]). " + - "Server supports protocol versions [3]")); - assertThat(v.second, is(-1)); + assertEquals(errorMsg.toString(), + "Could not parse X-Yahoo-Feed-Protocol-Versionheader of request (values: [1000000000]). " + + "Server supports protocol versions [3]"); + assertEquals(Integer.valueOf(-1), v.second); } @Test - public void testThreeTooLarge() throws Exception { + public void testThreeTooLarge() { Tuple2<HttpResponse, Integer> v = FeedHandler.doCheckProtocolVersion(THREE_TOO_LARGE_NUMBER); - assertThat(v.first, nullValue()); - assertThat(v.second, is(3)); + assertNull(v.first); + assertEquals(Integer.valueOf(3), v.second); } @Test - public void testTwoCommaTooLarge() throws Exception { + public void testTwoCommaTooLarge() { Tuple2<HttpResponse, Integer> v = FeedHandler.doCheckProtocolVersion(THREE_COMMA_TOO_LARGE_NUMBER); - assertThat(v.first, nullValue()); - assertThat(v.second, is(3)); + assertNull(v.first); + assertEquals(Integer.valueOf(3), v.second); } @Test - public void testGarbage() throws Exception { + public void testGarbage() { Tuple2<HttpResponse, Integer> v = FeedHandler.doCheckProtocolVersion(GARBAGE); - assertThat(v.first, instanceOf(ErrorHttpResponse.class)); - assertThat(v.second, is(-1)); + assertTrue(v.first instanceof ErrorHttpResponse); + assertEquals(Integer.valueOf(-1), v.second); } } diff --git a/vespajlib/src/main/java/com/yahoo/io/TeeInputStream.java b/vespajlib/src/main/java/com/yahoo/io/TeeInputStream.java deleted file mode 100644 index c199fedf395..00000000000 --- a/vespajlib/src/main/java/com/yahoo/io/TeeInputStream.java +++ /dev/null @@ -1,100 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.io; - -import java.io.InputStream; -import java.io.OutputStream; -import java.io.IOException; - -/** - * Forwards input from a source InputStream while making a copy of it into an outputstream. - * Note that it also does read-ahead and copies up to 64K of data more than was used. - * - * @author arnej - */ -class TeeInputStream extends InputStream { - final InputStream src; - final OutputStream dst; - - static final int CAPACITY = 65536; - - byte[] buf = new byte[CAPACITY]; - int readPos = 0; - int writePos = 0; - - private int inBuf() { return writePos - readPos; } - - private void fillBuf() throws IOException { - if (readPos == writePos) { - readPos = 0; - writePos = 0; - } - if (readPos * 3 > CAPACITY) { - int had = inBuf(); - System.arraycopy(buf, readPos, buf, 0, had); - readPos = 0; - writePos = had; - } - int wantToRead = CAPACITY - writePos; - if (inBuf() > 0) { - // if we have data already, do not block, read only what is available - wantToRead = Math.min(wantToRead, src.available()); - } - if (wantToRead > 0) { - int got = src.read(buf, writePos, wantToRead); - if (got > 0) { - dst.write(buf, writePos, got); - writePos += got; - } - } - } - - /** Construct a Tee */ - public TeeInputStream(InputStream from, OutputStream to) { - super(); - this.src = from; - this.dst = to; - } - - @Override - public int available() throws IOException { - return inBuf() + src.available(); - } - - @Override - public void close() throws IOException { - src.close(); - dst.close(); - } - - @Override - public int read() throws IOException { - fillBuf(); - if (inBuf() > 0) { - int r = buf[readPos++]; - return r & 0xff; - } - return -1; - } - - @Override - public int read(byte[] b) throws IOException { - return read(b, 0, b.length); - } - - @Override - public int read(byte[] b, int off, int len) throws IOException { - if (len <= 0) { - return 0; - } - fillBuf(); - int had = inBuf(); - if (had > 0) { - len = Math.min(len, had); - System.arraycopy(buf, readPos, b, off, len); - readPos += len; - return len; - } - return -1; - } - -} diff --git a/vespajlib/src/main/java/com/yahoo/tensor/TensorType.java b/vespajlib/src/main/java/com/yahoo/tensor/TensorType.java index 5b23d5d92ae..dfbcb06c365 100644 --- a/vespajlib/src/main/java/com/yahoo/tensor/TensorType.java +++ b/vespajlib/src/main/java/com/yahoo/tensor/TensorType.java @@ -1,7 +1,6 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.tensor; -import com.google.common.collect.ImmutableList; import com.yahoo.text.Ascii7BitMatcher; import java.util.ArrayList; @@ -86,7 +85,7 @@ public class TensorType { private final Value valueType; /** Sorted list of the dimensions of this */ - private final ImmutableList<Dimension> dimensions; + private final List<Dimension> dimensions; private final TensorType mappedSubtype; @@ -94,7 +93,7 @@ public class TensorType { this.valueType = valueType; List<Dimension> dimensionList = new ArrayList<>(dimensions); Collections.sort(dimensionList); - this.dimensions = ImmutableList.copyOf(dimensionList); + this.dimensions = List.copyOf(dimensionList); if (dimensionList.stream().allMatch(d -> d.isIndexed())) mappedSubtype = empty; diff --git a/vespajlib/src/test/java/com/yahoo/collections/HashletTestCase.java b/vespajlib/src/test/java/com/yahoo/collections/HashletTestCase.java index 59f0583cbd4..ddee8147acd 100644 --- a/vespajlib/src/test/java/com/yahoo/collections/HashletTestCase.java +++ b/vespajlib/src/test/java/com/yahoo/collections/HashletTestCase.java @@ -3,7 +3,6 @@ package com.yahoo.collections; import org.junit.Test; -import static org.hamcrest.CoreMatchers.*; import static org.junit.Assert.*; public class HashletTestCase { @@ -12,14 +11,14 @@ public class HashletTestCase { public void testCopyEmptyHashlet() { Hashlet<String, Integer> hash = new Hashlet<>(); Hashlet<String, Integer> hash2 = new Hashlet<>(hash); - assertThat(hash.size(), is(0)); - assertThat(hash2.size(), is(0)); + assertEquals(0, hash.size()); + assertEquals(0, hash2.size()); hash.put("foo", 5); hash2.put("bar", 7); - assertThat(hash.get("foo"), is(5)); - assertThat(hash.get("bar"), nullValue()); - assertThat(hash2.get("foo"), nullValue()); - assertThat(hash2.get("bar"), is(7)); + assertEquals(5, hash.get("foo").intValue()); + assertNull(hash.get("bar")); + assertNull(hash2.get("foo")); + assertEquals(7, hash2.get("bar").intValue()); } private void verifyEquals(Object a, Object b) { @@ -112,17 +111,17 @@ public class HashletTestCase { hash.put("foo", 5); hash.put("bar", 7); Hashlet<String, Integer> hash2 = new Hashlet<>(hash); - assertThat(hash2.size(), is(2)); - assertThat(hash2.get("foo"), is(5)); - assertThat(hash2.get("bar"), is(7)); - assertThat(hash2.key(0), is("foo")); - assertThat(hash2.key(1), is("bar")); - assertThat(hash2.value(0), is(5)); - assertThat(hash2.value(1), is(7)); - assertThat(hash2.key(0), sameInstance(hash.key(0))); - assertThat(hash2.key(1), sameInstance(hash.key(1))); - assertThat(hash2.value(0), sameInstance(hash.value(0))); - assertThat(hash2.value(1), sameInstance(hash.value(1))); + assertEquals(2, hash2.size()); + assertEquals(5, hash2.get("foo").intValue()); + assertEquals(7, hash2.get("bar").intValue()); + assertEquals("foo", hash2.key(0)); + assertEquals("bar", hash2.key(1)); + assertEquals(5, hash2.value(0).intValue()); + assertEquals(7, hash2.value(1).intValue()); + assertSame(hash2.key(0), hash.key(0)); + assertSame(hash2.key(1), hash.key(1)); + assertSame(hash2.value(0), hash.value(0)); + assertSame(hash2.value(1), hash.value(1)); } @Test @@ -130,21 +129,21 @@ public class HashletTestCase { Hashlet<String, Integer> hash = new Hashlet<>(); hash.put("foo", 5); hash.put("bar", 7); - assertThat(hash.size(), is(2)); - assertThat(hash.get("foo"), is(5)); - assertThat(hash.get("bar"), is(7)); - assertThat(hash.key(0), is("foo")); - assertThat(hash.key(1), is("bar")); - assertThat(hash.value(0), is(5)); - assertThat(hash.value(1), is(7)); + assertEquals(2, hash.size()); + assertEquals(5, hash.get("foo").intValue()); + assertEquals(7, hash.get("bar").intValue()); + assertEquals("foo", hash.key(0)); + assertEquals("bar", hash.key(1)); + assertEquals(5, hash.value(0).intValue()); + assertEquals(7, hash.value(1).intValue()); hash.put("foo", null); - assertThat(hash.size(), is(2)); - assertThat(hash.get("foo"), nullValue()); - assertThat(hash.get("bar"), is(7)); - assertThat(hash.key(0), is("foo")); - assertThat(hash.key(1), is("bar")); - assertThat(hash.value(0), nullValue()); - assertThat(hash.value(1), is(7)); + assertEquals(2, hash.size()); + assertNull(hash.get("foo")); + assertEquals(7, hash.get("bar").intValue()); + assertEquals("foo", hash.key(0)); + assertEquals("bar", hash.key(1)); + assertNull(hash.value(0)); + assertEquals(7, hash.value(1).intValue()); } @Test @@ -155,11 +154,11 @@ public class HashletTestCase { String str = ("" + i + "_str_" + i); hash.put(str, i); } - assertThat(hash.size(), is(n)); + assertEquals(n, hash.size()); for (int i = 0; i < n; i++) { String str = ("" + i + "_str_" + i); - assertThat(hash.key(i), is(str)); - assertThat(hash.value(i), is(i)); + assertEquals(str, hash.key(i)); + assertEquals(i, hash.value(i).intValue()); } } @@ -169,17 +168,17 @@ public class HashletTestCase { Hashlet<String, Integer> hash = new Hashlet<>(); for (int i = 0; i < n; i++) { String str = ("" + i + "_str_" + i); - assertThat(hash.get(str), nullValue()); + assertNull(hash.get(str)); switch (i % 2) { - case 1: assertThat(hash.put(str, i), nullValue()); + case 1: assertNull(hash.put(str, i)); } } - assertThat(hash.size(), is(n / 2)); + assertEquals(n / 2, hash.size()); for (int i = 0; i < n; i++) { String str = ("" + i + "_str_" + i); switch (i % 2) { - case 0: assertThat(hash.get(str), nullValue()); break; - case 1: assertThat(hash.get(str), is(i)); break; + case 0: assertNull(hash.get(str)); break; + case 1: assertEquals(i, hash.get(str).intValue()); break; } } } diff --git a/vespajlib/src/test/java/com/yahoo/io/TeeInputStreamTest.java b/vespajlib/src/test/java/com/yahoo/io/TeeInputStreamTest.java deleted file mode 100644 index 7b944e92d48..00000000000 --- a/vespajlib/src/test/java/com/yahoo/io/TeeInputStreamTest.java +++ /dev/null @@ -1,101 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.io; - -import java.io.*; -import java.nio.charset.StandardCharsets; - -import org.junit.Test; -import static org.hamcrest.Matchers.greaterThan; -import static org.hamcrest.core.Is.is; -import static org.junit.Assert.*; - -/** - * @author arnej - */ -public class TeeInputStreamTest { - - @Test - public void testSimpleInput() throws IOException { - byte[] input = "very simple input".getBytes(StandardCharsets.UTF_8); - ByteArrayInputStream in = new ByteArrayInputStream(input); - ByteArrayOutputStream gotten = new ByteArrayOutputStream(); - ByteArrayOutputStream output = new ByteArrayOutputStream(); - - TeeInputStream tee = new TeeInputStream(in, gotten); - int b = tee.read(); - assertThat(b, is((int)'v')); - output.write(b); - assertThat(gotten.toString(), is("very simple input")); - for (int i = 0; i < 16; i++) { - b = tee.read(); - // System.out.println("got["+i+"]: "+(char)b); - assertThat(b, is(greaterThan(0))); - output.write(b); - } - assertThat(tee.read(), is(-1)); - assertThat(gotten.toString(), is("very simple input")); - assertThat(output.toString(), is("very simple input")); - } - - private class Generator implements Runnable { - private OutputStream dst; - public Generator(OutputStream dst) { this.dst = dst; } - @Override - public void run() { - for (int i = 0; i < 123456789; i++) { - int b = i & 0x7f; - if (b < 32) continue; - if (b > 126) b = '\n'; - try { - dst.write(b); - } catch (IOException e) { - return; - } - } - } - } - - @Test - public void testPipedInput() throws IOException { - PipedOutputStream input = new PipedOutputStream(); - PipedInputStream in = new PipedInputStream(input); - ByteArrayOutputStream gotten = new ByteArrayOutputStream(); - ByteArrayOutputStream output = new ByteArrayOutputStream(); - TeeInputStream tee = new TeeInputStream(in, gotten); - input.write("first input".getBytes(StandardCharsets.UTF_8)); - int b = tee.read(); - assertThat(b, is((int)'f')); - output.write(b); - assertThat(gotten.toString(), is("first input")); - input.write(" second input".getBytes(StandardCharsets.UTF_8)); - b = tee.read(); - assertThat(b, is((int)'i')); - output.write(b); - assertThat(gotten.toString(), is("first input second input")); - new Thread(new Generator(input)).start(); - b = tee.read(); - assertThat(b, is((int)'r')); - output.write(b); - byte[] ba = new byte[9]; - for (int i = 0; i < 12345; i++) { - b = tee.read(); - // System.out.println("got["+i+"]: "+(char)b); - assertThat(b, is(greaterThan(0))); - output.write(b); - int l = tee.read(ba); - assertThat(l, is(greaterThan(0))); - output.write(ba, 0, l); - l = tee.read(ba, 3, 3); - assertThat(l, is(greaterThan(0))); - output.write(ba, 3, l); - } - tee.close(); - String got = gotten.toString(); - // System.out.println("got length: "+got.length()); - // System.out.println("output length: "+output.toString().length()); - // System.out.println("got: "+got); - assertThat(got.length(), is(greaterThan(34567))); - assertTrue(got.startsWith(output.toString())); - } - -} diff --git a/vespajlib/src/test/java/com/yahoo/path/PathTest.java b/vespajlib/src/test/java/com/yahoo/path/PathTest.java index aa09c491586..ae4581f1e9d 100644 --- a/vespajlib/src/test/java/com/yahoo/path/PathTest.java +++ b/vespajlib/src/test/java/com/yahoo/path/PathTest.java @@ -3,9 +3,8 @@ package com.yahoo.path; import org.junit.Test; -import static org.hamcrest.core.Is.is; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertThat; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotEquals; import static org.junit.Assert.assertTrue; /** @@ -15,85 +14,85 @@ import static org.junit.Assert.assertTrue; public class PathTest { @Test public void testGetName() { - assertThat(getAbsolutePath().getName(), is("baz")); - assertThat(getRelativePath().getName(), is("baz")); - assertThat(getWithSlashes().getName(), is("baz")); - assertThat(getAppended().getName(), is("baz")); - assertThat(getOne().getName(), is("foo")); + assertEquals("baz", getAbsolutePath().getName()); + assertEquals("baz", getRelativePath().getName()); + assertEquals("baz", getWithSlashes().getName()); + assertEquals("baz", getAppended().getName()); + assertEquals("foo", getOne().getName()); } @Test public void testEquals() { - assertTrue(getAbsolutePath().equals(getAbsolutePath())); - assertTrue(getAbsolutePath().equals(getRelativePath())); - assertTrue(getAbsolutePath().equals(getWithSlashes())); - assertTrue(getAbsolutePath().equals(getAppended())); - assertFalse(getAbsolutePath().equals(getOne())); - - assertTrue(getRelativePath().equals(getAbsolutePath())); - assertTrue(getRelativePath().equals(getRelativePath())); - assertTrue(getRelativePath().equals(getWithSlashes())); - assertTrue(getRelativePath().equals(getAppended())); - assertFalse(getRelativePath().equals(getOne())); - - assertTrue(getWithSlashes().equals(getAbsolutePath())); - assertTrue(getWithSlashes().equals(getRelativePath())); - assertTrue(getWithSlashes().equals(getWithSlashes())); - assertTrue(getWithSlashes().equals(getAppended())); - assertFalse(getWithSlashes().equals(getOne())); - - assertTrue(getAppended().equals(getAbsolutePath())); - assertTrue(getAppended().equals(getRelativePath())); - assertTrue(getAppended().equals(getWithSlashes())); - assertTrue(getAppended().equals(getAppended())); - assertFalse(getAppended().equals(getOne())); - - assertFalse(getOne().equals(getAbsolutePath())); - assertFalse(getOne().equals(getRelativePath())); - assertFalse(getOne().equals(getWithSlashes())); - assertFalse(getOne().equals(getAppended())); - assertTrue(getOne().equals(getOne())); + assertEquals(getAbsolutePath(), getAbsolutePath()); + assertEquals(getAbsolutePath(), getRelativePath()); + assertEquals(getAbsolutePath(), getWithSlashes()); + assertEquals(getAbsolutePath(), getAppended()); + assertNotEquals(getAbsolutePath(), getOne()); + + assertEquals(getRelativePath(), getAbsolutePath()); + assertEquals(getRelativePath(), getRelativePath()); + assertEquals(getRelativePath(), getWithSlashes()); + assertEquals(getRelativePath(), getAppended()); + assertNotEquals(getRelativePath(), getOne()); + + assertEquals(getWithSlashes(), getAbsolutePath()); + assertEquals(getWithSlashes(), getRelativePath()); + assertEquals(getWithSlashes(), getWithSlashes()); + assertEquals(getWithSlashes(), getAppended()); + assertNotEquals(getWithSlashes(), getOne()); + + assertEquals(getAppended(), getAbsolutePath()); + assertEquals(getAppended(), getRelativePath()); + assertEquals(getAppended(), getWithSlashes()); + assertEquals(getAppended(), getAppended()); + assertNotEquals(getAppended(), getOne()); + + assertNotEquals(getOne(), getAbsolutePath()); + assertNotEquals(getOne(), getRelativePath()); + assertNotEquals(getOne(), getWithSlashes()); + assertNotEquals(getOne(), getAppended()); + assertEquals(getOne(), getOne()); } @Test public void testGetPath() { - assertThat(getAbsolutePath().getRelative(), is("foo/bar/baz")); - assertThat(getRelativePath().getRelative(), is("foo/bar/baz")); - assertThat(getWithSlashes().getRelative(), is("foo/bar/baz")); - assertThat(getAppended().getRelative(), is("foo/bar/baz")); - assertThat(getOne().getRelative(), is("foo")); + assertEquals("foo/bar/baz", getAbsolutePath().getRelative()); + assertEquals("foo/bar/baz", getRelativePath().getRelative()); + assertEquals("foo/bar/baz", getWithSlashes().getRelative()); + assertEquals("foo/bar/baz", getAppended().getRelative()); + assertEquals("foo", getOne().getRelative()); } @Test public void testGetParentPath() { - assertThat(getAbsolutePath().getParentPath().getRelative(), is("foo/bar")); - assertThat(getRelativePath().getParentPath().getRelative(), is("foo/bar")); - assertThat(getWithSlashes().getParentPath().getRelative(), is("foo/bar")); - assertThat(getAppended().getParentPath().getRelative(), is("foo/bar")); - assertThat(getOne().getParentPath().getRelative(), is("")); + assertEquals("foo/bar", getAbsolutePath().getParentPath().getRelative()); + assertEquals("foo/bar", getRelativePath().getParentPath().getRelative()); + assertEquals("foo/bar", getWithSlashes().getParentPath().getRelative()); + assertEquals("foo/bar", getAppended().getParentPath().getRelative()); + assertTrue(getOne().getParentPath().getRelative().isEmpty()); } @Test public void testGetAbsolutePath() { - assertThat(getAbsolutePath().getAbsolute(), is("/foo/bar/baz")); - assertThat(getAbsolutePath().getParentPath().getAbsolute(), is("/foo/bar")); + assertEquals("/foo/bar/baz", getAbsolutePath().getAbsolute()); + assertEquals("/foo/bar", getAbsolutePath().getParentPath().getAbsolute()); } @Test public void testEmptyPath() { - assertThat(Path.createRoot().getName(), is("")); - assertThat(Path.createRoot().getRelative(), is("")); - assertThat(Path.createRoot().getParentPath().getRelative(), is("")); + assertTrue(Path.createRoot().getName().isEmpty()); + assertTrue(Path.createRoot().getRelative().isEmpty()); + assertTrue(Path.createRoot().getParentPath().getRelative().isEmpty()); assertTrue(Path.createRoot().isRoot()); } @Test public void testDelimiters() { - assertThat(Path.fromString("foo/bar", ",").getName(), is("foo/bar")); - assertThat(Path.fromString("foo/bar", "/").getName(), is("bar")); - assertThat(Path.fromString("foo,bar", "/").getName(), is("foo,bar")); - assertThat(Path.fromString("foo,bar", ",").getName(), is("bar")); - assertThat(Path.createRoot(",").append("foo").append("bar").getRelative(), is("foo,bar")); + assertEquals("foo/bar", Path.fromString("foo/bar", ",").getName()); + assertEquals("bar", Path.fromString("foo/bar", "/").getName()); + assertEquals("foo,bar", Path.fromString("foo,bar", "/").getName()); + assertEquals("bar", Path.fromString("foo,bar", ",").getName()); + assertEquals("foo,bar", Path.createRoot(",").append("foo").append("bar").getRelative()); } @Test @@ -101,9 +100,9 @@ public class PathTest { Path p1 = getAbsolutePath(); Path p2 = getAbsolutePath(); Path p3 = p1.append(p2); - assertThat(p1.getAbsolute(), is("/foo/bar/baz")); - assertThat(p2.getAbsolute(), is("/foo/bar/baz")); - assertThat(p3.getAbsolute(), is("/foo/bar/baz/foo/bar/baz")); + assertEquals("/foo/bar/baz", p1.getAbsolute()); + assertEquals("/foo/bar/baz", p2.getAbsolute()); + assertEquals("/foo/bar/baz/foo/bar/baz", p3.getAbsolute()); } private Path getRelativePath() { diff --git a/vespajlib/src/test/java/com/yahoo/reflection/CastingTest.java b/vespajlib/src/test/java/com/yahoo/reflection/CastingTest.java index f2fa14e8958..1b781c265f4 100644 --- a/vespajlib/src/test/java/com/yahoo/reflection/CastingTest.java +++ b/vespajlib/src/test/java/com/yahoo/reflection/CastingTest.java @@ -6,9 +6,9 @@ import org.junit.Test; import java.util.Optional; import static com.yahoo.text.StringUtilities.quote; -import static org.hamcrest.core.Is.is; -import static org.junit.Assert.*; import static com.yahoo.reflection.Casting.cast; +import static org.junit.Assert.assertSame; +import static org.junit.Assert.assertTrue; public class CastingTest { @Test @@ -16,7 +16,7 @@ public class CastingTest { Object objectToCast = 12; Optional<Integer> value = cast(Integer.class, objectToCast); assertTrue("Value is not present", value.isPresent()); - assertThat(value.get(), is(objectToCast)); + assertSame(objectToCast, value.get()); } @Test diff --git a/vespajlib/src/test/java/com/yahoo/slime/BinaryFormatTestCase.java b/vespajlib/src/test/java/com/yahoo/slime/BinaryFormatTestCase.java index 19ded5c8db1..5c3126ce3cf 100644 --- a/vespajlib/src/test/java/com/yahoo/slime/BinaryFormatTestCase.java +++ b/vespajlib/src/test/java/com/yahoo/slime/BinaryFormatTestCase.java @@ -13,8 +13,7 @@ import static com.yahoo.slime.BinaryFormat.encode_double; import static com.yahoo.slime.BinaryFormat.encode_type_and_meta; import static com.yahoo.slime.BinaryFormat.encode_zigzag; import static org.hamcrest.CoreMatchers.is; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertThat; +import static org.hamcrest.MatcherAssert.assertThat; public class BinaryFormatTestCase { diff --git a/vespajlib/src/test/java/com/yahoo/slime/JsonFormatTestCase.java b/vespajlib/src/test/java/com/yahoo/slime/JsonFormatTestCase.java index 67a11f8be5a..763f8cb221c 100644 --- a/vespajlib/src/test/java/com/yahoo/slime/JsonFormatTestCase.java +++ b/vespajlib/src/test/java/com/yahoo/slime/JsonFormatTestCase.java @@ -2,15 +2,12 @@ package com.yahoo.slime; import com.yahoo.text.Utf8; -import org.junit.Ignore; import org.junit.Test; import java.io.ByteArrayOutputStream; import java.io.IOException; import java.nio.charset.StandardCharsets; -import static org.hamcrest.CoreMatchers.is; -import static org.hamcrest.MatcherAssert.assertThat; import static org.junit.Assert.assertEquals; import static org.junit.Assert.fail; @@ -108,7 +105,7 @@ public class JsonFormatTestCase { Slime slime = new Slime(); String str = null; slime.setString(str); - assertThat(slime.get().type(), is(Type.NIX)); + assertEquals(Type.NIX, slime.get().type()); verifyEncoding(slime, "null"); } @@ -117,7 +114,7 @@ public class JsonFormatTestCase { Slime slime = new Slime(); byte[] utf8 = null; slime.setString(utf8); - assertThat(slime.get().type(), is(Type.NIX)); + assertEquals(Type.NIX, slime.get().type()); verifyEncoding(slime, "null"); } @@ -125,7 +122,7 @@ public class JsonFormatTestCase { public void testNullDataNixFallback() { Slime slime = new Slime(); slime.setData(null); - assertThat(slime.get().type(), is(Type.NIX)); + assertEquals(Type.NIX, slime.get().type()); verifyEncoding(slime, "null"); } @@ -203,7 +200,7 @@ public class JsonFormatTestCase { Slime slime = new Slime(); new JsonDecoder().decode(slime, Utf8.toBytesStd(json)); Cursor a = slime.get().field("body"); - assertThat(a.asString(), is("some text&more text")); + assertEquals("some text&more text", a.asString()); } @Test @@ -226,11 +223,11 @@ public class JsonFormatTestCase { Slime slime = new Slime(); slime = new JsonDecoder().decode(slime, Utf8.toBytesStd(json)); Cursor a = slime.get().field("rules"); - assertThat(a.asString(), is(str)); + assertEquals(str, a.asString()); } @Test(expected = UnsupportedOperationException.class) - public void testThatDecodeIsNotImplemented() throws IOException { + public void testThatDecodeIsNotImplemented() { new JsonFormat(true).decode(null, null); } @@ -244,7 +241,7 @@ public class JsonFormatTestCase { slime.setString("M\u00E6L"); ByteArrayOutputStream a = new ByteArrayOutputStream(); new JsonFormat(true).encode(a, slime); - String val = new String(a.toByteArray(), "UTF-8"); + String val = a.toString(StandardCharsets.UTF_8); assertEquals("\"M\u00E6L\"", val); } @@ -252,7 +249,7 @@ public class JsonFormatTestCase { try { ByteArrayOutputStream a = new ByteArrayOutputStream(); new JsonFormat(compact).encode(a, slime); - assertEquals(expected, new String(a.toByteArray(), StandardCharsets.UTF_8)); + assertEquals(expected, a.toString(StandardCharsets.UTF_8)); } catch (Exception e) { fail("Exception thrown when encoding slime: " + e.getMessage()); } @@ -276,7 +273,7 @@ public class JsonFormatTestCase { slime.setDouble(value); ByteArrayOutputStream a = new ByteArrayOutputStream(); new JsonFormat(true).encode(a, slime); - return new String(a.toByteArray(), StandardCharsets.UTF_8); + return a.toString(StandardCharsets.UTF_8); } catch (Exception e) { return ""; } diff --git a/vespajlib/src/test/java/com/yahoo/slime/SlimeTestCase.java b/vespajlib/src/test/java/com/yahoo/slime/SlimeTestCase.java index 33abfe39345..6ee8fb6c7e2 100644 --- a/vespajlib/src/test/java/com/yahoo/slime/SlimeTestCase.java +++ b/vespajlib/src/test/java/com/yahoo/slime/SlimeTestCase.java @@ -3,8 +3,9 @@ package com.yahoo.slime; import org.junit.Test; -import static org.junit.Assert.assertThat; -import static org.hamcrest.CoreMatchers.*; +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.CoreMatchers.is; +import static org.hamcrest.CoreMatchers.sameInstance; public class SlimeTestCase { diff --git a/vespajlib/src/test/java/com/yahoo/stream/CustomCollectorsTest.java b/vespajlib/src/test/java/com/yahoo/stream/CustomCollectorsTest.java index 6a0e6366775..a5b2accec48 100644 --- a/vespajlib/src/test/java/com/yahoo/stream/CustomCollectorsTest.java +++ b/vespajlib/src/test/java/com/yahoo/stream/CustomCollectorsTest.java @@ -3,9 +3,7 @@ package com.yahoo.stream; import com.google.common.collect.Lists; import com.yahoo.stream.CustomCollectors.DuplicateKeyException; -import org.junit.Rule; import org.junit.Test; -import org.junit.rules.ExpectedException; import java.util.HashMap; import java.util.List; @@ -16,15 +14,13 @@ import static com.yahoo.stream.CustomCollectors.toCustomMap; import static com.yahoo.stream.CustomCollectors.toLinkedMap; import static java.util.function.Function.identity; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.fail; /** * @author gjoranv */ public class CustomCollectorsTest { - @Rule - public ExpectedException thrown = ExpectedException.none(); - @Test public void linked_map_collector_returns_map_with_insertion_order() { List<String> stringList = numberList(); @@ -48,8 +44,12 @@ public class CustomCollectorsTest { public void custom_map_collector_throws_exception_upon_duplicate_keys() { List<String> duplicates = Lists.newArrayList("same", "same"); - thrown.expect(DuplicateKeyException.class); - duplicates.stream().collect(toCustomMap(Function.identity(), Function.identity(), HashMap::new)); + try { + duplicates.stream().collect(toCustomMap(Function.identity(), Function.identity(), HashMap::new)); + fail(); + } catch (DuplicateKeyException e) { + + } } private static List<String> numberList() { diff --git a/vespajlib/src/test/java/com/yahoo/tensor/TensorTypeTestCase.java b/vespajlib/src/test/java/com/yahoo/tensor/TensorTypeTestCase.java index c5009fa3ab2..738697d4521 100644 --- a/vespajlib/src/test/java/com/yahoo/tensor/TensorTypeTestCase.java +++ b/vespajlib/src/test/java/com/yahoo/tensor/TensorTypeTestCase.java @@ -3,10 +3,8 @@ package com.yahoo.tensor; import org.junit.Test; -import static org.hamcrest.Matchers.containsString; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertThat; import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; @@ -117,7 +115,7 @@ public class TensorTypeTestCase { TensorType.fromSpec(typeSpec); fail("Expected exception to be thrown with message: '" + messageSubstring + "'"); } catch (IllegalArgumentException e) { - assertThat(e.getMessage(), containsString(messageSubstring)); + assertTrue(e.getMessage().contains(messageSubstring)); } } diff --git a/vespajlib/src/test/java/com/yahoo/tensor/TypeResolverTestCase.java b/vespajlib/src/test/java/com/yahoo/tensor/TypeResolverTestCase.java index 055603c5f59..82b7e73fb20 100644 --- a/vespajlib/src/test/java/com/yahoo/tensor/TypeResolverTestCase.java +++ b/vespajlib/src/test/java/com/yahoo/tensor/TypeResolverTestCase.java @@ -9,10 +9,7 @@ import java.util.List; import org.junit.Test; import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertThat; import static org.junit.Assert.assertTrue; -import static org.junit.Assert.fail; /** * @author arnej diff --git a/vespajlib/src/test/java/com/yahoo/text/LowercaseTestCase.java b/vespajlib/src/test/java/com/yahoo/text/LowercaseTestCase.java index 70374adbb16..ca3b316c11b 100644 --- a/vespajlib/src/test/java/com/yahoo/text/LowercaseTestCase.java +++ b/vespajlib/src/test/java/com/yahoo/text/LowercaseTestCase.java @@ -6,9 +6,7 @@ import org.junit.Test; import java.util.Locale; -import static org.hamcrest.CoreMatchers.equalTo; import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertThat; /** * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a> @@ -20,32 +18,32 @@ public class LowercaseTestCase { public void testAZ() { { String lowercase = Lowercase.toLowerCase("ABCDEFGHIJKLMNOPQRSTUVWXYZ"); - assertThat(lowercase, equalTo("abcdefghijklmnopqrstuvwxyz")); + assertEquals("abcdefghijklmnopqrstuvwxyz", lowercase); } { String lowercase = Lowercase.toLowerCase("abcdefghijklmnopqrstuvwxyz"); - assertThat(lowercase, equalTo("abcdefghijklmnopqrstuvwxyz")); + assertEquals("abcdefghijklmnopqrstuvwxyz", lowercase); } { String lowercase = Lowercase.toLowerCase("AbCDEfGHIJklmnoPQRStuvwXyz"); - assertThat(lowercase, equalTo("abcdefghijklmnopqrstuvwxyz")); + assertEquals("abcdefghijklmnopqrstuvwxyz", lowercase); } { String lowercase = Lowercase.toLowerCase("@+#"); - assertThat(lowercase, equalTo("@+#")); + assertEquals("@+#", lowercase); } { String lowercase = Lowercase.toLowerCase("[]"); - assertThat(lowercase, equalTo("[]")); + assertEquals("[]", lowercase); } { String lowercase = Lowercase.toLowerCase("{}"); - assertThat(lowercase, equalTo("{}")); + assertEquals("{}", lowercase); } { String lowercase = Lowercase.toLowerCase("\u00cd\u00f4"); - assertThat(lowercase, equalTo("\u00ed\u00f4")); + assertEquals("\u00ed\u00f4", lowercase); } } diff --git a/vespajlib/src/test/java/com/yahoo/text/StringUtilitiesTest.java b/vespajlib/src/test/java/com/yahoo/text/StringUtilitiesTest.java index b1ec37cb7d7..b68cca6e54f 100644 --- a/vespajlib/src/test/java/com/yahoo/text/StringUtilitiesTest.java +++ b/vespajlib/src/test/java/com/yahoo/text/StringUtilitiesTest.java @@ -1,12 +1,15 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.text; -import java.util.Arrays; +import java.util.List; import org.junit.Test; -import static org.hamcrest.core.Is.is; -import static org.junit.Assert.*; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotSame; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertSame; +import static org.junit.jupiter.api.Assertions.assertTrue; public class StringUtilitiesTest { @@ -40,7 +43,7 @@ public class StringUtilitiesTest { @Test public void testImplode() { - assertEquals(StringUtilities.implode(null, null), null); + assertNull(StringUtilities.implode(null, null)); assertEquals(StringUtilities.implode(new String[0], null), ""); assertEquals(StringUtilities.implode(new String[] {"foo"}, null), "foo"); assertEquals(StringUtilities.implode(new String[] {"foo"}, "asdfsdfsadfsadfasdfs"), "foo"); @@ -55,17 +58,17 @@ public class StringUtilitiesTest { @Test public void testImplodeMultiline() { - assertEquals(StringUtilities.implodeMultiline(Arrays.asList("foo", "bar")), "foo\nbar"); - assertEquals(StringUtilities.implodeMultiline(Arrays.asList("")), ""); - assertEquals(StringUtilities.implodeMultiline(null), null); - assertEquals(StringUtilities.implodeMultiline(Arrays.asList("\n")), "\n"); + assertEquals(StringUtilities.implodeMultiline(List.of("foo", "bar")), "foo\nbar"); + assertEquals(StringUtilities.implodeMultiline(List.of("")), ""); + assertNull(StringUtilities.implodeMultiline(null)); + assertEquals(StringUtilities.implodeMultiline(List.of("\n")), "\n"); } @Test public void testTruncation() { String a = "abbc"; - assertTrue(a == StringUtilities.truncateSequencesIfNecessary(a, 2)); - assertTrue(a != StringUtilities.truncateSequencesIfNecessary(a, 1)); + assertSame(a, StringUtilities.truncateSequencesIfNecessary(a, 2)); + assertNotSame(a, StringUtilities.truncateSequencesIfNecessary(a, 1)); assertEquals("abc", StringUtilities.truncateSequencesIfNecessary(a, 1)); assertEquals("abc", StringUtilities.truncateSequencesIfNecessary("aabbccc", 1)); assertEquals("abc", StringUtilities.truncateSequencesIfNecessary("abcc", 1)); @@ -77,9 +80,9 @@ public class StringUtilitiesTest { @Test public void testStripSuffix() { - assertThat(StringUtilities.stripSuffix("abc.def", ".def"), is("abc")); - assertThat(StringUtilities.stripSuffix("abc.def", ""), is("abc.def")); - assertThat(StringUtilities.stripSuffix("", ".def"), is("")); - assertThat(StringUtilities.stripSuffix("", ""), is("")); + assertEquals("abc", StringUtilities.stripSuffix("abc.def", ".def")); + assertEquals("abc.def", StringUtilities.stripSuffix("abc.def", "")); + assertTrue(StringUtilities.stripSuffix("", ".def").isEmpty()); + assertTrue(StringUtilities.stripSuffix("", "").isEmpty()); } } diff --git a/vespajlib/src/test/java/com/yahoo/text/Utf8TestCase.java b/vespajlib/src/test/java/com/yahoo/text/Utf8TestCase.java index 9e43cb3c6ea..926d19f433f 100644 --- a/vespajlib/src/test/java/com/yahoo/text/Utf8TestCase.java +++ b/vespajlib/src/test/java/com/yahoo/text/Utf8TestCase.java @@ -18,9 +18,9 @@ import static com.yahoo.text.Utf8.calculateBytePositions; import static com.yahoo.text.Utf8.calculateStringPositions; import static org.hamcrest.CoreMatchers.equalTo; import static org.hamcrest.CoreMatchers.is; +import static org.hamcrest.MatcherAssert.assertThat; import static org.junit.Assert.assertArrayEquals; import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertThat; import static org.junit.Assert.assertTrue; /** diff --git a/vespajlib/src/test/java/com/yahoo/vespa/objects/FieldBaseTestCase.java b/vespajlib/src/test/java/com/yahoo/vespa/objects/FieldBaseTestCase.java index 40792ac05c1..32b87a4b373 100644 --- a/vespajlib/src/test/java/com/yahoo/vespa/objects/FieldBaseTestCase.java +++ b/vespajlib/src/test/java/com/yahoo/vespa/objects/FieldBaseTestCase.java @@ -6,9 +6,7 @@ import org.junit.Test; import static org.hamcrest.CoreMatchers.not; import static org.hamcrest.CoreMatchers.equalTo; import static org.hamcrest.CoreMatchers.is; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertThat; -import static org.junit.Assert.assertTrue; +import static org.hamcrest.MatcherAssert.assertThat; /** * @author arnej27959 diff --git a/vespajlib/src/test/java/com/yahoo/vespa/objects/ObjectDumperTestCase.java b/vespajlib/src/test/java/com/yahoo/vespa/objects/ObjectDumperTestCase.java index 7694548344c..ab0da1b7d33 100644 --- a/vespajlib/src/test/java/com/yahoo/vespa/objects/ObjectDumperTestCase.java +++ b/vespajlib/src/test/java/com/yahoo/vespa/objects/ObjectDumperTestCase.java @@ -3,13 +3,9 @@ package com.yahoo.vespa.objects; import org.junit.Test; -import java.nio.ByteBuffer; - -import static org.hamcrest.CoreMatchers.equalTo; -import static org.hamcrest.CoreMatchers.is; import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertThat; -import static org.junit.Assert.assertTrue; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.fail; /** * @author arnej27959 @@ -37,7 +33,7 @@ public class ObjectDumperTestCase { oneOD.visit("biggie", b); - assertThat(oneOD.toString(), equalTo( + assertEquals( "biggie: BigIdClass {\n"+ " classId: 42\n"+ " : <NULL>\n"+ @@ -69,11 +65,11 @@ public class ObjectDumperTestCase { " [3]: 4\n"+ " [4]: 5\n"+ " }\n"+ -"}\n")); +"}\n", oneOD.toString()); ObjectDumper defOD = new ObjectDumper(); defOD.visit("", b); - assertThat(b.toString(), equalTo(b.toString())); + assertEquals(b.toString(), b.toString()); } @Test @@ -100,7 +96,7 @@ public class ObjectDumperTestCase { defOD.visit("s5", s5); oneOD.visit("s6", s5); - assertThat(defOD.toString(), is("s5: FooBarIdClass {\n"+ + assertEquals("s5: FooBarIdClass {\n"+ " classId: 17\n"+ " foo: 'def-foo'\n"+ " bar: 42\n"+ @@ -109,8 +105,8 @@ public class ObjectDumperTestCase { " [1]: 42\n"+ " [2]: 666\n"+ " }\n"+ - "}\n")); - assertThat(oneOD.toString(), is("s6: FooBarIdClass {\n"+ + "}\n", defOD.toString()); + assertEquals("s6: FooBarIdClass {\n"+ " classId: 17\n"+ " foo: 'def-foo'\n"+ " bar: 42\n"+ @@ -119,31 +115,29 @@ public class ObjectDumperTestCase { " [1]: 42\n"+ " [2]: 666\n"+ " }\n"+ - "}\n")); + "}\n", oneOD.toString()); } @Test public void testRegistry() { - assertThat(FooBarIdClass.classId, is(17)); + assertEquals(17, FooBarIdClass.classId); int x = Identifiable.registerClass(17, FooBarIdClass.class); - assertThat(x, is(17)); - boolean caught = false; + assertEquals(17, x); try { - x = Identifiable.registerClass(17, SomeIdClass.class); + x = Identifiable.registerClass(17, SomeIdClass.class); + fail(); } catch (IllegalArgumentException e) { - caught = true; - assertThat(e.getMessage(), is( + assertEquals(e.getMessage(), "Can not register class 'class com.yahoo.vespa.objects.SomeIdClass' with id 17,"+ -" because it already maps to class 'class com.yahoo.vespa.objects.FooBarIdClass'.")); +" because it already maps to class 'class com.yahoo.vespa.objects.FooBarIdClass'."); } - assertThat(x, is(17)); - assertThat(caught, is(true)); + assertEquals(17, x); Identifiable s7 = Identifiable.createFromId(17); ObjectDumper defOD = new ObjectDumper(); defOD.visit("s7", s7); - assertThat(defOD.toString(), is("s7: FooBarIdClass {\n"+ + assertEquals("s7: FooBarIdClass {\n"+ " classId: 17\n"+ " foo: 'def-foo'\n"+ " bar: 42\n"+ @@ -152,10 +146,9 @@ public class ObjectDumperTestCase { " [1]: 42\n"+ " [2]: 666\n"+ " }\n"+ - "}\n")); + "}\n", defOD.toString()); - Identifiable nsi = Identifiable.createFromId(717273); - assertThat(nsi, is((Identifiable)null)); + assertNull(Identifiable.createFromId(717273)); } } diff --git a/vespajlib/src/test/java/com/yahoo/vespa/objects/SerializeTestCase.java b/vespajlib/src/test/java/com/yahoo/vespa/objects/SerializeTestCase.java index ce4f6846248..31c43f6008d 100644 --- a/vespajlib/src/test/java/com/yahoo/vespa/objects/SerializeTestCase.java +++ b/vespajlib/src/test/java/com/yahoo/vespa/objects/SerializeTestCase.java @@ -8,7 +8,7 @@ import org.junit.Test; import static org.hamcrest.CoreMatchers.equalTo; import static org.hamcrest.CoreMatchers.is; -import static org.junit.Assert.assertThat; +import static org.hamcrest.MatcherAssert.assertThat; /** * @author arnej27959 diff --git a/vespalib/CMakeLists.txt b/vespalib/CMakeLists.txt index 400c1ec5d1a..9f98d23d6a9 100644 --- a/vespalib/CMakeLists.txt +++ b/vespalib/CMakeLists.txt @@ -153,7 +153,6 @@ vespa_define_module( src/tests/visit_ranges src/tests/invokeservice src/tests/wakeup - src/tests/websocket src/tests/zcurve LIBS @@ -181,5 +180,4 @@ vespa_define_module( src/vespa/vespalib/time src/vespa/vespalib/trace src/vespa/vespalib/util - src/vespa/vespalib/websocket ) diff --git a/vespalib/src/tests/websocket/.gitignore b/vespalib/src/tests/websocket/.gitignore deleted file mode 100644 index 379d76b3ece..00000000000 --- a/vespalib/src/tests/websocket/.gitignore +++ /dev/null @@ -1,3 +0,0 @@ -/websocket_server -vespalib_websocket_test_app -vespalib_websocket_server_app diff --git a/vespalib/src/tests/websocket/CMakeLists.txt b/vespalib/src/tests/websocket/CMakeLists.txt deleted file mode 100644 index 8581523a98e..00000000000 --- a/vespalib/src/tests/websocket/CMakeLists.txt +++ /dev/null @@ -1,14 +0,0 @@ -# Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -vespa_add_executable(vespalib_websocket_test_app TEST - SOURCES - websocket_test.cpp - DEPENDS - vespalib -) -vespa_add_test(NAME vespalib_websocket_test_app COMMAND vespalib_websocket_test_app) -vespa_add_executable(vespalib_websocket_server_app - SOURCES - websocket_server.cpp - DEPENDS - vespalib -) diff --git a/vespalib/src/tests/websocket/favicon.ico b/vespalib/src/tests/websocket/favicon.ico Binary files differdeleted file mode 100644 index 22f61482bd0..00000000000 --- a/vespalib/src/tests/websocket/favicon.ico +++ /dev/null diff --git a/vespalib/src/tests/websocket/index.html b/vespalib/src/tests/websocket/index.html deleted file mode 100644 index 144d42be14d..00000000000 --- a/vespalib/src/tests/websocket/index.html +++ /dev/null @@ -1,5 +0,0 @@ -<!-- Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. --> -<html> -<h1>Websocket server</h1> -<a href="test.html">run test</a> -</html> diff --git a/vespalib/src/tests/websocket/test.html b/vespalib/src/tests/websocket/test.html deleted file mode 100644 index f898cfb8da3..00000000000 --- a/vespalib/src/tests/websocket/test.html +++ /dev/null @@ -1,70 +0,0 @@ -<!DOCTYPE html> -<!-- Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. --> - -<meta charset="utf-8" /> - -<title>WebSocket Test</title> - -<script language="javascript" type="text/javascript"> - - var wsUri = "ws://[SELF]/echo"; - var output; - - function init() - { - output = document.getElementById("output"); - testWebSocket(); - } - - function testWebSocket() - { - websocket = new WebSocket(wsUri); - websocket.onopen = function(evt) { onOpen(evt) }; - websocket.onclose = function(evt) { onClose(evt) }; - websocket.onmessage = function(evt) { onMessage(evt) }; - websocket.onerror = function(evt) { onError(evt) }; - } - - function onOpen(evt) - { - writeToScreen("CONNECTED"); - doSend("WebSocket rocks"); - } - - function onClose(evt) - { - writeToScreen("DISCONNECTED"); - } - - function onMessage(evt) - { - writeToScreen('<span style="color: blue;">RESPONSE: ' + evt.data+'</span>'); - websocket.close(); - } - - function onError(evt) - { - writeToScreen('<span style="color: red;">ERROR:</span> ' + evt.data); - } - - function doSend(message) - { - writeToScreen("SENT: " + message); - websocket.send(message); - } - - function writeToScreen(message) - { - var pre = document.createElement("p"); - pre.style.wordWrap = "break-word"; - pre.innerHTML = message; - output.appendChild(pre); - } - - window.addEventListener("load", init, false); - -</script> - -<h2>WebSocket Test</h2> - -<div id="output"></div> diff --git a/vespalib/src/tests/websocket/websocket_server.cpp b/vespalib/src/tests/websocket/websocket_server.cpp deleted file mode 100644 index 7a9cf46cb5f..00000000000 --- a/vespalib/src/tests/websocket/websocket_server.cpp +++ /dev/null @@ -1,51 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/vespalib/testkit/test_kit.h> -#include <vespa/vespalib/websocket/websocket_server.h> -#include <vespa/vespalib/util/host_name.h> -#include <vespa/vespalib/util/signalhandler.h> -#include <vespa/vespalib/io/mapped_file_input.h> -#include <thread> -#include <chrono> - -using namespace vespalib; - -vespalib::string read_file(const vespalib::string &file_name) { - return MappedFileInput(file_name).get().make_string(); -} - -vespalib::string find_content_type(const vespalib::string &file_name) { - if (ends_with(file_name, ".html")) { - return "text/html"; - } - if (ends_with(file_name, ".js")) { - return "text/javascript"; - } - if (ends_with(file_name, ".ico")) { - return "image/x-icon"; - } - return "text/plain"; -} - -int main(int, char **) { - ws::WebsocketServer::StaticRepo repo; - for (vespalib::string file_name: { "index.html", "test.html", "favicon.ico" }) { - vespalib::string content = read_file(file_name); - vespalib::string content_type = find_content_type(file_name); - if (!content.empty()) { - fprintf(stderr, "loaded file: %s as content %s\n", file_name.c_str(), content_type.c_str()); - repo.emplace("/" + file_name, ws::WebsocketServer::StaticPage{content_type, content}); - } - } - ws::WebsocketServer server(0, std::move(repo)); - int port = server.port(); - SignalHandler::INT.hook(); - fprintf(stderr, "running websocket server at http://%s:%d/index.html\n", - HostName::get().c_str(), port); - fprintf(stderr, "use ^C (SIGINT) to exit\n"); - while (!SignalHandler::INT.check()) { - std::this_thread::sleep_for(std::chrono::seconds(1)); - } - fprintf(stderr, "exiting...\n"); - kill(getpid(), SIGTERM); - return 0; -} diff --git a/vespalib/src/tests/websocket/websocket_test.cpp b/vespalib/src/tests/websocket/websocket_test.cpp deleted file mode 100644 index 5b346df4243..00000000000 --- a/vespalib/src/tests/websocket/websocket_test.cpp +++ /dev/null @@ -1,117 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/vespalib/testkit/test_kit.h> -#include <vespa/vespalib/net/socket_spec.h> -#include <vespa/vespalib/websocket/handler.h> -#include <vespa/vespalib/websocket/acceptor.h> -#include <vespa/vespalib/websocket/key.h> -#include <vespa/vespalib/websocket/buffer.h> -#include <vespa/vespalib/util/gate.h> - -using namespace vespalib; -using namespace vespalib::ws; - -template <typename T> -struct Receptor : vespalib::ws::Handler<T> { - std::unique_ptr<T> obj; - vespalib::Gate gate; - ~Receptor(); - void handle(std::unique_ptr<T> t) override { - obj = std::move(t); - gate.countDown(); - } -}; - -template <typename T> -Receptor<T>::~Receptor() = default; - -vespalib::string read_bytes(Socket &socket, size_t wanted_bytes) { - char tmp[64]; - vespalib::string result; - while (result.size() < wanted_bytes) { - size_t read_size = std::min(sizeof(tmp), wanted_bytes - result.size()); - size_t read_result = socket.read(tmp, read_size); - if (static_cast<ssize_t>(read_result) <= 0) { - return result; - } - result.append(tmp, read_result); - } - return result; -} - -void verify_socket_io(bool is_server, Socket &socket) { - vespalib::string server_message = "hello, this is the server speaking"; - vespalib::string client_message = "please pick up, I need to talk to you"; - if(is_server) { - socket.write(server_message.data(), server_message.size()); - vespalib::string read = read_bytes(socket, client_message.size()); - EXPECT_EQUAL(client_message, read); - } else { - socket.write(client_message.data(), client_message.size()); - vespalib::string read = read_bytes(socket, server_message.size()); - EXPECT_EQUAL(server_message, read); - } -} - -void verify_socket_io_async(Socket &server, Socket &client) { - std::thread server_thread(verify_socket_io, true, std::ref(server)); - std::thread client_thread(verify_socket_io, false, std::ref(client)); - server_thread.join(); - client_thread.join(); -} - -void check_buffer_stats(const Buffer &buffer, size_t dead, size_t used, size_t free) { - EXPECT_EQUAL(dead, buffer.dead()); - EXPECT_EQUAL(used, buffer.used()); - EXPECT_EQUAL(free, buffer.free()); -} - -TEST("require that basic reserve/commit/obtain/evict buffer cycle works") { - Buffer buffer; - check_buffer_stats(buffer, 0, 0, 0); - char *a = buffer.reserve(1); - check_buffer_stats(buffer, 0, 0, 1); - *a = 'x'; - buffer.commit(1); - check_buffer_stats(buffer, 0, 1, 0); - EXPECT_EQUAL('x', *buffer.obtain()); - check_buffer_stats(buffer, 0, 1, 0); - buffer.evict(1); - check_buffer_stats(buffer, 1, 0, 0); -} - -TEST("require that buffer moves contained data when more space is needed") { - Buffer buffer; - memcpy(buffer.reserve(3), "xyz", 3); - buffer.commit(3); - EXPECT_EQUAL('x', *buffer.obtain()); - buffer.evict(1); - EXPECT_EQUAL('y', *buffer.obtain()); - check_buffer_stats(buffer, 1, 2, 0); - buffer.reserve(1); - check_buffer_stats(buffer, 0, 2, 1); - EXPECT_EQUAL('y', *buffer.obtain()); - buffer.evict(1); - EXPECT_EQUAL('z', *buffer.obtain()); - check_buffer_stats(buffer, 1, 1, 1); - buffer.reserve(3); - check_buffer_stats(buffer, 0, 1, 3); - EXPECT_EQUAL('z', *buffer.obtain()); -} - -TEST("require that an acceptor can accept connections asynchronously") { - Receptor<Socket> server; - Acceptor acceptor(0, server); - Socket::UP client = SimpleSocket::connect(SocketSpec::from_port(acceptor.port())); - server.gate.await(60s); - ASSERT_TRUE(server.obj.get() != nullptr); - ASSERT_TRUE(client.get() != nullptr); - TEST_DO(verify_socket_io_async(*server.obj, *client)); -} - -TEST("require that websocket accept tokens are generated correctly") { - vespalib::string key("dGhlIHNhbXBsZSBub25jZQ=="); - vespalib::string accept_token("s3pPLMBiTxaQ9kYGzzhZRbK+xOo="); - EXPECT_EQUAL(accept_token, Key::accept(key)); -} - -TEST_MAIN() { TEST_RUN_ALL(); } diff --git a/vespalib/src/vespa/vespalib/CMakeLists.txt b/vespalib/src/vespa/vespalib/CMakeLists.txt index 738e0c6c766..765ed0d5634 100644 --- a/vespalib/src/vespa/vespalib/CMakeLists.txt +++ b/vespalib/src/vespa/vespalib/CMakeLists.txt @@ -24,7 +24,6 @@ vespa_add_library(vespalib $<TARGET_OBJECTS:vespalib_vespalib_time> $<TARGET_OBJECTS:vespalib_vespalib_trace> $<TARGET_OBJECTS:vespalib_vespalib_util> - $<TARGET_OBJECTS:vespalib_vespalib_websocket> INSTALL lib64 DEPENDS ${VESPA_GCC_LIB} diff --git a/vespalib/src/vespa/vespalib/websocket/CMakeLists.txt b/vespalib/src/vespa/vespalib/websocket/CMakeLists.txt deleted file mode 100644 index 1aee013d2bc..00000000000 --- a/vespalib/src/vespa/vespalib/websocket/CMakeLists.txt +++ /dev/null @@ -1,13 +0,0 @@ -# Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -vespa_add_library(vespalib_vespalib_websocket OBJECT - SOURCES - acceptor.cpp - buffer.cpp - connection.cpp - frame.cpp - handler.cpp - key.cpp - request.cpp - websocket_server.cpp - DEPENDS -) diff --git a/vespalib/src/vespa/vespalib/websocket/README b/vespalib/src/vespa/vespalib/websocket/README deleted file mode 100644 index 5ecb11cbb12..00000000000 --- a/vespalib/src/vespa/vespalib/websocket/README +++ /dev/null @@ -1,22 +0,0 @@ -NOTE: - -(0) if you use this code and something breaks; it is your own fault. - -(1) this code is experimental and will probably undergo large changes -before it is ready for production use. - -(2) please do not use this code in your applications just yet. - -(3) if you have any questions/suggestions about the code, please -contact havardpe@yahoo-inc.com - -(4) the goal of this library: -- accept websocket connections for application resources -- serve simple HTTP GET requests (read only) -- negotiate custom binary connections (fs4 packet protocol, fnet rpc) - -(5) what to build on top: -- Remote Slime Message Passing over websockets (next-gen rpc) -- back-end state API serving -- json (text) binding to slime messaging (rpc from browser javascript) -- HTTP tunneling of application data (cross-colo feeding) diff --git a/vespalib/src/vespa/vespalib/websocket/acceptor.cpp b/vespalib/src/vespa/vespalib/websocket/acceptor.cpp deleted file mode 100644 index a8fbb0e4231..00000000000 --- a/vespalib/src/vespa/vespalib/websocket/acceptor.cpp +++ /dev/null @@ -1,34 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#include "acceptor.h" -#include <vespa/vespalib/net/socket_spec.h> -#include <functional> - -namespace vespalib::ws { - -void -Acceptor::accept_main(Handler<Socket> &socket_handler) -{ - while (!_is_closed) { - SocketHandle handle = _server_socket.accept(); - if (handle.valid()) { - socket_handler.handle(std::make_unique<SimpleSocket>(std::move(handle))); - } - } -} - -Acceptor::Acceptor(int port_in, Handler<Socket> &socket_handler) - : _server_socket(port_in), - _is_closed(false), - _accept_thread(&Acceptor::accept_main, this, std::ref(socket_handler)) -{ -} - -Acceptor::~Acceptor() -{ - _server_socket.shutdown(); - _is_closed = true; - _accept_thread.join(); -} - -} // namespace vespalib::ws diff --git a/vespalib/src/vespa/vespalib/websocket/acceptor.h b/vespalib/src/vespa/vespalib/websocket/acceptor.h deleted file mode 100644 index 711c9684161..00000000000 --- a/vespalib/src/vespa/vespalib/websocket/acceptor.h +++ /dev/null @@ -1,29 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#pragma once - -#include "handler.h" -#include <vespa/vespalib/net/socket.h> -#include <vespa/vespalib/net/server_socket.h> -#include <thread> -#include <atomic> - -namespace vespalib { -namespace ws { - -class Acceptor { -private: - ServerSocket _server_socket; - std::atomic<bool> _is_closed; - std::thread _accept_thread; - - void accept_main(Handler<Socket> &socket_handler); - -public: - Acceptor(int port_in, Handler<Socket> &socket_handler); - ~Acceptor(); - int port() { return _server_socket.address().port(); } -}; - -} // namespace vespalib::ws -} // namespace vespalib diff --git a/vespalib/src/vespa/vespalib/websocket/buffer.cpp b/vespalib/src/vespa/vespalib/websocket/buffer.cpp deleted file mode 100644 index c35fd4a3597..00000000000 --- a/vespalib/src/vespa/vespalib/websocket/buffer.cpp +++ /dev/null @@ -1,18 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#include "buffer.h" - -namespace vespalib::ws { - -void -Buffer::ensure_free(size_t bytes) -{ - memmove(&_data[0], &_data[_read_pos], used()); - _write_pos -= _read_pos; - _read_pos = 0; - if (free() < bytes) { - _data.resize(_write_pos + bytes); - } -} - -} // namespace vespalib::ws diff --git a/vespalib/src/vespa/vespalib/websocket/buffer.h b/vespalib/src/vespa/vespalib/websocket/buffer.h deleted file mode 100644 index 382c60d0b0c..00000000000 --- a/vespalib/src/vespa/vespalib/websocket/buffer.h +++ /dev/null @@ -1,46 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - - -#pragma once - -#include <vespa/vespalib/stllike/string.h> -#include <vector> - -namespace vespalib { -namespace ws { - -class Buffer -{ -private: - std::vector<char> _data; - size_t _read_pos; - size_t _write_pos; - void ensure_free(size_t bytes); -public: - Buffer() : _data(), _read_pos(0), _write_pos(0) {} - void clear() { - _read_pos = 0; - _write_pos = 0; - } - size_t dead() const { return _read_pos; } - size_t used() const { return (_write_pos - _read_pos); } - size_t free() const { return (_data.size() - _write_pos); } - bool has_next() const { return (used() > 0); } - char next() { return _data[_read_pos++]; } - void push(char value) { - *reserve(1) = value; - commit(1); - } - const char *obtain() const { return &_data[_read_pos]; } - void evict(size_t bytes) { _read_pos += bytes; } - char *reserve(size_t bytes) { - if (free() < bytes) { - ensure_free(bytes); - } - return &_data[_write_pos]; - } - void commit(size_t bytes) { _write_pos += bytes; } -}; - -} // namespace vespalib::ws -} // namespace vespalib diff --git a/vespalib/src/vespa/vespalib/websocket/connection.cpp b/vespalib/src/vespa/vespalib/websocket/connection.cpp deleted file mode 100644 index 173839f376b..00000000000 --- a/vespalib/src/vespa/vespalib/websocket/connection.cpp +++ /dev/null @@ -1,207 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#include "connection.h" -#include <vespa/vespalib/util/size_literals.h> -#include <cstdarg> -#include <cassert> - -namespace vespalib::ws { - -namespace { - -void stripCR(vespalib::string &dst) { - if (!dst.empty() && dst[dst.size() - 1] == '\r') { - dst.resize(dst.size() - 1); - } -} - -Frame::Type type_from_opcode(char opcode) { - switch (opcode) { - case 0x0: return Frame::Type::NONE; - case 0x1: return Frame::Type::TEXT; - case 0x2: return Frame::Type::DATA; - case 0x8: return Frame::Type::PING; - case 0x9: return Frame::Type::PONG; - case 0xa: return Frame::Type::CLOSE; - default: return Frame::Type::INVALID; - } -} - -char opcode_from_type(Frame::Type type) { - switch (type) { - case Frame::Type::NONE: return 0x0; - case Frame::Type::TEXT: return 0x1; - case Frame::Type::DATA: return 0x2; - case Frame::Type::PING: return 0x8; - case Frame::Type::PONG: return 0x9; - case Frame::Type::CLOSE: return 0xa; - default: return 0xf; - } -} - -} // namespace vespalib::ws::<unnamed> - - -Connection::Connection(Socket::UP socket) - : _socket(std::move(socket)), - _input(), - _output() -{ -} - -bool -Connection::fill_input(size_t min_bytes) -{ - while (_input.used() < min_bytes) { - size_t max_read = (8_Ki); - char *ptr = _input.reserve(max_read); - ssize_t read_res = _socket->read(ptr, max_read); - if (read_res > 0) { - _input.commit(read_res); - } else { - return false; - } - } - return true; -} - -bool -Connection::read_line(string &dst) -{ - dst.clear(); - for (int c = read_byte(); c >= 0; c = read_byte()) { - if (c != '\n') { - dst.push_back(c); - } else { - stripCR(dst); - return true; - } - } - return !dst.empty(); -} - -bool -Connection::read_frame(Frame &frame) -{ - if (!fill_input(2)) { - return false; - } - char h1 = _input.next(); - char h2 = _input.next(); - frame.type = type_from_opcode(h1 & 0x0f); - frame.last = ((h1 & 0x80) != 0); - frame.payload.clear(); - size_t len = (h2 & 0x7f); - if (len > 125) { - size_t len_bytes = (len == 127) ? 8 : 2; - if (!fill_input(len_bytes)) { - return false; - } - len = 0; - for (size_t i = 0; i < len_bytes; ++i) { - len = (len << 8) + (_input.next() & 0xff); - } - } - char mask[4]; - bool use_mask = ((h2 & 0x80) != 0); - if (use_mask) { - if (!fill_input(4)) { - return false; - } - for (size_t i = 0; i < 4; ++i) { - mask[i] = _input.next(); - } - } - if (!fill_input(len)) { - return false; - } - const char *src = _input.obtain(); - char *dst = frame.payload.reserve(len); - if (use_mask) { - for (size_t i = 0; i < len; ++i) { - dst[i] = (src[i] ^ mask[i & 0x3]); - } - } else { - memcpy(dst, src, len); - } - frame.payload.commit(len); - _input.evict(len); - return true; -} - -void -Connection::write_frame(const Frame &frame) -{ - size_t len = frame.payload.used(); - bool large_len = (len > 125); - bool huge_len = (len > 0xffFF); - char h1 = opcode_from_type(frame.type); - if (frame.last) { - h1 |= 0x80; - } - char h2 = (large_len) ? 126 : len; - if (huge_len) { - ++h2; - } - _output.push(h1); - _output.push(h2); - if (large_len) { - size_t len_bytes = (huge_len) ? 8 : 2; - size_t len_src = len; - char *len_dst = _output.reserve(len_bytes); - for (size_t i = len_bytes; i > 0; --i) { - len_dst[i - 1] = (len_src & 0xff); - len_src >>= 8; - } - _output.commit(len_bytes); - } - char *dst = _output.reserve(len); - memcpy(dst, frame.payload.obtain(), len); - _output.commit(len); -} - -void -Connection::printf(const char *fmt, ...) -{ - char *dst = _output.reserve(256); - int space = _output.free(); - int size; - va_list ap; - va_start(ap, fmt); - size = vsnprintf(dst, space, fmt, ap); - va_end(ap); - assert(size >= 0); - if (size >= space) { - space = size + 1; - dst = _output.reserve(space); - va_start(ap, fmt); - size = vsnprintf(dst, space, fmt, ap); - va_end(ap); - assert((size + 1) == space); - } - _output.commit(size); -} - -void -Connection::write(const char *data, size_t len) -{ - char *dst = _output.reserve(len); - memcpy(dst, data, len); - _output.commit(len); -} - -bool -Connection::flush() -{ - while (_output.used() > 0) { - ssize_t write_res = _socket->write(_output.obtain(), _output.used()); - if (write_res > 0) { - _output.evict(write_res); - } else { - return false; - } - } - return true; -} - -} // namespace vespalib::ws diff --git a/vespalib/src/vespa/vespalib/websocket/connection.h b/vespalib/src/vespa/vespalib/websocket/connection.h deleted file mode 100644 index fdfcd227f98..00000000000 --- a/vespalib/src/vespa/vespalib/websocket/connection.h +++ /dev/null @@ -1,54 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - - -#pragma once - -#include <vespa/vespalib/net/socket.h> -#include "buffer.h" -#include "frame.h" - -namespace vespalib { -namespace ws { - -class Connection -{ -private: - Socket::UP _socket; - Buffer _input; - Buffer _output; - - bool fill_input(size_t min_bytes); - -public: - typedef std::unique_ptr<Connection> UP; - explicit Connection(Socket::UP socket); - - int read_byte() { - if (!_input.has_next()) { - if (!fill_input(1)) { - return -1; - } - } - return (_input.next() & 0xff); - } - - bool read_line(vespalib::string &dst); - - bool read_frame(Frame &frame); - - void write_frame(const Frame &frame); - - void printf(const char *fmt, ...) -#ifdef __GNUC__ - // Add printf format checks with gcc - __attribute__ ((format (printf,2,3))) -#endif - ; - - void write(const char *data, size_t len); - - bool flush(); -}; - -} // namespace vespalib::ws -} // namespace vespalib diff --git a/vespalib/src/vespa/vespalib/websocket/frame.cpp b/vespalib/src/vespa/vespalib/websocket/frame.cpp deleted file mode 100644 index d742d95698b..00000000000 --- a/vespalib/src/vespa/vespalib/websocket/frame.cpp +++ /dev/null @@ -1,7 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#include "frame.h" - -namespace vespalib::ws { - -} // namespace vespalib::ws diff --git a/vespalib/src/vespa/vespalib/websocket/frame.h b/vespalib/src/vespa/vespalib/websocket/frame.h deleted file mode 100644 index 17a9d2008e8..00000000000 --- a/vespalib/src/vespa/vespalib/websocket/frame.h +++ /dev/null @@ -1,22 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - - -#pragma once - -#include <memory> -#include <vespa/vespalib/stllike/string.h> -#include "buffer.h" - -namespace vespalib { -namespace ws { - -struct Frame { - enum class Type { NONE, TEXT, DATA, PING, PONG, CLOSE, INVALID }; - Type type; - bool last; - Buffer payload; - Frame() : type(Type::INVALID), last(true), payload() {} -}; - -} // namespace vespalib::ws -} // namespace vespalib diff --git a/vespalib/src/vespa/vespalib/websocket/handler.cpp b/vespalib/src/vespa/vespalib/websocket/handler.cpp deleted file mode 100644 index bbff65447d8..00000000000 --- a/vespalib/src/vespa/vespalib/websocket/handler.cpp +++ /dev/null @@ -1,15 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#include "handler.h" - -namespace vespalib::ws { - -namespace { - -struct DummyItem {}; - -} // namespace vespalib::ws::<unnamed> - -template struct Handler<DummyItem>; - -} // namespace vespalib::ws diff --git a/vespalib/src/vespa/vespalib/websocket/handler.h b/vespalib/src/vespa/vespalib/websocket/handler.h deleted file mode 100644 index 8919f4bfdcc..00000000000 --- a/vespalib/src/vespa/vespalib/websocket/handler.h +++ /dev/null @@ -1,22 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - - -#pragma once - -#include <memory> - -namespace vespalib { -namespace ws { - -/** - * A Handler is a component to whom you can pass an object. - **/ -template <typename T> -struct Handler -{ - virtual void handle(std::unique_ptr<T> obj) = 0; - virtual ~Handler() {} -}; - -} // namespace vespalib::ws -} // namespace vespalib diff --git a/vespalib/src/vespa/vespalib/websocket/key.cpp b/vespalib/src/vespa/vespalib/websocket/key.cpp deleted file mode 100644 index 0760c10fe31..00000000000 --- a/vespalib/src/vespa/vespalib/websocket/key.cpp +++ /dev/null @@ -1,48 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#include "key.h" -#include <vespa/vespalib/util/sha1.h> - -namespace vespalib::ws { - -namespace { - -const char *base64_chars = "ABCDEFGHIJKLMNOPQRSTUVWXYZ" - "abcdefghijklmnopqrstuvwxyz" - "0123456789+/"; - -char id(int value) { return base64_chars[value & 0x3f]; } -vespalib::string encode64(const char *data, size_t len) { - uint8_t tmp[3]; - vespalib::string result; - for (size_t i = 0; i < len; i += 3) { - tmp[0] = data[i]; - tmp[1] = (i + 1 < len) ? data[i + 1] : 0; - tmp[2] = (i + 2 < len) ? data[i + 2] : 0; - result.append(id(tmp[0] >> 2)); - result.append(id((tmp[0] << 4) | (tmp[1] >> 4))); - result.append((i + 1 < len) ? id((tmp[1] << 2) | (tmp[2] >> 6)) : '='); - result.append((i + 2 < len) ? id(tmp[2]) : '='); - } - return result; -} - -} // namespace vespalib::ws::<unnamed> - -vespalib::string -Key::create() -{ - return "dGhlIHNhbXBsZSBub25jZQ=="; -} - -vespalib::string -Key::accept(const vespalib::string &key) -{ - char hash[20]; - vespalib::string hash_input(key); - hash_input.append("258EAFA5-E914-47DA-95CA-C5AB0DC85B11"); - Sha1::hash(hash_input.data(), hash_input.size(), hash, 20); - return encode64(hash, 20); -} - -} // namespace vespalib::ws diff --git a/vespalib/src/vespa/vespalib/websocket/key.h b/vespalib/src/vespa/vespalib/websocket/key.h deleted file mode 100644 index 76062594f38..00000000000 --- a/vespalib/src/vespa/vespalib/websocket/key.h +++ /dev/null @@ -1,27 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - - -#pragma once - -#include <vespa/vespalib/stllike/string.h> - -namespace vespalib { -namespace ws { - -struct Key -{ - /** - * Create a new random key that can be used by a client to request - * a version 13 websocket connection upgrade. - **/ - static vespalib::string create(); - - /** - * Generate a version 13 websocket handshake accept token for a - * client key. - **/ - static vespalib::string accept(const vespalib::string &key); -}; - -} // namespace vespalib::ws -} // namespace vespalib diff --git a/vespalib/src/vespa/vespalib/websocket/request.cpp b/vespalib/src/vespa/vespalib/websocket/request.cpp deleted file mode 100644 index 44b05d9947e..00000000000 --- a/vespalib/src/vespa/vespalib/websocket/request.cpp +++ /dev/null @@ -1,142 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#include "request.h" -#include <cassert> -#include <algorithm> - -namespace vespalib::ws { - -namespace { - -void split(vespalib::stringref str, vespalib::stringref sep, - std::vector<vespalib::string> &dst) -{ - dst.clear(); - vespalib::string tmp; - for (size_t i = 0; i < str.size(); ++i) { - if (sep.find(str[i]) != vespalib::string::npos) { - if (!tmp.empty()) { - dst.push_back(tmp); - tmp.clear(); - } - } else { - tmp.push_back(str[i]); - } - } - if (!tmp.empty()) { - dst.push_back(tmp); - } -} - -} // namespace vespalib::ws::<unnamed> - -Request::Request() { } -Request::~Request() { } - -bool -Request::handle_header(vespalib::string &header_name, - const vespalib::string &header_line) -{ - assert(!header_line.empty()); - size_t pos = 0; - size_t end = header_line.size(); - bool continuation = (header_line[0] == ' ') || (header_line[0] == '\t'); - if (!continuation) { - pos = header_line.find(":"); - if (pos == vespalib::string::npos) { - return false; - } else { - header_name.assign(header_line, 0, pos++); - std::transform(header_name.begin(), header_name.end(), - header_name.begin(), ::tolower); - } - } - while ((pos < end) && (isspace(header_line[pos]))) { - ++pos; // strip leading whitespace - } - while ((pos < end) && (isspace(header_line[end - 1]))) { - --end; // strip trailing whitespace - } - if (header_name.empty()) { - return false; - } - auto header_insert_result = _headers.insert(std::make_pair(header_name, vespalib::string())); - bool header_found = !header_insert_result.second; - vespalib::string &header_value = header_insert_result.first->second; - if (!header_found) { - header_value.assign(header_line, pos, end - pos); - } else { - if (continuation) { - header_value.push_back(' '); - } else { // duplicate header - header_value.push_back(','); - } - header_value.append(header_line.data() + pos, end - pos); - } - return true; -} - -bool -Request::read_header(Connection &conn) -{ - vespalib::string line; - vespalib::string header_name_space; - std::vector<vespalib::string> parts; - if (!conn.read_line(line)) { - return false; - } - split(line, "\t ", parts); - if (parts.size() != 3) { - return false; - } - _method = parts[0]; - _uri = parts[1]; - _version = parts[2]; - while (conn.read_line(line)) { - if (line.empty()) { - fprintf(stderr, "request: %s %s %s\n", - _method.c_str(), _uri.c_str(), _version.c_str()); - for (const auto &h: _headers) { - fprintf(stderr, "request: '%s' -> '%s'\n", - h.first.c_str(), h.second.c_str()); - } - return true; // done - } - if (!handle_header(header_name_space, line)) { - return false; // malformed header - } - } - return false; // incomplete headers -} - -const vespalib::string & -Request::get_header(const vespalib::string &name) const -{ - auto pos = _headers.find(name); - if (pos == _headers.end()) { - return _empty; - } - return pos->second; -} - -bool -Request::has_connection_token(const vespalib::string &token) const -{ - std::vector<vespalib::string> tokens; - split(get_header("connection"), ",\t ", tokens); - for (const auto &t: tokens) { - if (strcasecmp(t.c_str(), token.c_str()) == 0) { - return true; - } - } - return false; -} - -bool -Request::is_ws_upgrade() const -{ - return ((strcasecmp(get_header("upgrade").c_str(), "websocket") == 0) && - has_connection_token("upgrade")); -} - -} diff --git a/vespalib/src/vespa/vespalib/websocket/request.h b/vespalib/src/vespa/vespalib/websocket/request.h deleted file mode 100644 index fb4dc25b0ab..00000000000 --- a/vespalib/src/vespa/vespalib/websocket/request.h +++ /dev/null @@ -1,38 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - - -#pragma once - -#include "connection.h" -#include <vespa/vespalib/stllike/string.h> -#include <map> -#include <vector> - -namespace vespalib { -namespace ws { - -class Request -{ -private: - vespalib::string _method; - vespalib::string _uri; - vespalib::string _version; - std::map<vespalib::string, vespalib::string> _headers; - vespalib::string _empty; - - bool handle_header(vespalib::string &header_name, - const vespalib::string &header_line); - -public: - Request(); - ~Request(); - bool read_header(Connection &conn); - bool is_get() const { return _method == "GET"; } - const vespalib::string &get_header(const vespalib::string &name) const; - bool has_connection_token(const vespalib::string &token) const; - bool is_ws_upgrade() const; - const vespalib::string &uri() const { return _uri; } -}; - -} // namespace vespalib::ws -} // namespace vespalib diff --git a/vespalib/src/vespa/vespalib/websocket/websocket_server.cpp b/vespalib/src/vespa/vespalib/websocket/websocket_server.cpp deleted file mode 100644 index dc9b38cd87c..00000000000 --- a/vespalib/src/vespa/vespalib/websocket/websocket_server.cpp +++ /dev/null @@ -1,164 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#include "websocket_server.h" -#include "connection.h" -#include "request.h" -#include "key.h" -#include <vespa/vespalib/util/stringfmt.h> -#include <vespa/vespalib/util/host_name.h> - -namespace vespalib::ws { - -namespace { - -vespalib::string magic = "[SELF]"; - -void respond_static(Connection &conn, const WebsocketServer::StaticPage &page, const vespalib::string &self) { - conn.printf("HTTP/1.1 200 OK\r\n"); - conn.printf("Connection: close\r\n"); - conn.printf("Content-Type: %s\r\n", page.content_type.c_str()); - conn.printf("\r\n"); - size_t pos = 0; - while (pos < page.content.size()) { - size_t next = page.content.find(magic, pos); - if (next == vespalib::string::npos) { - conn.write(page.content.data() + pos, page.content.size() - pos); - pos = page.content.size(); - } else { - conn.write(page.content.data() + pos, next - pos); - conn.write(self.data(), self.size()); - pos = next + magic.size(); - } - } - conn.flush(); -} - -void respond_error(Connection &conn, int code, const vespalib::string &message) { - conn.printf("HTTP/1.1 %d %s\r\n", code, message.c_str()); - conn.printf("Connection: close\r\n"); - conn.printf("Content-Type: text/html\r\n"); - conn.printf("\r\n"); - conn.printf("<html><body><h2>%d %s</h2></body></html>", - code, message.c_str()); - conn.flush(); -} - -void respond_upgrade(Connection &conn, const vespalib::string &accept_token) { - conn.printf("HTTP/1.1 101 Switching Protocols\r\n"); - conn.printf("Upgrade: websocket\r\n"); - conn.printf("Connection: Upgrade\r\n"); - conn.printf("Sec-WebSocket-Accept: %s\r\n", accept_token.c_str()); - conn.printf("\r\n"); - conn.flush(); -} - -void respond_upgrade_failed(Connection &conn) { - conn.printf("HTTP/1.1 400 Upgrade Failed\r\n"); - conn.printf("Connection: close\r\n"); - conn.printf("Sec-WebSocket-Version: 13\r\n"); - conn.printf("\r\n"); - conn.flush(); -} - -const char *name_from_type(Frame::Type type) { - switch (type) { - case Frame::Type::NONE: return "NONE"; - case Frame::Type::TEXT: return "TEXT"; - case Frame::Type::DATA: return "DATA"; - case Frame::Type::PING: return "PING"; - case Frame::Type::PONG: return "PONG"; - case Frame::Type::CLOSE: return "CLOSE"; - default: return "INVALID"; - } -} - -void handle_echo(Connection &conn) { - fprintf(stderr, "server: got ws connection\n"); - Frame frame; - bool done = false; - while (!done && conn.read_frame(frame)) { - fprintf(stderr, "server: got frame of type %s with size %zu\n", - name_from_type(frame.type), frame.payload.used()); - if (frame.type == Frame::Type::TEXT) { - fprintf(stderr, "server: got text: %s\n", - vespalib::string(frame.payload.obtain(), - frame.payload.used()).c_str()); - } - if (frame.type == Frame::Type::INVALID) { - break; - } - if (frame.type == Frame::Type::PONG) { - continue; - } - if (frame.type == Frame::Type::PING) { - frame.type = Frame::Type::PONG; - } - if (frame.type == Frame::Type::CLOSE) { - done = true; - } - conn.write_frame(frame); - conn.flush(); - } - fprintf(stderr, "server: closing ws connection\n"); -} - -void handle_upgrade(Connection &conn, Request &req) { - const vespalib::string version = req.get_header("sec-websocket-version"); - vespalib::string accept_token = Key::accept(req.get_header("sec-websocket-key")); - if (version == "13") { - respond_upgrade(conn, accept_token); - handle_echo(conn); - } else { - respond_upgrade_failed(conn); - } -} - -} // namespace vespalib::ws::<unnamed> - -WebsocketServer::StaticPage::StaticPage(const vespalib::string & type, const vespalib::string & content_in) - : content_type(type), - content(content_in) -{} -WebsocketServer::StaticPage::StaticPage(const StaticPage &) = default; -WebsocketServer::StaticPage & WebsocketServer::StaticPage::operator = (const StaticPage &) = default; -WebsocketServer::StaticPage::~StaticPage() = default; - -WebsocketServer::WebsocketServer(int port_in, StaticRepo &&repo) - : _acceptor(port_in, *this), - _static_repo(std::move(repo)), - _self(make_string("%s:%d", HostName::get().c_str(), _acceptor.port())) -{ -} - -WebsocketServer::~WebsocketServer() = default; - -void -WebsocketServer::handle(std::unique_ptr<Socket> socket) -{ - Connection conn(std::move(socket)); - Request req; - if (!req.read_header(conn)) { - respond_error(conn, 400, "Bad Request"); - return; - } - if (!req.is_get()) { - respond_error(conn, 501, "Not Implemented"); - return; - } - if (req.is_ws_upgrade()) { - if (req.uri() == "/echo") { - handle_upgrade(conn, req); - } else { - respond_error(conn, 404, "Not Found"); - } - } else { - auto page = _static_repo.find(req.uri()); - if (page != _static_repo.end()) { - respond_static(conn, page->second, _self); - } else { - respond_error(conn, 404, "Not Found"); - } - } -} - -} diff --git a/vespalib/src/vespa/vespalib/websocket/websocket_server.h b/vespalib/src/vespa/vespalib/websocket/websocket_server.h deleted file mode 100644 index f30daafb726..00000000000 --- a/vespalib/src/vespa/vespalib/websocket/websocket_server.h +++ /dev/null @@ -1,38 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#pragma once - -#include "handler.h" -#include "acceptor.h" -#include <vespa/vespalib/stllike/string.h> -#include <map> - -namespace vespalib::ws { - -class WebsocketServer : public Handler<Socket> { -public: - struct StaticPage { - StaticPage(const vespalib::string & type, const vespalib::string & content_in); - StaticPage(const StaticPage &); - StaticPage & operator = (const StaticPage &); - StaticPage(StaticPage &&) = default; - StaticPage & operator = (StaticPage &&) = default; - ~StaticPage(); - vespalib::string content_type; - vespalib::string content; - }; - typedef std::map<vespalib::string, StaticPage> StaticRepo; - -private: - Acceptor _acceptor; - StaticRepo _static_repo; - vespalib::string _self; - -public: - WebsocketServer(int port_in, StaticRepo &&repo = StaticRepo()); - ~WebsocketServer() override; - void handle(std::unique_ptr<Socket> socket) override; - int port() { return _acceptor.port(); } -}; - -} // namespace vespalib::ws diff --git a/yolean/src/test/java/com/yahoo/yolean/chain/ChainBuilderTest.java b/yolean/src/test/java/com/yahoo/yolean/chain/ChainBuilderTest.java index 349ba8998bd..1d5b6a6cac7 100644 --- a/yolean/src/test/java/com/yahoo/yolean/chain/ChainBuilderTest.java +++ b/yolean/src/test/java/com/yahoo/yolean/chain/ChainBuilderTest.java @@ -10,7 +10,6 @@ import java.util.List; import static com.yahoo.yolean.chain.Dependencies.after; import static com.yahoo.yolean.chain.Dependencies.before; import static com.yahoo.yolean.chain.Dependencies.provides; -import static org.hamcrest.MatcherAssert.assertThat; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; diff --git a/zookeeper-client-common/pom.xml b/zookeeper-client-common/pom.xml index f991fa5dea9..7e23e0cb1c8 100644 --- a/zookeeper-client-common/pom.xml +++ b/zookeeper-client-common/pom.xml @@ -25,6 +25,16 @@ <artifactId>zookeeper</artifactId> <version>${zookeeper.client.version}</version> <scope>provided</scope> + <exclusions> + <exclusion> + <groupId>log4j</groupId> + <artifactId>log4j</artifactId> + </exclusion> + <exclusion> + <groupId>org.slf4j</groupId> + <artifactId>slf4j-log4j12</artifactId> + </exclusion> + </exclusions> </dependency> <!-- compile scope --> diff --git a/zookeeper-command-line-client/pom.xml b/zookeeper-command-line-client/pom.xml index f1fc49dcb1c..a8105c78881 100644 --- a/zookeeper-command-line-client/pom.xml +++ b/zookeeper-command-line-client/pom.xml @@ -15,6 +15,17 @@ <groupId>org.apache.zookeeper</groupId> <artifactId>zookeeper</artifactId> <version>${zookeeper.client.version}</version> + <exclusions> + <exclusion> + <groupId>log4j</groupId> + <artifactId>log4j</artifactId> + </exclusion> + <exclusion> + <!-- This is the log4j 1.2 binding for slf4j. Must not be used with log4j-over-slf4j (see below) --> + <groupId>org.slf4j</groupId> + <artifactId>slf4j-log4j12</artifactId> + </exclusion> + </exclusions> </dependency> <dependency> <groupId>com.yahoo.vespa</groupId> @@ -38,6 +49,19 @@ <artifactId>slf4j-api</artifactId> <scope>compile</scope> </dependency> + <dependency> + <!-- Bind to slf4j's SimpleLogger --> + <groupId>org.slf4j</groupId> + <artifactId>slf4j-simple</artifactId> + <version>${slf4j.version}</version> + </dependency> + <dependency> + <!-- slf4j's replacement for log4j. See http://www.slf4j.org/legacy.html#log4j-over-slf4j --> + <groupId>org.slf4j</groupId> + <artifactId>log4j-over-slf4j</artifactId> + <scope>compile</scope> + </dependency> + </dependencies> <build> <plugins> diff --git a/zookeeper-command-line-client/src/main/java/com/yahoo/vespa/zookeeper/cli/FourLetterWordMain.java b/zookeeper-command-line-client/src/main/java/com/yahoo/vespa/zookeeper/cli/FourLetterWordMain.java index 9d2cdc1c4b8..ed79fed908c 100644 --- a/zookeeper-command-line-client/src/main/java/com/yahoo/vespa/zookeeper/cli/FourLetterWordMain.java +++ b/zookeeper-command-line-client/src/main/java/com/yahoo/vespa/zookeeper/cli/FourLetterWordMain.java @@ -3,6 +3,7 @@ package com.yahoo.vespa.zookeeper.cli; import com.yahoo.vespa.zookeeper.client.ZkClientConfigBuilder; import org.apache.zookeeper.common.X509Exception; +import org.slf4j.impl.SimpleLogger; import java.io.IOException; import java.util.Map; @@ -14,6 +15,10 @@ import java.util.Map; */ public class FourLetterWordMain { + static { + System.setProperty(SimpleLogger.DEFAULT_LOG_LEVEL_KEY, "WARN"); + } + public static void main(String[] args) throws X509Exception.SSLContextException, IOException { Map<String, String> zkClientConfig = new ZkClientConfigBuilder().toConfigProperties(); zkClientConfig.forEach(System::setProperty); diff --git a/zookeeper-command-line-client/src/main/java/com/yahoo/vespa/zookeeper/cli/Main.java b/zookeeper-command-line-client/src/main/java/com/yahoo/vespa/zookeeper/cli/Main.java index 3c89e5105ca..7f7965f82eb 100644 --- a/zookeeper-command-line-client/src/main/java/com/yahoo/vespa/zookeeper/cli/Main.java +++ b/zookeeper-command-line-client/src/main/java/com/yahoo/vespa/zookeeper/cli/Main.java @@ -3,6 +3,7 @@ package com.yahoo.vespa.zookeeper.cli; import com.yahoo.vespa.zookeeper.client.ZkClientConfigBuilder; import org.apache.zookeeper.ZooKeeperMain; +import org.slf4j.impl.SimpleLogger; import java.io.IOException; @@ -10,6 +11,11 @@ import java.io.IOException; * @author bjorncs */ public class Main { + + static { + System.setProperty(SimpleLogger.DEFAULT_LOG_LEVEL_KEY, "WARN"); + } + public static void main(String[] args) throws IOException, InterruptedException { new ZkClientConfigBuilder() .toConfigProperties() |