diff options
86 files changed, 1329 insertions, 1080 deletions
diff --git a/application/pom.xml b/application/pom.xml index 2f0586145b2..8f9dc7999a0 100644 --- a/application/pom.xml +++ b/application/pom.xml @@ -139,8 +139,8 @@ <!-- START JETTY embedded jars --> <dependency> - <groupId>org.eclipse.jetty.http2</groupId> - <artifactId>http2-common</artifactId> + <groupId>org.eclipse.jetty.alpn</groupId> + <artifactId>alpn-api</artifactId> </dependency> <dependency> <groupId>org.eclipse.jetty.http2</groupId> @@ -152,43 +152,35 @@ </dependency> <dependency> <groupId>org.eclipse.jetty</groupId> - <artifactId>jetty-alpn-server</artifactId> - </dependency> - <dependency> - <groupId>org.eclipse.jetty</groupId> <artifactId>jetty-client</artifactId> + <scope>test</scope> </dependency> <dependency> <groupId>org.eclipse.jetty</groupId> - <artifactId>jetty-http</artifactId> - </dependency> - <dependency> - <groupId>org.eclipse.jetty</groupId> - <artifactId>jetty-io</artifactId> + <artifactId>jetty-continuation</artifactId> + <scope>test</scope> </dependency> <dependency> <groupId>org.eclipse.jetty</groupId> <artifactId>jetty-jmx</artifactId> + <scope>test</scope> </dependency> <dependency> <groupId>org.eclipse.jetty</groupId> <artifactId>jetty-server</artifactId> + <scope>test</scope> </dependency> <dependency> <groupId>org.eclipse.jetty</groupId> <artifactId>jetty-servlet</artifactId> + <scope>test</scope> </dependency> <dependency> <groupId>org.eclipse.jetty</groupId> - <artifactId>jetty-util</artifactId> + <artifactId>jetty-servlets</artifactId> + <scope>test</scope> </dependency> <dependency> - <groupId>org.eclipse.jetty.toolchain</groupId> - <artifactId>jetty-jakarta-servlet-api</artifactId> - </dependency> - <!-- END JETTY embedded jars --> - - <dependency> <groupId>org.junit.jupiter</groupId> <artifactId>junit-jupiter-api</artifactId> <scope>test</scope> @@ -198,6 +190,8 @@ <artifactId>junit-jupiter-engine</artifactId> <scope>test</scope> </dependency> + <!-- END JETTY embedded jars --> + </dependencies> <build> diff --git a/client/go/vespa/xml/config.go b/client/go/vespa/xml/config.go index a167b67de8b..c2bd88307e5 100644 --- a/client/go/vespa/xml/config.go +++ b/client/go/vespa/xml/config.go @@ -228,7 +228,8 @@ func IsProdRegion(s string, system vespa.System) bool { } switch s { case "aws-us-east-1c", "aws-us-west-2a", - "aws-eu-west-1a", "aws-ap-northeast-1a": + "aws-eu-west-1a", "aws-ap-northeast-1a", + "gcp-us-central1-f": return true } return false diff --git a/client/src/main/java/ai/vespa/client/dsl/EndQuery.java b/client/src/main/java/ai/vespa/client/dsl/EndQuery.java index 61ffe84e5a9..d1d4b6eb883 100644 --- a/client/src/main/java/ai/vespa/client/dsl/EndQuery.java +++ b/client/src/main/java/ai/vespa/client/dsl/EndQuery.java @@ -76,15 +76,21 @@ public class EndQuery { } /** - * Semicolon. - * turn a query into fixed query. + * Calls fix() * - * @return the fixed query + * @deprecated use {link #fix} */ - public FixedQuery semicolon() { + @Deprecated // TODO: Remove on Vespa 9 + public FixedQuery semicolon() { return fix(); } + + /** Returns a fixed query containing this. */ + public FixedQuery fix() { return new FixedQuery(this); } + /** Calls fix().build() */ + public String build() { return fix().build(); } + /** * Group. * https://docs.vespa.ai/en/reference/query-language-reference.html#grouping @@ -163,7 +169,7 @@ public class EndQuery { StringBuilder sb = new StringBuilder(); String orderStr = order.stream().map(array -> A.empty().equals(array[0]) ? Text.format("%s %s", array[1], array[2]) - : Text.format("[%s]%s %s", array[0], array[1], array[2])) + : Text.format("%s%s %s", array[0], array[1], array[2])) .collect(Collectors.joining(", ")); String others = map.entrySet().stream() diff --git a/client/src/main/java/ai/vespa/client/dsl/FixedQuery.java b/client/src/main/java/ai/vespa/client/dsl/FixedQuery.java index d957217a9c7..b8fc3094937 100644 --- a/client/src/main/java/ai/vespa/client/dsl/FixedQuery.java +++ b/client/src/main/java/ai/vespa/client/dsl/FixedQuery.java @@ -332,7 +332,6 @@ public class FixedQuery { return this; } - /** * build the query map from the query * @@ -342,7 +341,6 @@ public class FixedQuery { if (queryMap != null) { return queryMap; } - assignIndex(); StringBuilder sb = new StringBuilder(); sb.append("select ") @@ -355,7 +353,6 @@ public class FixedQuery { if (!"".equals(endQuery.toString())) { sb.append(' ').append(endQuery); } - sb.append(";"); queryMap = new LinkedHashMap<>(); // for the order queryMap.put("yql", sb.toString()); @@ -374,23 +371,6 @@ public class FixedQuery { .collect(Collectors.joining("&")); } - private void assignIndex() { - assignIndex(endQuery.queryChain.getQuery(), new AtomicInteger()); - } - - private void assignIndex(QueryChain q, AtomicInteger ai) { - q.setIndex(ai.incrementAndGet()); - if (q instanceof Query) { - assignIndex((Query) q, ai); - } - } - - private void assignIndex(Query q, AtomicInteger ai) { - q.queries.stream() - .filter(QueryChain::nonEmpty) - .forEach(qu -> assignIndex(qu, ai)); - } - private Map<String, String> getUserInputs() { return getUserInputs(endQuery.queryChain.getQuery()); } diff --git a/client/src/main/java/ai/vespa/client/dsl/Query.java b/client/src/main/java/ai/vespa/client/dsl/Query.java index bc5be2280c4..36718ced814 100644 --- a/client/src/main/java/ai/vespa/client/dsl/Query.java +++ b/client/src/main/java/ai/vespa/client/dsl/Query.java @@ -290,13 +290,27 @@ public class Query extends QueryChain { } /** - * Semicolon. - * turn a query into fixed query. + * Calls fix() * * @return the fixed query + * @deprecated use {@link #fix()}, {@link #end()} or {@link #build} instead */ - public FixedQuery semicolon() { - return new FixedQuery(new EndQuery(this)); + @Deprecated // TODO: Remove on Vespa 9 + public FixedQuery semicolon() { return fix(); } + + /** Returns this as an ended query. */ + public EndQuery end() { + return new EndQuery(this); + } + + /** Calls end().fix(). */ + public FixedQuery fix() { + return end().fix(); + } + + /** Calls fix().build(). */ + public String build() { + return fix().build(); } @Override diff --git a/client/src/main/java/ai/vespa/client/dsl/QueryChain.java b/client/src/main/java/ai/vespa/client/dsl/QueryChain.java index 31b5220e871..58b1563a222 100644 --- a/client/src/main/java/ai/vespa/client/dsl/QueryChain.java +++ b/client/src/main/java/ai/vespa/client/dsl/QueryChain.java @@ -4,7 +4,6 @@ package ai.vespa.client.dsl; public abstract class QueryChain { String op; - int index; // for distinct each query chain Sources sources; Select select; Query query; @@ -18,10 +17,6 @@ public abstract class QueryChain { return op; } - void setIndex(int index) { - this.index = index; - } - Sources getSources() { return sources; } diff --git a/client/src/main/java/ai/vespa/client/dsl/Wand.java b/client/src/main/java/ai/vespa/client/dsl/Wand.java index 56bf3e3cf1d..df89d235139 100644 --- a/client/src/main/java/ai/vespa/client/dsl/Wand.java +++ b/client/src/main/java/ai/vespa/client/dsl/Wand.java @@ -66,7 +66,7 @@ public class Wand extends QueryChain { public String toString() { boolean hasAnnotation = A.hasAnnotation(annotation); String s = Text.format("wand(%s, %s)", fieldName, Q.toJson(value)); - return hasAnnotation ? Text.format("([%s]%s)", annotation, s) : s; + return hasAnnotation ? Text.format("(%s%s)", annotation, s) : s; } } diff --git a/client/src/main/java/ai/vespa/client/dsl/WeakAnd.java b/client/src/main/java/ai/vespa/client/dsl/WeakAnd.java index 45a616b5a8a..1e0ea8ad700 100644 --- a/client/src/main/java/ai/vespa/client/dsl/WeakAnd.java +++ b/client/src/main/java/ai/vespa/client/dsl/WeakAnd.java @@ -54,7 +54,7 @@ public class WeakAnd extends QueryChain { s = Text.format("weakAnd(%s)", value.queries.stream().map(Object::toString).collect(Collectors.joining(", "))); - return hasAnnotation ? Text.format("([%s]%s)", annotation, s) : s; + return hasAnnotation ? Text.format("(%s%s)", annotation, s) : s; } }
\ No newline at end of file diff --git a/client/src/test/java/ai/vespa/client/dsl/QTest.java b/client/src/test/java/ai/vespa/client/dsl/QTest.java index 114cbee0177..b3c26e3db6b 100644 --- a/client/src/test/java/ai/vespa/client/dsl/QTest.java +++ b/client/src/test/java/ai/vespa/client/dsl/QTest.java @@ -25,10 +25,9 @@ class QTest { String q = Q.select("f1", "f2") .from("sd1") .where("f1").contains("v1") - .semicolon() .build(); - assertEquals(q, "yql=select f1, f2 from sd1 where f1 contains \"v1\";"); + assertEquals(q, "yql=select f1, f2 from sd1 where f1 contains \"v1\""); } @Test @@ -36,10 +35,9 @@ class QTest { String q = Q.select("*") .from("sd1") .where("f1").contains("v1") - .semicolon() .build(); - assertEquals(q, "yql=select * from sd1 where f1 contains \"v1\";"); + assertEquals(q, "yql=select * from sd1 where f1 contains \"v1\""); } @Test @@ -47,10 +45,9 @@ class QTest { String q = Q.select("*") .from("sd1", "sd2") .where("f1").contains("v1") - .semicolon() .build(); - assertEquals(q, "yql=select * from sources sd1, sd2 where f1 contains \"v1\";"); + assertEquals(q, "yql=select * from sources sd1, sd2 where f1 contains \"v1\""); } @Test @@ -66,11 +63,11 @@ class QTest { .timeout(3) .orderByDesc("f1") .orderByAsc("f2") - .semicolon() + .fix() .param("paramk1", "paramv1") .build(); - assertEquals(q, "yql=select * from sd1 where f1 contains \"v1\" and f2 contains \"v2\" or f3 contains \"v3\" and !(f4 contains \"v4\") order by f1 desc, f2 asc limit 2 offset 1 timeout 3;¶mk1=paramv1"); + assertEquals(q, "yql=select * from sd1 where f1 contains \"v1\" and f2 contains \"v2\" or f3 contains \"v3\" and !(f4 contains \"v4\") order by f1 desc, f2 asc limit 2 offset 1 timeout 3¶mk1=paramv1"); } @Test @@ -81,10 +78,9 @@ class QTest { .and("f2").matches("v2") .or("f3").matches("v3") .andnot("f4").matches("v4") - .semicolon() .build(); - assertEquals(q, "yql=select * from sd1 where f1 matches \"v1\" and f2 matches \"v2\" or f3 matches \"v3\" and !(f4 matches \"v4\");"); + assertEquals(q, "yql=select * from sd1 where f1 matches \"v1\" and f2 matches \"v2\" or f3 matches \"v3\" and !(f4 matches \"v4\")"); } @Test @@ -97,10 +93,9 @@ class QTest { .and("f4").gt(4) .and("f5").eq(5) .and("f6").inRange(6, 7) - .semicolon() .build(); - assertEquals(q, "yql=select * from sd1 where f1 <= 1 and f2 < 2 and f3 >= 3 and f4 > 4 and f5 = 5 and range(f6, 6, 7);"); + assertEquals(q, "yql=select * from sd1 where f1 <= 1 and f2 < 2 and f3 >= 3 and f4 > 4 and f5 = 5 and range(f6, 6, 7)"); } @Test @@ -113,10 +108,9 @@ class QTest { .and("f4").gt(4L) .and("f5").eq(5L) .and("f6").inRange(6L, 7L) - .semicolon() .build(); - assertEquals(q, "yql=select * from sd1 where f1 <= 1L and f2 < 2L and f3 >= 3L and f4 > 4L and f5 = 5L and range(f6, 6L, 7L);"); + assertEquals(q, "yql=select * from sd1 where f1 <= 1L and f2 < 2L and f3 >= 3L and f4 > 4L and f5 = 5L and range(f6, 6L, 7L)"); } @Test @@ -129,10 +123,9 @@ class QTest { .and("f4").gt(4.4) .and("f5").eq(5.5) .and("f6").inRange(6.6, 7.7) - .semicolon() .build(); - assertEquals(q, "yql=select * from sd1 where f1 <= 1.1 and f2 < 2.2 and f3 >= 3.3 and f4 > 4.4 and f5 = 5.5 and range(f6, 6.6, 7.7);"); + assertEquals(q, "yql=select * from sd1 where f1 <= 1.1 and f2 < 2.2 and f3 >= 3.3 and f4 > 4.4 and f5 = 5.5 and range(f6, 6.6, 7.7)"); } @Test @@ -145,10 +138,9 @@ class QTest { .and("f4").gt(4.4D) .and("f5").eq(5.5D) .and("f6").inRange(6.6D, 7.7D) - .semicolon() .build(); - assertEquals(q, "yql=select * from sd1 where f1 <= 1.1 and f2 < 2.2 and f3 >= 3.3 and f4 > 4.4 and f5 = 5.5 and range(f6, 6.6, 7.7);"); + assertEquals(q, "yql=select * from sd1 where f1 <= 1.1 and f2 < 2.2 and f3 >= 3.3 and f4 > 4.4 and f5 = 5.5 and range(f6, 6.6, 7.7)"); } @Test @@ -158,10 +150,9 @@ class QTest { .where("f1").contains("1") .andnot(Q.p(Q.p("f2").contains("2").and("f3").contains("3")) .or(Q.p("f2").contains("4").andnot("f3").contains("5"))) - .semicolon() .build(); - assertEquals(q, "yql=select * from sd1 where f1 contains \"1\" and !((f2 contains \"2\" and f3 contains \"3\") or (f2 contains \"4\" and !(f3 contains \"5\")));"); + assertEquals(q, "yql=select * from sd1 where f1 contains \"1\" and !((f2 contains \"2\" and f3 contains \"3\") or (f2 contains \"4\" and !(f3 contains \"5\")))"); } @Test @@ -170,9 +161,8 @@ class QTest { .from("sd1") .where(Q.ui("value1")) .and(Q.ui("index", "value2")) - .semicolon() .build(); - assertEquals("yql=select * from sd1 where userInput(\"value1\") and ({\"defaultIndex\":\"index\"}userInput(\"value2\"));", q); + assertEquals("yql=select * from sd1 where userInput(\"value1\") and ({\"defaultIndex\":\"index\"}userInput(\"value2\"))", q); } @Test @@ -181,8 +171,8 @@ class QTest { .from("site") .where(Q.rank(Q.p("docQ").nearestNeighbor("vectorQuery"), Q.ui("@query"))) - .semicolon().build(); - assertEquals("yql=select url from site where rank(nearestNeighbor(docQ, vectorQuery), userInput(@query));", q); + .build(); + assertEquals("yql=select url from site where rank(nearestNeighbor(docQ, vectorQuery), userInput(@query))", q); } @Test @@ -191,10 +181,9 @@ class QTest { .from("sd1") .where(Q.dotPdt("f1", stringIntMap("a", 1, "b", 2, "c", 3))) .and("f2").contains("1") - .semicolon() .build(); - assertEquals("yql=select * from sd1 where dotProduct(f1, {\"a\":1,\"b\":2,\"c\":3}) and f2 contains \"1\";", q); + assertEquals("yql=select * from sd1 where dotProduct(f1, {\"a\":1,\"b\":2,\"c\":3}) and f2 contains \"1\"", q); } @Test @@ -203,10 +192,9 @@ class QTest { .from("sd1") .where(Q.wtdSet("f1", stringIntMap("a", 1, "b", 2, "c", 3))) .and("f2").contains("1") - .semicolon() .build(); - assertEquals(q, "yql=select * from sd1 where weightedSet(f1, {\"a\":1,\"b\":2,\"c\":3}) and f2 contains \"1\";"); + assertEquals(q, "yql=select * from sd1 where weightedSet(f1, {\"a\":1,\"b\":2,\"c\":3}) and f2 contains \"1\""); } @Test @@ -215,10 +203,9 @@ class QTest { .from("sd1") .where(Q.nonEmpty(Q.p("f1").contains("v1"))) .and("f2").contains("v2") - .semicolon() .build(); - assertEquals(q, "yql=select * from sd1 where nonEmpty(f1 contains \"v1\") and f2 contains \"v2\";"); + assertEquals(q, "yql=select * from sd1 where nonEmpty(f1 contains \"v1\") and f2 contains \"v2\""); } @@ -232,10 +219,9 @@ class QTest { Q.wand("f3", Arrays.asList(Arrays.asList(1, 1), Arrays.asList(2, 2))) .annotate(A.a("scoreThreshold", 0.13)) ) - .semicolon() .build(); - assertEquals(q, "yql=select * from sd1 where wand(f1, {\"a\":1,\"b\":2,\"c\":3}) and wand(f2, [[1,1],[2,2]]) and ([{\"scoreThreshold\":0.13}]wand(f3, [[1,1],[2,2]]));"); + assertEquals(q, "yql=select * from sd1 where wand(f1, {\"a\":1,\"b\":2,\"c\":3}) and wand(f2, [[1,1],[2,2]]) and ({\"scoreThreshold\":0.13}wand(f3, [[1,1],[2,2]]))"); } @Test @@ -246,10 +232,9 @@ class QTest { .and(Q.weakand(Q.p("f1").contains("v1").and("f2").contains("v2")) .annotate(A.a("scoreThreshold", 0.13)) ) - .semicolon() .build(); - assertEquals(q, "yql=select * from sd1 where weakAnd(f1 contains \"v1\", f2 contains \"v2\") and ([{\"scoreThreshold\":0.13}]weakAnd(f1 contains \"v1\", f2 contains \"v2\"));"); + assertEquals(q, "yql=select * from sd1 where weakAnd(f1 contains \"v1\", f2 contains \"v2\") and ({\"scoreThreshold\":0.13}weakAnd(f1 contains \"v1\", f2 contains \"v2\"))"); } @Test @@ -257,10 +242,9 @@ class QTest { String q = Q.select("*") .from("sd1") .where("a").contains("b").and(Q.geoLocation("taiwan", 25.105497, 121.597366, "200km")) - .semicolon() .build(); - assertEquals(q, "yql=select * from sd1 where a contains \"b\" and geoLocation(taiwan, 25.105497, 121.597366, \"200km\");"); + assertEquals(q, "yql=select * from sd1 where a contains \"b\" and geoLocation(taiwan, 25.105497, 121.597366, \"200km\")"); } @Test @@ -271,9 +255,8 @@ class QTest { .and(Q.nearestNeighbor("vec1", "vec2") .annotate(A.a("targetHits", 10, "approximate", false)) ) - .semicolon() .build(); - assertEquals(q, "yql=select * from sd1 where a contains \"b\" and ([{\"approximate\":false,\"targetHits\":10}]nearestNeighbor(vec1, vec2));"); + assertEquals(q, "yql=select * from sd1 where a contains \"b\" and ([{\"approximate\":false,\"targetHits\":10}]nearestNeighbor(vec1, vec2))"); } @Test @@ -282,7 +265,6 @@ class QTest { () -> Q.select("*") .from("sd1") .where("a").contains("b").and(Q.nearestNeighbor("vec1", "vec2")) - .semicolon() .build()); } @@ -295,10 +277,9 @@ class QTest { Q.p("f1").contains("v1") ) ) - .semicolon() .build(); - assertEquals(q, "yql=select * from sd1 where rank(f1 contains \"v1\");"); + assertEquals(q, "yql=select * from sd1 where rank(f1 contains \"v1\")"); } @Test @@ -310,10 +291,9 @@ class QTest { Q.p("f2").contains("v2"), Q.p("f3").eq(3)) ) - .semicolon() .build(); - assertEquals(q, "yql=select * from sd1 where rank(f1 contains \"v1\", f2 contains \"v2\", f3 = 3);"); + assertEquals(q, "yql=select * from sd1 where rank(f1 contains \"v1\", f2 contains \"v2\", f3 = 3)"); } @Test @@ -325,25 +305,22 @@ class QTest { Q.p("f1").contains("v1"), ranks) ) - .semicolon() .build(); - assertEquals(q, "yql=select * from sd1 where rank(f1 contains \"v1\", f2 contains \"v2\", f3 = 3);"); + assertEquals(q, "yql=select * from sd1 where rank(f1 contains \"v1\", f2 contains \"v2\", f3 = 3)"); } @Test void stringfunction_annotations() { - { Annotation annotation = A.filter(); String expected = "[{\"filter\":true}]"; String q = Q.select("*") .from("sd1") .where("f1").contains(annotation, "v1") - .semicolon() .build(); - assertEquals(q, "yql=select * from sd1 where f1 contains (" + expected + "\"v1\");"); + assertEquals(q, "yql=select * from sd1 where f1 contains (" + expected + "\"v1\")"); } { Annotation annotation = A.defaultIndex("idx"); @@ -351,10 +328,9 @@ class QTest { String q = Q.select("*") .from("sd1") .where("f1").contains(annotation, "v1") - .semicolon() .build(); - assertEquals(q, "yql=select * from sd1 where f1 contains (" + expected + "\"v1\");"); + assertEquals(q, "yql=select * from sd1 where f1 contains (" + expected + "\"v1\")"); } { Annotation annotation = A.a(stringObjMap("a1", stringObjMap("k1", "v1", "k2", 2))); @@ -362,10 +338,9 @@ class QTest { String q = Q.select("*") .from("sd1") .where("f1").contains(annotation, "v1") - .semicolon() .build(); - assertEquals(q, "yql=select * from sd1 where f1 contains (" + expected + "\"v1\");"); + assertEquals(q, "yql=select * from sd1 where f1 contains (" + expected + "\"v1\")"); } } @@ -375,10 +350,9 @@ class QTest { String q = Q.select("*") .from("sd1") .where("f1").contains("v1").annotate(A.a("ak1", "av1")) - .semicolon() .build(); - assertEquals(q, "yql=select * from sd1 where ([{\"ak1\":\"av1\"}](f1 contains \"v1\"));"); + assertEquals(q, "yql=select * from sd1 where ([{\"ak1\":\"av1\"}](f1 contains \"v1\"))"); } @Test @@ -386,10 +360,9 @@ class QTest { String q = Q.select("*") .from("sd1") .where(Q.p("f1").contains("v1").annotate(A.a("ak1", "av1")).and("f2").contains("v2")) - .semicolon() .build(); - assertEquals(q, "yql=select * from sd1 where ([{\"ak1\":\"av1\"}](f1 contains \"v1\" and f2 contains \"v2\"));"); + assertEquals(q, "yql=select * from sd1 where ([{\"ak1\":\"av1\"}](f1 contains \"v1\" and f2 contains \"v2\"))"); } @Test @@ -400,19 +373,17 @@ class QTest { Q.p("f1").contains("v1").annotate(A.a("ak1", "av1"))) .and("f2").contains("v2") ) - .semicolon() .build(); - assertEquals(q, "yql=select * from sd1 where (([{\"ak1\":\"av1\"}](f1 contains \"v1\")) and f2 contains \"v2\");"); + assertEquals(q, "yql=select * from sd1 where (([{\"ak1\":\"av1\"}](f1 contains \"v1\")) and f2 contains \"v2\")"); } @Test void build_query_which_created_from_Q_b_without_select_and_sources() { String q = Q.p("f1").contains("v1") - .semicolon() .build(); - assertEquals(q, "yql=select * from sources * where f1 contains \"v1\";"); + assertEquals(q, "yql=select * from sources * where f1 contains \"v1\""); } @Test @@ -422,126 +393,107 @@ class QTest { .orderByAsc(A.a(stringObjMap("function", "uca", "locale", "en_US", "strength", "IDENTICAL")), "f3") .orderByDesc("f4") .orderByDesc(A.a(stringObjMap("function", "lowercase")), "f5") - .semicolon() .build(); - assertEquals(q, "yql=select * from sources * where f1 contains \"v1\" order by f2 asc, [{\"function\":\"uca\",\"locale\":\"en_US\",\"strength\":\"IDENTICAL\"}]f3 asc, f4 desc, [{\"function\":\"lowercase\"}]f5 desc;"); + assertEquals(q, "yql=select * from sources * where f1 contains \"v1\" order by f2 asc, {\"function\":\"uca\",\"locale\":\"en_US\",\"strength\":\"IDENTICAL\"}f3 asc, f4 desc, {\"function\":\"lowercase\"}f5 desc"); } @Test void contains_sameElement() { String q = Q.p("f1").containsSameElement(Q.p("stime").le(1).and("etime").gt(2)) - .semicolon() .build(); - assertEquals(q, "yql=select * from sources * where f1 contains sameElement(stime <= 1, etime > 2);"); + assertEquals(q, "yql=select * from sources * where f1 contains sameElement(stime <= 1, etime > 2)"); } @Test void contains_phrase_near_onear_equiv() { { String q1 = Q.p("f1").containsPhrase("p1", "p2", "p3") - .semicolon() .build(); String q2 = Q.p("f1").containsPhrase(Arrays.asList("p1", "p2", "p3")) - .semicolon() .build(); - assertEquals(q1, "yql=select * from sources * where f1 contains phrase(\"p1\", \"p2\", \"p3\");"); - assertEquals(q2, "yql=select * from sources * where f1 contains phrase(\"p1\", \"p2\", \"p3\");"); + assertEquals(q1, "yql=select * from sources * where f1 contains phrase(\"p1\", \"p2\", \"p3\")"); + assertEquals(q2, "yql=select * from sources * where f1 contains phrase(\"p1\", \"p2\", \"p3\")"); } { String q1 = Q.p("f1").containsNear("p1", "p2", "p3") - .semicolon() .build(); String q2 = Q.p("f1").containsNear(Arrays.asList("p1", "p2", "p3")) - .semicolon() .build(); - assertEquals(q1, "yql=select * from sources * where f1 contains near(\"p1\", \"p2\", \"p3\");"); - assertEquals(q2, "yql=select * from sources * where f1 contains near(\"p1\", \"p2\", \"p3\");"); + assertEquals(q1, "yql=select * from sources * where f1 contains near(\"p1\", \"p2\", \"p3\")"); + assertEquals(q2, "yql=select * from sources * where f1 contains near(\"p1\", \"p2\", \"p3\")"); } { String q1 = Q.p("f1").containsOnear("p1", "p2", "p3") - .semicolon() .build(); String q2 = Q.p("f1").containsOnear(Arrays.asList("p1", "p2", "p3")) - .semicolon() .build(); - assertEquals(q1, "yql=select * from sources * where f1 contains onear(\"p1\", \"p2\", \"p3\");"); - assertEquals(q2, "yql=select * from sources * where f1 contains onear(\"p1\", \"p2\", \"p3\");"); + assertEquals(q1, "yql=select * from sources * where f1 contains onear(\"p1\", \"p2\", \"p3\")"); + assertEquals(q2, "yql=select * from sources * where f1 contains onear(\"p1\", \"p2\", \"p3\")"); } { String q1 = Q.p("f1").containsEquiv("p1", "p2", "p3") - .semicolon() .build(); String q2 = Q.p("f1").containsEquiv(Arrays.asList("p1", "p2", "p3")) - .semicolon() .build(); - assertEquals(q1, "yql=select * from sources * where f1 contains equiv(\"p1\", \"p2\", \"p3\");"); - assertEquals(q2, "yql=select * from sources * where f1 contains equiv(\"p1\", \"p2\", \"p3\");"); + assertEquals(q1, "yql=select * from sources * where f1 contains equiv(\"p1\", \"p2\", \"p3\")"); + assertEquals(q2, "yql=select * from sources * where f1 contains equiv(\"p1\", \"p2\", \"p3\")"); } } @Test void contains_uri() { String q = Q.p("f1").containsUri("https://test.uri") - .semicolon() .build(); - assertEquals(q, "yql=select * from sources * where f1 contains uri(\"https://test.uri\");"); + assertEquals(q, "yql=select * from sources * where f1 contains uri(\"https://test.uri\")"); } @Test void contains_uri_with_annotation() { String q = Q.p("f1").containsUri(A.a("key", "value"), "https://test.uri") - .semicolon() .build(); - assertEquals(q, "yql=select * from sources * where f1 contains ([{\"key\":\"value\"}]uri(\"https://test.uri\"));"); + assertEquals(q, "yql=select * from sources * where f1 contains ([{\"key\":\"value\"}]uri(\"https://test.uri\"))"); } @Test void nearestNeighbor() { String q = Q.p("f1").nearestNeighbor("query_vector") - .semicolon() .build(); - assertEquals(q, "yql=select * from sources * where nearestNeighbor(f1, query_vector);"); + assertEquals(q, "yql=select * from sources * where nearestNeighbor(f1, query_vector)"); } @Test void nearestNeighbor_with_annotation() { String q = Q.p("f1").nearestNeighbor(A.a("targetHits", 10), "query_vector") - .semicolon() .build(); - assertEquals(q, "yql=select * from sources * where ([{\"targetHits\":10}]nearestNeighbor(f1, query_vector));"); + assertEquals(q, "yql=select * from sources * where ([{\"targetHits\":10}]nearestNeighbor(f1, query_vector))"); } @Test void use_contains_instead_of_contains_equiv_when_input_size_is_1() { String q = Q.p("f1").containsEquiv(Collections.singletonList("p1")) - .semicolon() .build(); - assertEquals(q, "yql=select * from sources * where f1 contains \"p1\";"); + assertEquals(q, "yql=select * from sources * where f1 contains \"p1\""); } @Test void contains_phrase_near_onear_equiv_empty_list_should_throw_illegal_argument_exception() { assertThrows(IllegalArgumentException.class, () -> Q.p("f1").containsPhrase(Collections.emptyList()) - .semicolon() .build()); assertThrows(IllegalArgumentException.class, () -> Q.p("f1").containsNear(Collections.emptyList()) - .semicolon() .build()); assertThrows(IllegalArgumentException.class, () -> Q.p("f1").containsOnear(Collections.emptyList()) - .semicolon() .build()); assertThrows(IllegalArgumentException.class, () -> Q.p("f1").containsEquiv(Collections.emptyList()) - .semicolon() .build()); } @@ -550,17 +502,15 @@ class QTest { void contains_near_onear_with_annotation() { { String q = Q.p("f1").containsNear(A.a("distance", 5), "p1", "p2", "p3") - .semicolon() .build(); - assertEquals(q, "yql=select * from sources * where f1 contains ([{\"distance\":5}]near(\"p1\", \"p2\", \"p3\"));"); + assertEquals(q, "yql=select * from sources * where f1 contains ([{\"distance\":5}]near(\"p1\", \"p2\", \"p3\"))"); } { String q = Q.p("f1").containsOnear(A.a("distance", 5), "p1", "p2", "p3") - .semicolon() .build(); - assertEquals(q, "yql=select * from sources * where f1 contains ([{\"distance\":5}]onear(\"p1\", \"p2\", \"p3\"));"); + assertEquals(q, "yql=select * from sources * where f1 contains ([{\"distance\":5}]onear(\"p1\", \"p2\", \"p3\"))"); } } @@ -588,10 +538,9 @@ class QTest { )) )) ) - .semicolon() .build(); - assertEquals(q, "yql=select * from sources * where f1 contains \"v1\" | all(group(a) max(5) each(output(count()) all(max(1) each(output(summary()))) all(group(b) each(output(count()) all(max(1) each(output(summary()))) all(group(c) each(output(count()) all(max(1) each(output(summary())))))))));"); + assertEquals(q, "yql=select * from sources * where f1 contains \"v1\" | all(group(a) max(5) each(output(count()) all(max(1) each(output(summary()))) all(group(b) each(output(count()) all(max(1) each(output(summary()))) all(group(c) each(output(count()) all(max(1) each(output(summary())))))))))"); } @Test @@ -608,10 +557,9 @@ class QTest { */ String q = Q.p("f1").contains("v1") .group("all(group(a) max(5) each(output(count()) all(max(1) each(output(summary()))) all(group(b) each(output(count()) all(max(1) each(output(summary()))) all(group(c) each(output(count()) all(max(1) each(output(summary())))))))))") - .semicolon() .build(); - assertEquals(q, "yql=select * from sources * where f1 contains \"v1\" | all(group(a) max(5) each(output(count()) all(max(1) each(output(summary()))) all(group(b) each(output(count()) all(max(1) each(output(summary()))) all(group(c) each(output(count()) all(max(1) each(output(summary())))))))));"); + assertEquals(q, "yql=select * from sources * where f1 contains \"v1\" | all(group(a) max(5) each(output(count()) all(max(1) each(output(summary()))) all(group(b) each(output(count()) all(max(1) each(output(summary()))) all(group(c) each(output(count()) all(max(1) each(output(summary())))))))))"); } @Test @@ -631,7 +579,7 @@ class QTest { .reduce(Query::and) .get(); - assertEquals(q.semicolon().build(), "yql=select * from sources * where a contains \"1\" and b contains \"2\" and c contains \"3\";"); + assertEquals(q.build(), "yql=select * from sources * where a contains \"1\" and b contains \"2\" and c contains \"3\""); } @Test @@ -641,14 +589,14 @@ class QTest { map.forEach((k, v) -> q.and(Q.p(k).contains(v))); - assertEquals(q.semicolon().build(), "yql=select * from sources * where a contains \"1\" and b contains \"2\" and c contains \"3\";"); + assertEquals(q.build(), "yql=select * from sources * where a contains \"1\" and b contains \"2\" and c contains \"3\""); } @Test void empty_queries_should_not_print_out() { - String q = Q.p(Q.p(Q.p().andnot(Q.p()).and(Q.p()))).and("a").contains("1").semicolon().build(); + String q = Q.p(Q.p(Q.p().andnot(Q.p()).and(Q.p()))).and("a").contains("1").build(); - assertEquals(q, "yql=select * from sources * where a contains \"1\";"); + assertEquals(q, "yql=select * from sources * where a contains \"1\""); } @Test diff --git a/cloud-tenant-base-dependencies-enforcer/pom.xml b/cloud-tenant-base-dependencies-enforcer/pom.xml index 8ba9794d75c..05ffa0b5c97 100644 --- a/cloud-tenant-base-dependencies-enforcer/pom.xml +++ b/cloud-tenant-base-dependencies-enforcer/pom.xml @@ -43,7 +43,8 @@ <javax.servlet-api.version>3.1.0</javax.servlet-api.version> <javax.ws.rs-api.version>2.0.1</javax.ws.rs-api.version> <jaxb.version>2.3.0</jaxb.version> - <jetty.version>11.0.12</jetty.version> + <jetty.version>9.4.49.v20220914</jetty.version> + <jetty-alpn.version>1.1.3.v20160715</jetty-alpn.version> <org.lz4.version>1.8.0</org.lz4.version> <org.json.version>20220320</org.json.version> <!-- TODO: Remove on Vespa 9 --> <slf4j.version>1.7.32</slf4j.version> <!-- WARNING: when updated, also update c.y.v.tenant:base pom --> @@ -75,158 +76,161 @@ <enforceDependencies implementation="com.yahoo.vespa.maven.plugin.enforcer.EnforceDependencies"> <allowed> <!-- MUST BE KEPT IN SYNC WITH container-dependencies-enforcer pom --> - <include>aopalliance:aopalliance:[${aopalliance.version}]:jar:provided</include> - <include>com.fasterxml.jackson.core:jackson-annotations:[${jackson2.version}]:jar:provided</include> - <include>com.fasterxml.jackson.core:jackson-core:[${jackson2.version}]:jar:provided</include> - <include>com.fasterxml.jackson.core:jackson-databind:[${jackson-databind.version}]:jar:provided</include> - <include>com.fasterxml.jackson.datatype:jackson-datatype-jdk8:[${jackson2.version}]:jar:provided</include> - <include>com.fasterxml.jackson.datatype:jackson-datatype-jsr310:[${jackson2.version}]:jar:provided</include> + <include>aopalliance:aopalliance:${aopalliance.version}:provided</include> + <include>com.fasterxml.jackson.core:jackson-annotations:${jackson2.version}:provided</include> + <include>com.fasterxml.jackson.core:jackson-core:${jackson2.version}:provided</include> + <include>com.fasterxml.jackson.core:jackson-databind:${jackson-databind.version}:provided</include> + <include>com.fasterxml.jackson.datatype:jackson-datatype-jdk8:${jackson2.version}:provided</include> + <include>com.fasterxml.jackson.datatype:jackson-datatype-jsr310:${jackson2.version}:provided</include> <!-- Guava with its internal dependencies --> - <include>com.google.guava:guava:[${guava.version}]:jar:provided</include> - <include>com.google.errorprone:error_prone_annotations:[2.2.0]:jar:provided</include> - <include>com.google.guava:failureaccess:[1.0.1]:jar:provided</include> - <include>com.google.j2objc:j2objc-annotations:[1.1]:jar:provided</include> + <include>com.google.guava:guava:${guava.version}:provided</include> + <include>com.google.errorprone:error_prone_annotations:2.2.0:provided</include> + <include>com.google.guava:failureaccess:1.0.1:provided</include> + <include>com.google.j2objc:j2objc-annotations:1.1:provided</include> - <include>com.google.inject:guice:[${guice.version}]:jar:provided:no_aop</include> - <include>com.sun.activation:javax.activation:[1.2.0]:jar:provided</include> - <include>com.sun.xml.bind:jaxb-core:[${jaxb.version}]:jar:provided</include> - <include>com.sun.xml.bind:jaxb-impl:[${jaxb.version}]:jar:provided</include> - <include>commons-logging:commons-logging:[1.2]:jar:provided</include> - <include>javax.inject:javax.inject:[${javax.inject.version}]:jar:provided</include> - <include>javax.servlet:javax.servlet-api:[${javax.servlet-api.version}]:jar:provided</include> - <include>javax.ws.rs:javax.ws.rs-api:[${javax.ws.rs-api.version}]:jar:provided</include> - <include>javax.xml.bind:jaxb-api:[${jaxb.version}]:jar:provided</include> - <include>org.slf4j:jcl-over-slf4j:[${slf4j.version}]:jar:provided</include> - <include>org.slf4j:log4j-over-slf4j:[${slf4j.version}]:jar:provided</include> - <include>org.slf4j:slf4j-api:[${slf4j.version}]:jar:provided</include> - <include>org.slf4j:slf4j-jdk14:[${slf4j.version}]:jar:provided</include> - <include>xml-apis:xml-apis:[${xml-apis.version}]:jar:provided</include> + <include>com.google.inject:guice:jar:no_aop:${guice.version}:provided</include> + <include>com.sun.activation:javax.activation:1.2.0:provided</include> + <include>com.sun.xml.bind:jaxb-core:${jaxb.version}:provided</include> + <include>com.sun.xml.bind:jaxb-impl:${jaxb.version}:provided</include> + <include>commons-logging:commons-logging:1.2:provided</include> + <include>javax.inject:javax.inject:${javax.inject.version}:provided</include> + <include>javax.servlet:javax.servlet-api:${javax.servlet-api.version}:provided</include> + <include>javax.ws.rs:javax.ws.rs-api:${javax.ws.rs-api.version}:provided</include> + <include>javax.xml.bind:jaxb-api:${jaxb.version}:provided</include> + <include>org.slf4j:jcl-over-slf4j:${slf4j.version}:provided</include> + <include>org.slf4j:log4j-over-slf4j:${slf4j.version}:provided</include> + <include>org.slf4j:slf4j-api:${slf4j.version}:provided</include> + <include>org.slf4j:slf4j-jdk14:${slf4j.version}:provided</include> + <include>xml-apis:xml-apis:${xml-apis.version}:provided</include> <!-- Vespa provided dependencies --> - <include>com.yahoo.vespa:annotations:*:jar:provided</include> - <include>com.yahoo.vespa:component:*:jar:provided</include> - <include>com.yahoo.vespa:config-bundle:*:jar:provided</include> - <include>com.yahoo.vespa:config-lib:*:jar:provided</include> - <include>com.yahoo.vespa:config:*:jar:provided</include> - <include>com.yahoo.vespa:configdefinitions:*:jar:provided</include> - <include>com.yahoo.vespa:configgen:*:jar:provided</include> - <include>com.yahoo.vespa:container-core:*:jar:provided</include> - <include>com.yahoo.vespa:container-dev:*:jar:provided</include> - <include>com.yahoo.vespa:container-disc:*:jar:provided</include> - <include>com.yahoo.vespa:container-documentapi:*:jar:provided</include> - <include>com.yahoo.vespa:container-messagebus:*:jar:provided</include> - <include>com.yahoo.vespa:container-search-and-docproc:*:jar:provided</include> - <include>com.yahoo.vespa:container-search:*:jar:provided</include> - <include>com.yahoo.vespa:container:*:jar:provided</include> - <include>com.yahoo.vespa:defaults:*:jar:provided</include> - <include>com.yahoo.vespa:docproc:*:jar:provided</include> - <include>com.yahoo.vespa:document:*:jar:provided</include> - <include>com.yahoo.vespa:documentapi:*:jar:provided</include> - <include>com.yahoo.vespa:fileacquirer:*:jar:provided</include> - <include>com.yahoo.vespa:fsa:*:jar:provided</include> - <include>com.yahoo.vespa:hosted-zone-api:*:jar:provided</include> - <include>com.yahoo.vespa:http-utils:*:jar:provided</include> - <include>com.yahoo.vespa:jdisc_core:*:jar:provided</include> - <include>com.yahoo.vespa:jrt:*:jar:provided</include> - <include>com.yahoo.vespa:linguistics:*:jar:provided</include> - <include>com.yahoo.vespa:messagebus:*:jar:provided</include> - <include>com.yahoo.vespa:model-evaluation:*:jar:provided</include> - <include>com.yahoo.vespa:predicate-search-core:*:jar:provided</include> - <include>com.yahoo.vespa:provided-dependencies:*:jar:provided</include> - <include>com.yahoo.vespa:searchcore:*:jar:provided</include> - <include>com.yahoo.vespa:searchlib:*:jar:provided</include> - <include>com.yahoo.vespa:security-utils:*:jar:provided</include> - <include>com.yahoo.vespa:vdslib:*:jar:provided</include> - <include>com.yahoo.vespa:vespa-3party-bundles:*:pom:provided</include> - <include>com.yahoo.vespa:vespaclient-container-plugin:*:jar:provided</include> - <include>com.yahoo.vespa:vespajlib:*:jar:provided</include> - <include>com.yahoo.vespa:vespalog:*:jar:provided</include> + <include>com.yahoo.vespa:annotations:*:provided</include> + <include>com.yahoo.vespa:component:*:provided</include> + <include>com.yahoo.vespa:config-bundle:*:provided</include> + <include>com.yahoo.vespa:config-lib:*:provided</include> + <include>com.yahoo.vespa:config:*:provided</include> + <include>com.yahoo.vespa:configdefinitions:*:provided</include> + <include>com.yahoo.vespa:configgen:*:provided</include> + <include>com.yahoo.vespa:container-core:*:provided</include> + <include>com.yahoo.vespa:container-dev:*:provided</include> + <include>com.yahoo.vespa:container-disc:*:provided</include> + <include>com.yahoo.vespa:container-documentapi:*:provided</include> + <include>com.yahoo.vespa:container-messagebus:*:provided</include> + <include>com.yahoo.vespa:container-search-and-docproc:*:provided</include> + <include>com.yahoo.vespa:container-search:*:provided</include> + <include>com.yahoo.vespa:container:*:provided</include> + <include>com.yahoo.vespa:defaults:*:provided</include> + <include>com.yahoo.vespa:docproc:*:provided</include> + <include>com.yahoo.vespa:document:*:provided</include> + <include>com.yahoo.vespa:documentapi:*:provided</include> + <include>com.yahoo.vespa:fileacquirer:*:provided</include> + <include>com.yahoo.vespa:fsa:*:provided</include> + <include>com.yahoo.vespa:hosted-zone-api:*:provided</include> + <include>com.yahoo.vespa:http-utils:*:provided</include> + <include>com.yahoo.vespa:jdisc_core:*:provided</include> + <include>com.yahoo.vespa:jrt:*:provided</include> + <include>com.yahoo.vespa:linguistics:*:provided</include> + <include>com.yahoo.vespa:messagebus:*:provided</include> + <include>com.yahoo.vespa:model-evaluation:*:provided</include> + <include>com.yahoo.vespa:predicate-search-core:*:provided</include> + <include>com.yahoo.vespa:provided-dependencies:*:provided</include> + <include>com.yahoo.vespa:searchcore:*:provided</include> + <include>com.yahoo.vespa:searchlib:*:provided</include> + <include>com.yahoo.vespa:security-utils:*:provided</include> + <include>com.yahoo.vespa:vdslib:*:provided</include> + <include>com.yahoo.vespa:vespa-3party-bundles:pom:*:provided</include> + <include>com.yahoo.vespa:vespaclient-container-plugin:*:provided</include> + <include>com.yahoo.vespa:vespajlib:*:provided</include> + <include>com.yahoo.vespa:vespalog:*:provided</include> <!-- Vespa test dependencies --> - <include>com.yahoo.vespa:application:*:jar:test</include> - <include>com.yahoo.vespa:cloud-tenant-cd:*:jar:test</include> - <include>com.yahoo.vespa:config-application-package:*:jar:test</include> - <include>com.yahoo.vespa:config-model-api:*:jar:test</include> - <include>com.yahoo.vespa:config-model:*:jar:test</include> - <include>com.yahoo.vespa:config-provisioning:*:jar:test</include> - <include>com.yahoo.vespa:container-apache-http-client-bundle:*:jar:test</include> - <include>com.yahoo.vespa:container-test:*:jar:test</include> - <include>com.yahoo.vespa:hosted-api:*:jar:test</include> - <include>com.yahoo.vespa:indexinglanguage:*:jar:test</include> - <include>com.yahoo.vespa:logd:*:jar:test</include> - <include>com.yahoo.vespa:metrics-proxy:*:jar:test</include> - <include>com.yahoo.vespa:metrics:*:jar:test</include> - <include>com.yahoo.vespa:model-integration:*:jar:test</include> - <include>com.yahoo.vespa:searchsummary:*:jar:test</include> - <include>com.yahoo.vespa:standalone-container:*:jar:test</include> - <include>com.yahoo.vespa:storage:*:jar:test</include> - <include>com.yahoo.vespa:tenant-cd-api:*:jar:test</include> - <include>com.yahoo.vespa:tenant-cd-commons:*:jar:test</include> - <include>com.yahoo.vespa:vespa-feed-client:*:jar:test</include> - <include>com.yahoo.vespa:vespa-feed-client-api:*:jar:test</include> - <include>com.yahoo.vespa:vespaclient-core:*:jar:test</include> - <include>com.yahoo.vespa:vsm:*:jar:test</include> + <include>com.yahoo.vespa:application:*:test</include> + <include>com.yahoo.vespa:cloud-tenant-cd:*:test</include> + <include>com.yahoo.vespa:config-application-package:*:test</include> + <include>com.yahoo.vespa:config-model-api:*:test</include> + <include>com.yahoo.vespa:config-model:*:test</include> + <include>com.yahoo.vespa:config-provisioning:*:test</include> + <include>com.yahoo.vespa:container-apache-http-client-bundle:*:test</include> + <include>com.yahoo.vespa:container-test:*:test</include> + <include>com.yahoo.vespa:hosted-api:*:test</include> + <include>com.yahoo.vespa:indexinglanguage:*:test</include> + <include>com.yahoo.vespa:logd:*:test</include> + <include>com.yahoo.vespa:metrics-proxy:*:test</include> + <include>com.yahoo.vespa:metrics:*:test</include> + <include>com.yahoo.vespa:model-integration:*:test</include> + <include>com.yahoo.vespa:searchsummary:*:test</include> + <include>com.yahoo.vespa:standalone-container:*:test</include> + <include>com.yahoo.vespa:storage:*:test</include> + <include>com.yahoo.vespa:tenant-cd-api:*:test</include> + <include>com.yahoo.vespa:tenant-cd-commons:*:test</include> + <include>com.yahoo.vespa:vespa-feed-client:*:test</include> + <include>com.yahoo.vespa:vespa-feed-client-api:*:test</include> + <include>com.yahoo.vespa:vespaclient-core:*:test</include> + <include>com.yahoo.vespa:vsm:*:test</include> <!-- 3rd party test dependencies --> - <include>com.google.code.findbugs:jsr305:3.0.2:jar:test</include> - <include>com.google.protobuf:protobuf-java:3.7.0:jar:test</include> - <include>com.ibm.icu:icu4j:57.1:jar:test</include> - <include>com.microsoft.onnxruntime:onnxruntime:[${onnxruntime.version}]:jar:test</include> - <include>com.thaiopensource:jing:20091111:jar:test</include> - <include>commons-codec:commons-codec:[${commons-codec.version}]:jar:test</include> - <include>io.airlift:aircompressor:0.17:jar:test</include> - <include>io.airlift:airline:0.7:jar:test</include> - <include>io.prometheus:simpleclient:0.6.0:jar:test</include> - <include>io.prometheus:simpleclient_common:0.6.0:jar:test</include> - <include>junit:junit:4.13.2:jar:test</include> - <include>net.java.dev.jna:jna:5.11.0:jar:test</include> - <include>org.antlr:antlr-runtime:3.5.2:jar:test</include> - <include>org.antlr:antlr4-runtime:4.9.3:jar:test</include> - <include>org.apache.commons:commons-exec:1.3:jar:test</include> - <include>org.apache.commons:commons-compress:1.21:jar:test</include> - <include>org.apache.commons:commons-math3:3.6.1:jar:test</include> - <include>org.apache.felix:org.apache.felix.framework:[${felix.version}]:jar:test</include> - <include>org.apache.felix:org.apache.felix.log:1.0.1:jar:test</include> - <include>org.apache.httpcomponents.client5:httpclient5:${httpclient5.version}:jar:test</include> - <include>org.apache.httpcomponents.core5:httpcore5:${httpclient5.version}:jar:test</include> - <include>org.apache.httpcomponents.core5:httpcore5-h2:${httpclient5.version}:jar:test</include> - <include>org.apache.httpcomponents:httpclient:${httpclient.version}:jar:test</include> - <include>org.apache.httpcomponents:httpcore:${httpcore.version}:jar:test</include> - <include>org.apache.httpcomponents:httpmime:${httpclient.version}:jar:test</include> - <include>org.apache.opennlp:opennlp-tools:1.9.3:jar:test</include> - <include>org.apiguardian:apiguardian-api:1.1.0:jar:test</include> - <include>org.bouncycastle:bcpkix-jdk18on:[${bouncycastle.version}]:jar:test</include> - <include>org.bouncycastle:bcprov-jdk18on:[${bouncycastle.version}]:jar:test</include> - <include>org.bouncycastle:bcutil-jdk18on:[${bouncycastle.version}]:jar:test</include> - <include>org.eclipse.jetty.http2:http2-common:[${jetty.version}]:jar:test</include> - <include>org.eclipse.jetty.http2:http2-hpack:[${jetty.version}]:jar:test</include> - <include>org.eclipse.jetty.http2:http2-server:[${jetty.version}]:jar:test</include> - <include>org.eclipse.jetty.toolchain:jetty-jakarta-servlet-api:5.0.2:jar:test</include> - <include>org.eclipse.jetty:jetty-alpn-client:[${jetty.version}]:jar:test</include> - <include>org.eclipse.jetty:jetty-alpn-java-server:[${jetty.version}]:jar:test</include> - <include>org.eclipse.jetty:jetty-alpn-server:[${jetty.version}]:jar:test</include> - <include>org.eclipse.jetty:jetty-client:[${jetty.version}]:jar:test</include> - <include>org.eclipse.jetty:jetty-http:[${jetty.version}]:jar:test</include> - <include>org.eclipse.jetty:jetty-io:[${jetty.version}]:jar:test</include> - <include>org.eclipse.jetty:jetty-jmx:[${jetty.version}]:jar:test</include> - <include>org.eclipse.jetty:jetty-security:[${jetty.version}]:jar:test</include> - <include>org.eclipse.jetty:jetty-server:[${jetty.version}]:jar:test</include> - <include>org.eclipse.jetty:jetty-servlet:[${jetty.version}]:jar:test</include> - <include>org.eclipse.jetty:jetty-util:[${jetty.version}]:jar:test</include> - <include>org.hamcrest:hamcrest-core:1.3:jar:test</include> - <include>org.hdrhistogram:HdrHistogram:2.1.8:jar:test</include> - <include>org.json:json:${org.json.version}:jar:test</include> - <include>org.junit.jupiter:junit-jupiter-api:[${junit5.version}]:jar:test</include> - <include>org.junit.jupiter:junit-jupiter-engine:[${junit5.version}]:jar:test</include> - <include>org.junit.platform:junit-platform-commons:[${junit5.platform.version}]:jar:test</include> - <include>org.junit.platform:junit-platform-engine:[${junit5.platform.version}]:jar:test</include> - <include>org.junit.vintage:junit-vintage-engine:[${junit5.version}]:jar:test</include> - <include>org.lz4:lz4-java:[${org.lz4.version}]:jar:test</include> - <include>org.opentest4j:opentest4j:1.2.0:jar:test</include> - <include>org.osgi:org.osgi.compendium:4.1.0:jar:test</include> - <include>org.osgi:org.osgi.core:4.1.0:jar:test</include> - <include>xerces:xercesImpl:2.12.1:jar:test</include> + <include>com.google.code.findbugs:jsr305:3.0.2:test</include> + <include>com.google.protobuf:protobuf-java:3.21.7:test</include> + <include>com.ibm.icu:icu4j:70.1:test</include> + <include>com.microsoft.onnxruntime:onnxruntime:${onnxruntime.version}:test</include> + <include>com.thaiopensource:jing:20091111:test</include> + <include>commons-codec:commons-codec:${commons-codec.version}:test</include> + <include>io.airlift:aircompressor:0.21:test</include> + <include>io.airlift:airline:0.9:test</include> + <include>io.prometheus:simpleclient:0.6.0:test</include> + <include>io.prometheus:simpleclient_common:0.6.0:test</include> + <include>junit:junit:4.13.2:test</include> + <include>net.java.dev.jna:jna:5.11.0:test</include> + <include>org.antlr:antlr-runtime:3.5.2:test</include> + <include>org.antlr:antlr4-runtime:4.9.3:test</include> + <include>org.apache.commons:commons-exec:1.3:test</include> + <include>org.apache.commons:commons-compress:1.21:test</include> + <include>org.apache.commons:commons-math3:3.6.1:test</include> + <include>org.apache.felix:org.apache.felix.framework:${felix.version}:test</include> + <include>org.apache.felix:org.apache.felix.log:1.0.1:test</include> + <include>org.apache.httpcomponents.client5:httpclient5:${httpclient5.version}:test</include> + <include>org.apache.httpcomponents.core5:httpcore5:${httpclient5.version}:test</include> + <include>org.apache.httpcomponents.core5:httpcore5-h2:${httpclient5.version}:test</include> + <include>org.apache.httpcomponents:httpclient:${httpclient.version}:test</include> + <include>org.apache.httpcomponents:httpcore:${httpcore.version}:test</include> + <include>org.apache.httpcomponents:httpmime:${httpclient.version}:test</include> + <include>org.apache.opennlp:opennlp-tools:1.9.3:test</include> + <include>org.apiguardian:apiguardian-api:1.1.2:test</include> + <include>org.bouncycastle:bcpkix-jdk18on:${bouncycastle.version}:test</include> + <include>org.bouncycastle:bcprov-jdk18on:${bouncycastle.version}:test</include> + <include>org.bouncycastle:bcutil-jdk18on:${bouncycastle.version}:test</include> + <include>org.eclipse.jetty.alpn:alpn-api:jar:${jetty-alpn.version}:test</include> + <include>org.eclipse.jetty.http2:http2-common:${jetty.version}:test</include> + <include>org.eclipse.jetty.http2:http2-hpack:${jetty.version}:test</include> + <include>org.eclipse.jetty.http2:http2-server:${jetty.version}:test</include> + <include>org.eclipse.jetty:jetty-alpn-java-server:${jetty.version}:test</include> + <include>org.eclipse.jetty:jetty-alpn-server:${jetty.version}:test</include> + <include>org.eclipse.jetty:jetty-client:${jetty.version}:test</include> + <include>org.eclipse.jetty:jetty-continuation:jar:${jetty.version}:test</include> + <include>org.eclipse.jetty:jetty-http:${jetty.version}:test</include> + <include>org.eclipse.jetty:jetty-io:${jetty.version}:test</include> + <include>org.eclipse.jetty:jetty-jmx:${jetty.version}:test</include> + <include>org.eclipse.jetty:jetty-security:${jetty.version}:test</include> + <include>org.eclipse.jetty:jetty-server:${jetty.version}:test</include> + <include>org.eclipse.jetty:jetty-servlet:${jetty.version}:test</include> + <include>org.eclipse.jetty:jetty-servlets:jar:${jetty.version}:test</include> + <include>org.eclipse.jetty:jetty-util-ajax:jar:${jetty.version}:test</include> + <include>org.eclipse.jetty:jetty-util:${jetty.version}:test</include> + + <include>org.hamcrest:hamcrest-core:1.3:test</include> + <include>org.hdrhistogram:HdrHistogram:2.1.12:test</include> + <include>org.json:json:${org.json.version}:test</include> + <include>org.junit.jupiter:junit-jupiter-api:${junit5.version}:test</include> + <include>org.junit.jupiter:junit-jupiter-engine:${junit5.version}:test</include> + <include>org.junit.platform:junit-platform-commons:${junit5.platform.version}:test</include> + <include>org.junit.platform:junit-platform-engine:${junit5.platform.version}:test</include> + <include>org.junit.vintage:junit-vintage-engine:${junit5.version}:test</include> + <include>org.lz4:lz4-java:${org.lz4.version}:test</include> + <include>org.opentest4j:opentest4j:1.2.0:test</include> + <include>org.osgi:org.osgi.compendium:4.1.0:test</include> + <include>org.osgi:org.osgi.core:4.1.0:test</include> + <include>xerces:xercesImpl:2.12.2:test</include> </allowed> </enforceDependencies> </rules> diff --git a/container-core/pom.xml b/container-core/pom.xml index 52d7f3372f0..ed4b05495e3 100644 --- a/container-core/pom.xml +++ b/container-core/pom.xml @@ -14,7 +14,6 @@ <artifactId>container-core</artifactId> <version>8-SNAPSHOT</version> <packaging>container-plugin</packaging> - <dependencies> <!-- COMPILE scope --> @@ -115,119 +114,40 @@ <!-- START JETTY embedded jars --> <dependency> - <groupId>org.eclipse.jetty.http2</groupId> - <artifactId>http2-common</artifactId> - <exclusions> - <exclusion> - <groupId>org.slf4j</groupId> - <artifactId>slf4j-api</artifactId> - </exclusion> - </exclusions> + <groupId>org.eclipse.jetty.alpn</groupId> + <artifactId>alpn-api</artifactId> </dependency> <dependency> <groupId>org.eclipse.jetty.http2</groupId> <artifactId>http2-server</artifactId> - <exclusions> - <exclusion> - <groupId>org.slf4j</groupId> - <artifactId>slf4j-api</artifactId> - </exclusion> - </exclusions> </dependency> <dependency> - <!-- Required for JDK9ServerALPNProcessor through ServiceLoader API --> <groupId>org.eclipse.jetty</groupId> <artifactId>jetty-alpn-java-server</artifactId> - <exclusions> - <exclusion> - <groupId>org.slf4j</groupId> - <artifactId>slf4j-api</artifactId> - </exclusion> - </exclusions> - </dependency> - <dependency> - <groupId>org.eclipse.jetty</groupId> - <artifactId>jetty-alpn-server</artifactId> - <exclusions> - <exclusion> - <groupId>org.slf4j</groupId> - <artifactId>slf4j-api</artifactId> - </exclusion> - </exclusions> </dependency> <dependency> <groupId>org.eclipse.jetty</groupId> <artifactId>jetty-client</artifactId> - <exclusions> - <exclusion> - <groupId>org.slf4j</groupId> - <artifactId>slf4j-api</artifactId> - </exclusion> - </exclusions> </dependency> <dependency> <groupId>org.eclipse.jetty</groupId> - <artifactId>jetty-http</artifactId> - <exclusions> - <exclusion> - <groupId>org.slf4j</groupId> - <artifactId>slf4j-api</artifactId> - </exclusion> - </exclusions> - </dependency> - <dependency> - <groupId>org.eclipse.jetty</groupId> - <artifactId>jetty-io</artifactId> - <exclusions> - <exclusion> - <groupId>org.slf4j</groupId> - <artifactId>slf4j-api</artifactId> - </exclusion> - </exclusions> + <artifactId>jetty-continuation</artifactId> </dependency> <dependency> <groupId>org.eclipse.jetty</groupId> <artifactId>jetty-jmx</artifactId> - <exclusions> - <exclusion> - <groupId>org.slf4j</groupId> - <artifactId>slf4j-api</artifactId> - </exclusion> - </exclusions> </dependency> <dependency> <groupId>org.eclipse.jetty</groupId> <artifactId>jetty-server</artifactId> - <exclusions> - <exclusion> - <groupId>org.slf4j</groupId> - <artifactId>slf4j-api</artifactId> - </exclusion> - </exclusions> </dependency> <dependency> <groupId>org.eclipse.jetty</groupId> <artifactId>jetty-servlet</artifactId> - <exclusions> - <exclusion> - <groupId>org.slf4j</groupId> - <artifactId>slf4j-api</artifactId> - </exclusion> - </exclusions> </dependency> <dependency> <groupId>org.eclipse.jetty</groupId> - <artifactId>jetty-util</artifactId> - <exclusions> - <exclusion> - <groupId>org.slf4j</groupId> - <artifactId>slf4j-api</artifactId> - </exclusion> - </exclusions> - </dependency> - <dependency> - <groupId>org.eclipse.jetty.toolchain</groupId> - <artifactId>jetty-jakarta-servlet-api</artifactId> + <artifactId>jetty-servlets</artifactId> </dependency> <!-- END JETTY embedded jars --> @@ -336,6 +256,11 @@ <scope>provided</scope> </dependency> <dependency> + <groupId>javax.servlet</groupId> + <artifactId>javax.servlet-api</artifactId> + <scope>provided</scope> + </dependency> + <dependency> <groupId>javax.xml.bind</groupId> <artifactId>jaxb-api</artifactId> <scope>provided</scope> diff --git a/container-core/src/main/java/com/yahoo/container/jdisc/utils/MultiPartFormParser.java b/container-core/src/main/java/com/yahoo/container/jdisc/utils/MultiPartFormParser.java index f974eb5f26c..104d2f8ae4a 100644 --- a/container-core/src/main/java/com/yahoo/container/jdisc/utils/MultiPartFormParser.java +++ b/container-core/src/main/java/com/yahoo/container/jdisc/utils/MultiPartFormParser.java @@ -2,9 +2,9 @@ package com.yahoo.container.jdisc.utils; import com.yahoo.container.jdisc.HttpRequest; -import jakarta.servlet.http.Part; -import org.eclipse.jetty.server.MultiPartFormInputStream; +import org.eclipse.jetty.http.MultiPartFormInputStream; +import javax.servlet.http.Part; import java.io.IOException; import java.io.InputStream; import java.util.Map; diff --git a/container-core/src/main/java/com/yahoo/jdisc/http/Cookie.java b/container-core/src/main/java/com/yahoo/jdisc/http/Cookie.java index c2faa1cd10a..b194124294c 100644 --- a/container-core/src/main/java/com/yahoo/jdisc/http/Cookie.java +++ b/container-core/src/main/java/com/yahoo/jdisc/http/Cookie.java @@ -2,7 +2,7 @@ package com.yahoo.jdisc.http; import org.eclipse.jetty.http.HttpCookie; -import org.eclipse.jetty.server.Cookies; +import org.eclipse.jetty.server.CookieCutter; import java.util.Arrays; import java.util.HashSet; @@ -180,7 +180,7 @@ public class Cookie { } public static List<Cookie> fromCookieHeader(String headerVal) { - Cookies cookieCutter = new Cookies(); + CookieCutter cookieCutter = new CookieCutter(); cookieCutter.addCookieField(headerVal); return Arrays.stream(cookieCutter.getCookies()) .map(servletCookie -> { diff --git a/container-core/src/main/java/com/yahoo/jdisc/http/HttpRequest.java b/container-core/src/main/java/com/yahoo/jdisc/http/HttpRequest.java index 4ad38a9f965..598a924b327 100644 --- a/container-core/src/main/java/com/yahoo/jdisc/http/HttpRequest.java +++ b/container-core/src/main/java/com/yahoo/jdisc/http/HttpRequest.java @@ -8,14 +8,15 @@ import com.yahoo.jdisc.handler.ContentChannel; import com.yahoo.jdisc.handler.RequestHandler; import com.yahoo.jdisc.handler.ResponseHandler; import com.yahoo.jdisc.service.CurrentContainer; +import org.eclipse.jetty.http.HttpURI; import org.eclipse.jetty.util.MultiMap; -import org.eclipse.jetty.util.UrlEncoded; import java.net.InetAddress; import java.net.InetSocketAddress; import java.net.SocketAddress; import java.net.URI; import java.security.Principal; +import java.util.ArrayList; import java.util.Collections; import java.util.HashMap; import java.util.LinkedList; @@ -115,10 +116,15 @@ public class HttpRequest extends Request { } private static Map<String, List<String>> getUriQueryParameters(URI uri) { - if (uri.getRawQuery() == null) return Map.of(); - MultiMap<String> params = new MultiMap<>(); - UrlEncoded.decodeUtf8To(uri.getRawQuery(), params); - return Map.copyOf(params); + MultiMap<String> queryParameters = new MultiMap<>(); + new HttpURI(uri).decodeQueryTo(queryParameters); + + // Do a deep copy so we do not leak Jetty classes outside + Map<String, List<String>> deepCopiedQueryParameters = new HashMap<>(); + for (Map.Entry<String, List<String>> entry : queryParameters.entrySet()) { + deepCopiedQueryParameters.put(entry.getKey(), new ArrayList<>(entry.getValue())); + } + return deepCopiedQueryParameters; } public Method getMethod() { diff --git a/container-core/src/main/java/com/yahoo/jdisc/http/server/jetty/AccessLogRequestLog.java b/container-core/src/main/java/com/yahoo/jdisc/http/server/jetty/AccessLogRequestLog.java index 5b51eeee7d6..13a63efeaa9 100644 --- a/container-core/src/main/java/com/yahoo/jdisc/http/server/jetty/AccessLogRequestLog.java +++ b/container-core/src/main/java/com/yahoo/jdisc/http/server/jetty/AccessLogRequestLog.java @@ -8,7 +8,6 @@ import com.yahoo.container.logging.RequestLog; import com.yahoo.container.logging.RequestLogEntry; import com.yahoo.jdisc.http.HttpRequest; import com.yahoo.jdisc.http.ServerConfig; -import jakarta.servlet.http.HttpServletRequest; import org.eclipse.jetty.http2.HTTP2Stream; import org.eclipse.jetty.http2.server.HttpTransportOverHTTP2; import org.eclipse.jetty.server.HttpChannel; @@ -17,6 +16,7 @@ import org.eclipse.jetty.server.Request; import org.eclipse.jetty.server.Response; import org.eclipse.jetty.util.component.AbstractLifeCycle; +import javax.servlet.http.HttpServletRequest; import java.security.cert.X509Certificate; import java.time.Duration; import java.time.Instant; diff --git a/container-core/src/main/java/com/yahoo/jdisc/http/server/jetty/ConnectorFactory.java b/container-core/src/main/java/com/yahoo/jdisc/http/server/jetty/ConnectorFactory.java index 6632a9f6cf8..6282e334409 100644 --- a/container-core/src/main/java/com/yahoo/jdisc/http/server/jetty/ConnectorFactory.java +++ b/container-core/src/main/java/com/yahoo/jdisc/http/server/jetty/ConnectorFactory.java @@ -9,8 +9,6 @@ import com.yahoo.jdisc.http.ssl.impl.DefaultConnectorSsl; import com.yahoo.security.tls.MixedMode; import com.yahoo.security.tls.TransportSecurityUtils; import org.eclipse.jetty.alpn.server.ALPNServerConnectionFactory; -import org.eclipse.jetty.http.HttpCompliance; -import org.eclipse.jetty.http.UriCompliance; import org.eclipse.jetty.http2.server.AbstractHTTP2ServerConnectionFactory; import org.eclipse.jetty.http2.server.HTTP2CServerConnectionFactory; import org.eclipse.jetty.http2.server.HTTP2ServerConnectionFactory; @@ -139,17 +137,8 @@ public class ConnectorFactory { httpConfig.setOutputBufferSize(connectorConfig.outputBufferSize()); httpConfig.setRequestHeaderSize(connectorConfig.requestHeaderSize()); httpConfig.setResponseHeaderSize(connectorConfig.responseHeaderSize()); - - // Disable use of ByteBuffer.allocateDirect() - httpConfig.setUseInputDirectByteBuffers(false); - httpConfig.setUseOutputDirectByteBuffers(false); - - httpConfig.setHttpCompliance(HttpCompliance.RFC7230); - // TODO Vespa 9 Use default URI compliance (LEGACY == old Jetty 9.4 compliance) - httpConfig.setUriCompliance(UriCompliance.LEGACY); if (isSslEffectivelyEnabled(connectorConfig)) { - // Explicitly disable SNI checking as Jetty's SNI checking trust manager is not part of our SSLContext trust manager chain - httpConfig.addCustomizer(new SecureRequestCustomizer(false, false, -1, false)); + httpConfig.addCustomizer(new SecureRequestCustomizer()); } String serverNameFallback = connectorConfig.serverName().fallback(); if (!serverNameFallback.isBlank()) httpConfig.setServerAuthority(new HostPort(serverNameFallback)); @@ -161,7 +150,7 @@ public class ConnectorFactory { } private HTTP2ServerConnectionFactory newHttp2ConnectionFactory() { - HTTP2ServerConnectionFactory factory = new FixedHTTP2ServerConnectionFactory(newHttpConfiguration()); + HTTP2ServerConnectionFactory factory = new HTTP2ServerConnectionFactory(newHttpConfiguration()); setHttp2Config(factory); return factory; } @@ -180,14 +169,12 @@ public class ConnectorFactory { } private SslConnectionFactory newSslConnectionFactory(Metric metric, ConnectionFactory wrappedFactory) { - var fac = new SslConnectionFactory(createSslContextFactory(), wrappedFactory.getProtocol()); - fac.setDirectBuffersForDecryption(false); - fac.setDirectBuffersForDecryption(false); - fac.addBean(new SslHandshakeFailedListener(metric, connectorConfig.name(), connectorConfig.listenPort())); - return fac; + SslConnectionFactory connectionFactory = new SslConnectionFactory(createSslContextFactory(), wrappedFactory.getProtocol()); + connectionFactory.addBean(new SslHandshakeFailedListener(metric, connectorConfig.name(), connectorConfig.listenPort())); + return connectionFactory; } - private SslContextFactory.Server createSslContextFactory() { + private SslContextFactory createSslContextFactory() { DefaultConnectorSsl ssl = new DefaultConnectorSsl(); sslProvider.configureSsl(ssl, connectorConfig.name(), connectorConfig.listenPort()); return ssl.createSslContextFactory(); diff --git a/container-core/src/main/java/com/yahoo/jdisc/http/server/jetty/FixedHTTP2ServerConnectionFactory.java b/container-core/src/main/java/com/yahoo/jdisc/http/server/jetty/FixedHTTP2ServerConnectionFactory.java deleted file mode 100644 index cd82a3b2ff6..00000000000 --- a/container-core/src/main/java/com/yahoo/jdisc/http/server/jetty/FixedHTTP2ServerConnectionFactory.java +++ /dev/null @@ -1,122 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.jdisc.http.server.jetty; - -import org.eclipse.jetty.http2.HTTP2Connection; -import org.eclipse.jetty.http2.ISession; -import org.eclipse.jetty.http2.server.HTTP2ServerConnection; -import org.eclipse.jetty.http2.server.HTTP2ServerConnectionFactory; -import org.eclipse.jetty.io.Connection; -import org.eclipse.jetty.io.EndPoint; -import org.eclipse.jetty.server.Connector; -import org.eclipse.jetty.server.HttpConfiguration; -import org.eclipse.jetty.util.annotation.ManagedObject; -import org.eclipse.jetty.util.annotation.Name; -import org.eclipse.jetty.util.component.Dumpable; -import org.eclipse.jetty.util.component.Graceful; -import org.eclipse.jetty.util.component.LifeCycle; - -import java.io.IOException; -import java.util.HashSet; -import java.util.Set; -import java.util.concurrent.CompletableFuture; -import java.util.logging.Logger; - -/** - * Workaround for <a href="https://github.com/eclipse/jetty.project/issues/8811">eclipse/jetty.project#8811</a>. - * - * @author bjorncs - */ -class FixedHTTP2ServerConnectionFactory extends HTTP2ServerConnectionFactory { - - private static final Logger log = Logger.getLogger(FixedHTTP2ServerConnectionFactory.class.getName()); - - private final HTTP2SessionContainer originalSessionContainer; - private final FixedHTTP2SessionContainer fixedSessionContainer; - - FixedHTTP2ServerConnectionFactory(@Name("config") HttpConfiguration config) { - super(config); - fixedSessionContainer = new FixedHTTP2SessionContainer(); - originalSessionContainer = getBean(HTTP2SessionContainer.class); - removeBean(originalSessionContainer); - addBean(fixedSessionContainer); - } - - @Override - public Connection newConnection(Connector connector, EndPoint endPoint) { - var conn = (HTTP2ServerConnection) super.newConnection(connector, endPoint); - conn.removeEventListener(originalSessionContainer); - conn.addEventListener(fixedSessionContainer); - return conn; - } - - @ManagedObject("The container of HTTP/2 sessions") - private static class FixedHTTP2SessionContainer implements Connection.Listener, Graceful, Dumpable { - - private final Object monitor = new Object(); - private final Set<ISession> sessions = new HashSet<>(); - private CompletableFuture<Void> shutdown; - - @Override - public void onOpened(Connection conn) { - var session = session(conn); - boolean shuttingDown; - synchronized (monitor) { - sessions.add(session); - shuttingDown = shutdown != null; - } - log.fine(() -> "Added session %s".formatted(session)); - LifeCycle.start(session); - if (shuttingDown) session.shutdown(); - } - - @Override - public void onClosed(Connection conn) { - var session = session(conn); - boolean removed; - CompletableFuture<Void> shutdown; - synchronized (monitor) { - removed = sessions.remove(session); - shutdown = this.shutdown != null && sessions.size() == 0 && !this.shutdown.isDone() - ? this.shutdown : null; - } - log.fine(() -> "Removed session %s".formatted(session)); - if (removed) LifeCycle.stop(session); - if (shutdown != null) { - log.fine("Shutdown completed after last session removed"); - shutdown.complete(null); - } - } - - @Override - public void dump(Appendable out, String indent) throws IOException { - synchronized (monitor) { Dumpable.dumpObjects(out, indent, this, sessions); } - } - - @Override public CompletableFuture<Void> shutdown() { - CompletableFuture<Void> shutdown = null; - ISession[] sessionsToClose = null; - synchronized (monitor) { - if (this.shutdown == null) { - shutdown = (this.shutdown = new CompletableFuture<>()); - sessionsToClose = sessions.toArray(ISession[]::new); - } - } - if (sessionsToClose != null) { - log.fine("Shutdown initiated"); - if (sessionsToClose.length > 0) { - for (ISession session : sessionsToClose) { - session.shutdown(); - } - } else { - log.fine("Shutdown completed since no sessions"); - shutdown.complete(null); - } - } - return shutdown; - } - - @Override public boolean isShutdown() { synchronized (monitor) { return shutdown != null; } } - - private static ISession session(Connection conn) { return ((HTTP2Connection)conn).getSession(); } - } -} diff --git a/container-core/src/main/java/com/yahoo/jdisc/http/server/jetty/HealthCheckProxyHandler.java b/container-core/src/main/java/com/yahoo/jdisc/http/server/jetty/HealthCheckProxyHandler.java index 342d7ab9c4a..ac50cbbb518 100644 --- a/container-core/src/main/java/com/yahoo/jdisc/http/server/jetty/HealthCheckProxyHandler.java +++ b/container-core/src/main/java/com/yahoo/jdisc/http/server/jetty/HealthCheckProxyHandler.java @@ -4,24 +4,14 @@ package com.yahoo.jdisc.http.server.jetty; import com.yahoo.concurrent.DaemonThreadFactory; import com.yahoo.jdisc.http.ConnectorConfig; import com.yahoo.security.SslContextBuilder; -import com.yahoo.security.TrustAllX509TrustManager; import com.yahoo.security.tls.TransportSecurityOptions; import com.yahoo.security.tls.TransportSecurityUtils; -import jakarta.servlet.AsyncContext; -import jakarta.servlet.AsyncEvent; -import jakarta.servlet.AsyncListener; -import jakarta.servlet.ServletException; -import jakarta.servlet.ServletOutputStream; -import jakarta.servlet.WriteListener; -import jakarta.servlet.http.HttpServletRequest; -import jakarta.servlet.http.HttpServletResponse; +import com.yahoo.security.TrustAllX509TrustManager; import org.eclipse.jetty.client.HttpClient; import org.eclipse.jetty.client.ProxyProtocolClientConnectionFactory; import org.eclipse.jetty.client.api.ContentResponse; -import org.eclipse.jetty.client.http.HttpClientTransportOverHTTP; import org.eclipse.jetty.http.HttpField; import org.eclipse.jetty.http.HttpHeader; -import org.eclipse.jetty.io.ClientConnector; import org.eclipse.jetty.server.DetectorConnectionFactory; import org.eclipse.jetty.server.Request; import org.eclipse.jetty.server.SslConnectionFactory; @@ -29,6 +19,14 @@ import org.eclipse.jetty.server.handler.HandlerWrapper; import org.eclipse.jetty.util.ssl.SslContextFactory; import javax.net.ssl.SSLContext; +import javax.servlet.AsyncContext; +import javax.servlet.AsyncEvent; +import javax.servlet.AsyncListener; +import javax.servlet.ServletException; +import javax.servlet.ServletOutputStream; +import javax.servlet.WriteListener; +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; import java.io.IOException; import java.time.Duration; import java.util.HashMap; @@ -91,7 +89,7 @@ class HealthCheckProxyHandler extends HandlerWrapper { Optional.ofNullable(targetConnector.getConnectionFactory(SslConnectionFactory.class)) .or(() -> Optional.ofNullable(targetConnector.getConnectionFactory(DetectorConnectionFactory.class)) .map(detectorConnFactory -> detectorConnFactory.getBean(SslConnectionFactory.class))) - .map(SslConnectionFactory::getSslContextFactory) + .map(connFactory -> (SslContextFactory.Server) connFactory.getSslContextFactory()) .orElseThrow(() -> new IllegalArgumentException("Health check proxy can only target https port")); boolean proxyProtocol = targetConnector.connectorConfig().proxyProtocol().enabled(); return new ProxyTarget(targetPort, clientTimeout,handlerTimeout, cacheExpiry, sslContextFactory, proxyProtocol); @@ -271,14 +269,13 @@ class HealthCheckProxyHandler extends HandlerWrapper { synchronized (this) { if (client == null) { int timeoutMillis = (int) clientTimeout.toMillis(); - var clientSsl = new SslContextFactory.Client(); + SslContextFactory.Client clientSsl = new SslContextFactory.Client(); clientSsl.setHostnameVerifier((__, ___) -> true); clientSsl.setSslContext(getSslContext(serverSsl)); - var connector = new ClientConnector(); - connector.setSslContextFactory(clientSsl); - HttpClient client = new HttpClient(new HttpClientTransportOverHTTP(connector)); + HttpClient client = new HttpClient(clientSsl); client.setMaxConnectionsPerDestination(4); client.setConnectTimeout(timeoutMillis); + client.setStopTimeout(timeoutMillis); client.setIdleTimeout(timeoutMillis); client.setUserAgentField(new HttpField(HttpHeader.USER_AGENT, "health-check-proxy-client")); client.start(); diff --git a/container-core/src/main/java/com/yahoo/jdisc/http/server/jetty/HttpRequestDispatch.java b/container-core/src/main/java/com/yahoo/jdisc/http/server/jetty/HttpRequestDispatch.java index b4c933c1168..9292e2024df 100644 --- a/container-core/src/main/java/com/yahoo/jdisc/http/server/jetty/HttpRequestDispatch.java +++ b/container-core/src/main/java/com/yahoo/jdisc/http/server/jetty/HttpRequestDispatch.java @@ -12,11 +12,6 @@ import com.yahoo.jdisc.handler.RequestHandler; import com.yahoo.jdisc.http.ConnectorConfig; import com.yahoo.jdisc.http.HttpHeaders; import com.yahoo.jdisc.http.HttpRequest; -import jakarta.servlet.AsyncContext; -import jakarta.servlet.AsyncEvent; -import jakarta.servlet.AsyncListener; -import jakarta.servlet.http.HttpServletRequest; -import jakarta.servlet.http.HttpServletResponse; import org.eclipse.jetty.http2.ErrorCode; import org.eclipse.jetty.http2.server.HTTP2ServerConnection; import org.eclipse.jetty.io.Connection; @@ -25,6 +20,11 @@ import org.eclipse.jetty.server.HttpConnection; import org.eclipse.jetty.server.Request; import org.eclipse.jetty.util.Callback; +import javax.servlet.AsyncContext; +import javax.servlet.AsyncEvent; +import javax.servlet.AsyncListener; +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; import java.io.IOException; import java.nio.charset.StandardCharsets; import java.time.Instant; diff --git a/container-core/src/main/java/com/yahoo/jdisc/http/server/jetty/HttpRequestFactory.java b/container-core/src/main/java/com/yahoo/jdisc/http/server/jetty/HttpRequestFactory.java index d45a8789e4c..8a298fb3268 100644 --- a/container-core/src/main/java/com/yahoo/jdisc/http/server/jetty/HttpRequestFactory.java +++ b/container-core/src/main/java/com/yahoo/jdisc/http/server/jetty/HttpRequestFactory.java @@ -3,10 +3,10 @@ package com.yahoo.jdisc.http.server.jetty; import com.yahoo.jdisc.http.HttpRequest; import com.yahoo.jdisc.service.CurrentContainer; -import jakarta.servlet.http.HttpServletRequest; import org.eclipse.jetty.server.Request; import org.eclipse.jetty.util.Utf8Appendable; +import javax.servlet.http.HttpServletRequest; import java.net.InetSocketAddress; import java.net.URI; import java.security.cert.X509Certificate; @@ -94,6 +94,6 @@ class HttpRequestFactory { } private static X509Certificate[] getCertChain(HttpServletRequest servletRequest) { - return (X509Certificate[]) servletRequest.getAttribute(RequestUtils.SERVLET_REQUEST_X509CERT); + return (X509Certificate[]) servletRequest.getAttribute("javax.servlet.request.X509Certificate"); } } diff --git a/container-core/src/main/java/com/yahoo/jdisc/http/server/jetty/HttpResponseStatisticsCollector.java b/container-core/src/main/java/com/yahoo/jdisc/http/server/jetty/HttpResponseStatisticsCollector.java index 81789881b68..3fb81cb5352 100644 --- a/container-core/src/main/java/com/yahoo/jdisc/http/server/jetty/HttpResponseStatisticsCollector.java +++ b/container-core/src/main/java/com/yahoo/jdisc/http/server/jetty/HttpResponseStatisticsCollector.java @@ -4,11 +4,6 @@ package com.yahoo.jdisc.http.server.jetty; import com.yahoo.jdisc.Metric; import com.yahoo.jdisc.http.HttpRequest; import com.yahoo.jdisc.http.ServerConfig; -import jakarta.servlet.AsyncEvent; -import jakarta.servlet.AsyncListener; -import jakarta.servlet.ServletException; -import jakarta.servlet.http.HttpServletRequest; -import jakarta.servlet.http.HttpServletResponse; import org.eclipse.jetty.http.HttpHeader; import org.eclipse.jetty.http.HttpStatus; import org.eclipse.jetty.server.AsyncContextEvent; @@ -16,8 +11,14 @@ import org.eclipse.jetty.server.Handler; import org.eclipse.jetty.server.HttpChannelState; import org.eclipse.jetty.server.Request; import org.eclipse.jetty.server.handler.HandlerWrapper; +import org.eclipse.jetty.util.FutureCallback; import org.eclipse.jetty.util.component.Graceful; +import javax.servlet.AsyncEvent; +import javax.servlet.AsyncListener; +import javax.servlet.ServletException; +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; import java.io.IOException; import java.util.ArrayList; import java.util.Collection; @@ -26,10 +27,12 @@ import java.util.List; import java.util.Map; import java.util.Objects; import java.util.Set; -import java.util.concurrent.CompletableFuture; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; +import java.util.concurrent.Future; +import java.util.concurrent.TimeoutException; import java.util.concurrent.atomic.AtomicLong; +import java.util.concurrent.atomic.AtomicReference; import java.util.concurrent.atomic.LongAdder; import java.util.function.ObjLongConsumer; import java.util.stream.Collectors; @@ -46,7 +49,7 @@ class HttpResponseStatisticsCollector extends HandlerWrapper implements Graceful static final String requestTypeAttribute = "requestType"; - private final Shutdown shutdown; + private final AtomicReference<FutureCallback> shutdown = new AtomicReference<>(); private final List<String> monitoringHandlerPaths; private final List<String> searchHandlerPaths; private final Set<String> ignoredUserAgents; @@ -63,10 +66,6 @@ class HttpResponseStatisticsCollector extends HandlerWrapper implements Graceful this.monitoringHandlerPaths = monitoringHandlerPaths; this.searchHandlerPaths = searchHandlerPaths; this.ignoredUserAgents = Set.copyOf(ignoredUserAgents); - this.shutdown = new Shutdown(this) { - @Override public boolean isShutdownDone() { return inFlight.get() == 0; } - }; - } private final AsyncListener completionWatcher = new AsyncListener() { @@ -98,7 +97,7 @@ class HttpResponseStatisticsCollector extends HandlerWrapper implements Graceful try { Handler handler = getHandler(); - if (handler != null && !shutdown.isShutdown() && isStarted()) { + if (handler != null && shutdown.get() == null && isStarted()) { handler.handle(path, baseRequest, request, response); } else if ( ! baseRequest.isHandled()) { baseRequest.setHandled(true); @@ -130,9 +129,14 @@ class HttpResponseStatisticsCollector extends HandlerWrapper implements Graceful .increment()); } long live = inFlight.decrementAndGet(); - if (shutdown.isShutdown()) { - if (flushableResponse != null) flushableResponse.flushBuffer(); - if (live == 0) shutdown.check(); + FutureCallback shutdownCb = shutdown.get(); + if (shutdownCb != null) { + if (flushableResponse != null) { + flushableResponse.flushBuffer(); + } + if (live == 0) { + shutdownCb.succeeded(); + } } } @@ -158,19 +162,35 @@ class HttpResponseStatisticsCollector extends HandlerWrapper implements Graceful @Override protected void doStart() throws Exception { - shutdown.cancel(); + shutdown.set(null); super.doStart(); } @Override protected void doStop() throws Exception { - shutdown.cancel(); super.doStop(); + FutureCallback shutdownCb = shutdown.get(); + if ( ! shutdownCb.isDone()) { + shutdownCb.failed(new TimeoutException()); + } } - @Override public CompletableFuture<Void> shutdown() { return shutdown.shutdown(); } - @Override public boolean isShutdown() { return shutdown.isShutdown(); } + @Override + public Future<Void> shutdown() { + FutureCallback shutdownCb = new FutureCallback(false); + shutdown.compareAndSet(null, shutdownCb); + shutdownCb = shutdown.get(); + if (inFlight.get() == 0) { + shutdownCb.succeeded(); + } + return shutdownCb; + } + @Override + public boolean isShutdown() { + FutureCallback futureCallback = shutdown.get(); + return futureCallback != null && futureCallback.isDone(); + } static class Dimensions { final String protocol; diff --git a/container-core/src/main/java/com/yahoo/jdisc/http/server/jetty/JDiscHttpServlet.java b/container-core/src/main/java/com/yahoo/jdisc/http/server/jetty/JDiscHttpServlet.java index bd052f14867..4b4aff0a9bd 100644 --- a/container-core/src/main/java/com/yahoo/jdisc/http/server/jetty/JDiscHttpServlet.java +++ b/container-core/src/main/java/com/yahoo/jdisc/http/server/jetty/JDiscHttpServlet.java @@ -5,13 +5,13 @@ import com.yahoo.container.logging.AccessLogEntry; import com.yahoo.jdisc.Metric; import com.yahoo.jdisc.handler.OverloadException; import com.yahoo.jdisc.http.HttpRequest.Method; -import jakarta.servlet.ServletException; -import jakarta.servlet.annotation.WebServlet; -import jakarta.servlet.http.HttpServlet; -import jakarta.servlet.http.HttpServletRequest; -import jakarta.servlet.http.HttpServletResponse; import org.eclipse.jetty.server.Request; +import javax.servlet.ServletException; +import javax.servlet.annotation.WebServlet; +import javax.servlet.http.HttpServlet; +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; import java.io.IOException; import java.util.Enumeration; import java.util.Map; diff --git a/container-core/src/main/java/com/yahoo/jdisc/http/server/jetty/JDiscServerConnector.java b/container-core/src/main/java/com/yahoo/jdisc/http/server/jetty/JDiscServerConnector.java index b17877cee84..b3069a64821 100644 --- a/container-core/src/main/java/com/yahoo/jdisc/http/server/jetty/JDiscServerConnector.java +++ b/container-core/src/main/java/com/yahoo/jdisc/http/server/jetty/JDiscServerConnector.java @@ -3,13 +3,16 @@ package com.yahoo.jdisc.http.server.jetty; import com.yahoo.jdisc.Metric; import com.yahoo.jdisc.http.ConnectorConfig; -import jakarta.servlet.ServletRequest; -import jakarta.servlet.http.HttpServletRequest; +import org.eclipse.jetty.http.HttpCompliance; import org.eclipse.jetty.io.ConnectionStatistics; import org.eclipse.jetty.server.ConnectionFactory; import org.eclipse.jetty.server.Server; import org.eclipse.jetty.server.ServerConnector; +import javax.servlet.ServletRequest; +import javax.servlet.http.HttpServletRequest; +import java.net.Socket; +import java.net.SocketException; import java.util.HashMap; import java.util.Map; import java.util.Optional; @@ -23,6 +26,8 @@ class JDiscServerConnector extends ServerConnector { private final Metric.Context metricCtx; private final ConnectionStatistics statistics; private final ConnectorConfig config; + private final boolean tcpKeepAlive; + private final boolean tcpNoDelay; private final Metric metric; private final String connectorName; private final int listenPort; @@ -31,13 +36,14 @@ class JDiscServerConnector extends ServerConnector { ConnectionMetricAggregator connectionMetricAggregator, ConnectionFactory... factories) { super(server, factories); this.config = config; + this.tcpKeepAlive = config.tcpKeepAliveEnabled(); + this.tcpNoDelay = config.tcpNoDelay(); this.metric = metric; this.connectorName = config.name(); this.listenPort = config.listenPort(); this.metricCtx = metric.createContext(createConnectorDimensions(listenPort, connectorName, 0)); this.statistics = new ConnectionStatistics(); - setAcceptedTcpNoDelay(config.tcpNoDelay()); addBean(statistics); ConnectorConfig.Throttling throttlingConfig = config.throttling(); if (throttlingConfig.enabled()) { @@ -50,6 +56,17 @@ class JDiscServerConnector extends ServerConnector { setAcceptQueueSize(config.acceptQueueSize()); setReuseAddress(config.reuseAddress()); setIdleTimeout((long) (config.idleTimeout() * 1000)); + addBean(HttpCompliance.RFC7230); + } + + @Override + protected void configure(final Socket socket) { + super.configure(socket); + try { + socket.setKeepAlive(tcpKeepAlive); + socket.setTcpNoDelay(tcpNoDelay); + } catch (SocketException ignored) { + } } public ConnectionStatistics getStatistics() { diff --git a/container-core/src/main/java/com/yahoo/jdisc/http/server/jetty/JettyConnectionLogger.java b/container-core/src/main/java/com/yahoo/jdisc/http/server/jetty/JettyConnectionLogger.java index d9a97d621ae..2e2eb257b6a 100644 --- a/container-core/src/main/java/com/yahoo/jdisc/http/server/jetty/JettyConnectionLogger.java +++ b/container-core/src/main/java/com/yahoo/jdisc/http/server/jetty/JettyConnectionLogger.java @@ -30,7 +30,6 @@ import javax.net.ssl.SSLPeerUnverifiedException; import javax.net.ssl.SSLSession; import javax.net.ssl.StandardConstants; import java.net.InetSocketAddress; -import java.net.SocketAddress; import java.security.MessageDigest; import java.security.NoSuchAlgorithmException; import java.security.cert.CertificateEncodingException; @@ -114,7 +113,7 @@ class JettyConnectionLogger extends AbstractLifeCycle implements Connection.List info.setProxyProtocolVersion("v2"); } if (connection.getEndPoint() instanceof ProxyConnectionFactory.ProxyEndPoint) { - var remoteAddress = connection.getEndPoint().getRemoteSocketAddress(); + InetSocketAddress remoteAddress = connection.getEndPoint().getRemoteAddress(); info.setRemoteAddress(remoteAddress); } }); @@ -244,7 +243,7 @@ class JettyConnectionLogger extends AbstractLifeCycle implements Connection.List private long httpBytesSent = 0; private long requests = 0; private long responses = 0; - private SocketAddress remoteAddress; + private InetSocketAddress remoteAddress; private byte[] sslSessionId; private String sslProtocol; private String sslCipherSuite; @@ -291,7 +290,7 @@ class JettyConnectionLogger extends AbstractLifeCycle implements Connection.List synchronized ConnectionInfo incrementResponses() { ++this.responses; return this; } - synchronized ConnectionInfo setRemoteAddress(SocketAddress remoteAddress) { + synchronized ConnectionInfo setRemoteAddress(InetSocketAddress remoteAddress) { this.remoteAddress = remoteAddress; return this; } @@ -355,9 +354,9 @@ class JettyConnectionLogger extends AbstractLifeCycle implements Connection.List builder.withLocalAddress(localAddress.getHostString()) .withLocalPort(localAddress.getPort()); } - if (remoteAddress instanceof InetSocketAddress isa) { - builder.withRemoteAddress(isa.getHostString()) - .withRemotePort(isa.getPort()); + if (remoteAddress != null) { + builder.withRemoteAddress(remoteAddress.getHostString()) + .withRemotePort(remoteAddress.getPort()); } if (sslProtocol != null && sslCipherSuite != null && sslSessionId != null) { builder.withSslProtocol(sslProtocol) diff --git a/container-core/src/main/java/com/yahoo/jdisc/http/server/jetty/JettyHttpServer.java b/container-core/src/main/java/com/yahoo/jdisc/http/server/jetty/JettyHttpServer.java index 7b723b3a48e..775c903f5f8 100644 --- a/container-core/src/main/java/com/yahoo/jdisc/http/server/jetty/JettyHttpServer.java +++ b/container-core/src/main/java/com/yahoo/jdisc/http/server/jetty/JettyHttpServer.java @@ -27,6 +27,8 @@ import org.eclipse.jetty.server.handler.gzip.GzipHandler; import org.eclipse.jetty.server.handler.gzip.GzipHttpOutputInterceptor; import org.eclipse.jetty.servlet.ServletContextHandler; import org.eclipse.jetty.servlet.ServletHolder; +import org.eclipse.jetty.util.log.JavaUtilLog; +import org.eclipse.jetty.util.log.Log; import org.eclipse.jetty.util.thread.QueuedThreadPool; import javax.management.remote.JMXServiceURL; @@ -68,6 +70,8 @@ public class JettyHttpServer extends AbstractServerProvider { if (connectorFactories.allComponents().isEmpty()) throw new IllegalArgumentException("No connectors configured."); + initializeJettyLogging(); + server = new Server(); server.setStopTimeout((long)(serverConfig.stopTimeout() * 1000.0)); server.setRequestLog(new AccessLogRequestLog(requestLog, serverConfig.accessLog())); @@ -92,6 +96,15 @@ public class JettyHttpServer extends AbstractServerProvider { this.metricsReporter = new ServerMetricReporter(metric, server); } + private static void initializeJettyLogging() { + // Note: Jetty is logging stderr if no logger is explicitly configured + try { + Log.setLog(new JavaUtilLog()); + } catch (Exception e) { + throw new RuntimeException("Unable to initialize logging framework for Jetty"); + } + } + private static void setupJmx(Server server, ServerConfig serverConfig) { if (serverConfig.jmx().enabled()) { System.setProperty("java.rmi.server.hostname", "localhost"); @@ -139,7 +152,7 @@ public class JettyHttpServer extends AbstractServerProvider { } StatisticsHandler root = newGenericStatisticsHandler(); addChainToRoot(root, List.of( - newResponseStatisticsHandler(serverCfg), newGzipHandler(), perConnectorHandlers)); + newResponseStatisticsHandler(serverCfg), newGzipHandler(serverCfg), perConnectorHandlers)); return root; } @@ -240,18 +253,22 @@ public class JettyHttpServer extends AbstractServerProvider { return statisticsHandler; } - private static GzipHandler newGzipHandler() { return new GzipHandlerWithVaryHeaderFixed(); } + private static GzipHandler newGzipHandler(ServerConfig serverConfig) { + GzipHandler gzipHandler = new GzipHandlerWithVaryHeaderFixed(); + gzipHandler.setCompressionLevel(serverConfig.responseCompressionLevel()); + gzipHandler.setInflateBufferSize(8 * 1024); + gzipHandler.setIncludedMethods("GET", "POST", "PUT", "PATCH"); + return gzipHandler; + } /** A subclass which overrides Jetty's default behavior of including user-agent in the vary field */ private static class GzipHandlerWithVaryHeaderFixed extends GzipHandler { - GzipHandlerWithVaryHeaderFixed() { - setInflateBufferSize(8 * 1024); - setIncludedMethods("GET", "POST", "PUT", "PATCH"); + @Override + public HttpField getVaryField() { + return GzipHttpOutputInterceptor.VARY_ACCEPT_ENCODING; } - @Override public HttpField getVaryField() { return GzipHttpOutputInterceptor.VARY_ACCEPT_ENCODING; } - } } diff --git a/container-core/src/main/java/com/yahoo/jdisc/http/server/jetty/RequestUtils.java b/container-core/src/main/java/com/yahoo/jdisc/http/server/jetty/RequestUtils.java index da4de957739..1bc862bc787 100644 --- a/container-core/src/main/java/com/yahoo/jdisc/http/server/jetty/RequestUtils.java +++ b/container-core/src/main/java/com/yahoo/jdisc/http/server/jetty/RequestUtils.java @@ -1,12 +1,12 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.jdisc.http.server.jetty; -import jakarta.servlet.http.HttpServletRequest; import org.eclipse.jetty.http2.server.HTTP2ServerConnection; import org.eclipse.jetty.io.Connection; import org.eclipse.jetty.server.HttpConnection; import org.eclipse.jetty.server.Request; -import org.eclipse.jetty.server.SecureRequestCustomizer; + +import javax.servlet.http.HttpServletRequest; /** * @author bjorncs @@ -15,7 +15,7 @@ public class RequestUtils { public static final String JDISC_REQUEST_X509CERT = "jdisc.request.X509Certificate"; public static final String JDISC_REQUEST_CHAIN = "jdisc.request.chain"; public static final String JDISC_RESPONSE_CHAIN = "jdisc.response.chain"; - public static final String SERVLET_REQUEST_X509CERT = SecureRequestCustomizer.JAKARTA_SERVLET_REQUEST_X_509_CERTIFICATE; + public static final String SERVLET_REQUEST_X509CERT = "javax.servlet.request.X509Certificate"; // The local port as reported by servlet spec. This will be influenced by Host header and similar mechanisms. // The request URI uses the local listen port as the URI is used for handler routing/bindings. diff --git a/container-core/src/main/java/com/yahoo/jdisc/http/server/jetty/ServletOutputStreamWriter.java b/container-core/src/main/java/com/yahoo/jdisc/http/server/jetty/ServletOutputStreamWriter.java index d853282a5f5..4b66715fcf7 100644 --- a/container-core/src/main/java/com/yahoo/jdisc/http/server/jetty/ServletOutputStreamWriter.java +++ b/container-core/src/main/java/com/yahoo/jdisc/http/server/jetty/ServletOutputStreamWriter.java @@ -2,9 +2,9 @@ package com.yahoo.jdisc.http.server.jetty; import com.yahoo.jdisc.handler.CompletionHandler; -import jakarta.servlet.ServletOutputStream; -import jakarta.servlet.WriteListener; +import javax.servlet.ServletOutputStream; +import javax.servlet.WriteListener; import java.io.IOException; import java.nio.ByteBuffer; import java.util.ArrayDeque; diff --git a/container-core/src/main/java/com/yahoo/jdisc/http/server/jetty/ServletRequestReader.java b/container-core/src/main/java/com/yahoo/jdisc/http/server/jetty/ServletRequestReader.java index 2f2c48e0b48..3703878f595 100644 --- a/container-core/src/main/java/com/yahoo/jdisc/http/server/jetty/ServletRequestReader.java +++ b/container-core/src/main/java/com/yahoo/jdisc/http/server/jetty/ServletRequestReader.java @@ -3,10 +3,10 @@ package com.yahoo.jdisc.http.server.jetty; import com.yahoo.jdisc.handler.CompletionHandler; import com.yahoo.jdisc.handler.ContentChannel; -import jakarta.servlet.ReadListener; -import jakarta.servlet.ServletInputStream; -import jakarta.servlet.http.HttpServletRequest; +import javax.servlet.ReadListener; +import javax.servlet.ServletInputStream; +import javax.servlet.http.HttpServletRequest; import java.io.IOException; import java.nio.ByteBuffer; import java.util.Objects; diff --git a/container-core/src/main/java/com/yahoo/jdisc/http/server/jetty/ServletResponseController.java b/container-core/src/main/java/com/yahoo/jdisc/http/server/jetty/ServletResponseController.java index 6afb55f5b13..e90dde0e4eb 100644 --- a/container-core/src/main/java/com/yahoo/jdisc/http/server/jetty/ServletResponseController.java +++ b/container-core/src/main/java/com/yahoo/jdisc/http/server/jetty/ServletResponseController.java @@ -9,10 +9,10 @@ import com.yahoo.jdisc.handler.ResponseHandler; import com.yahoo.jdisc.http.HttpHeaders; import com.yahoo.jdisc.http.HttpResponse; import com.yahoo.jdisc.service.BindingSetNotFoundException; -import jakarta.servlet.http.HttpServletRequest; -import jakarta.servlet.http.HttpServletResponse; import org.eclipse.jetty.http.MimeTypes; +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; import java.io.IOException; import java.io.PrintWriter; import java.io.StringWriter; diff --git a/container-core/src/main/java/com/yahoo/jdisc/http/server/jetty/TlsClientAuthenticationEnforcer.java b/container-core/src/main/java/com/yahoo/jdisc/http/server/jetty/TlsClientAuthenticationEnforcer.java index 96f0cdebd62..b420aabc598 100644 --- a/container-core/src/main/java/com/yahoo/jdisc/http/server/jetty/TlsClientAuthenticationEnforcer.java +++ b/container-core/src/main/java/com/yahoo/jdisc/http/server/jetty/TlsClientAuthenticationEnforcer.java @@ -3,13 +3,13 @@ package com.yahoo.jdisc.http.server.jetty; import com.yahoo.jdisc.Response; import com.yahoo.jdisc.http.ConnectorConfig; -import jakarta.servlet.DispatcherType; -import jakarta.servlet.ServletException; -import jakarta.servlet.http.HttpServletRequest; -import jakarta.servlet.http.HttpServletResponse; import org.eclipse.jetty.server.Request; import org.eclipse.jetty.server.handler.HandlerWrapper; +import javax.servlet.DispatcherType; +import javax.servlet.ServletException; +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; import java.io.IOException; /** diff --git a/container-core/src/main/resources/configdefinitions/jdisc.http.jdisc.http.connector.def b/container-core/src/main/resources/configdefinitions/jdisc.http.jdisc.http.connector.def index bdcc3f9e40a..ecbc451ead1 100644 --- a/container-core/src/main/resources/configdefinitions/jdisc.http.jdisc.http.connector.def +++ b/container-core/src/main/resources/configdefinitions/jdisc.http.jdisc.http.connector.def @@ -28,8 +28,7 @@ reuseAddress bool default=true # The maximum idle time for a connection, which roughly translates to the Socket.setSoTimeout(int). idleTimeout double default=180.0 -# TODO Vespa 9 Remove -# Has no effect since Jetty 11 upgrade +# Whether or not to have socket keep alive turned on. tcpKeepAliveEnabled bool default=false # Enable/disable TCP_NODELAY (disable/enable Nagle's algorithm). diff --git a/container-core/src/main/resources/configdefinitions/jdisc.http.jdisc.http.server.def b/container-core/src/main/resources/configdefinitions/jdisc.http.jdisc.http.server.def index c15cb6b2cc4..f34fd523207 100644 --- a/container-core/src/main/resources/configdefinitions/jdisc.http.jdisc.http.server.def +++ b/container-core/src/main/resources/configdefinitions/jdisc.http.jdisc.http.server.def @@ -4,8 +4,7 @@ namespace=jdisc.http # Whether to enable developer mode, where stack traces etc are visible in response bodies. developerMode bool default=false -# TODO Vespa 9 Remove -# Has no effect since Jetty 11 upgrade +# The gzip compression level to use, if compression is enabled in a request. responseCompressionLevel int default=6 # Whether the request body of POSTed forms should be removed (form parameters are available as request parameters). diff --git a/container-core/src/test/java/com/yahoo/jdisc/http/server/jetty/ConnectorFactoryTest.java b/container-core/src/test/java/com/yahoo/jdisc/http/server/jetty/ConnectorFactoryTest.java index ce205b1a893..1ff2783cc53 100644 --- a/container-core/src/test/java/com/yahoo/jdisc/http/server/jetty/ConnectorFactoryTest.java +++ b/container-core/src/test/java/com/yahoo/jdisc/http/server/jetty/ConnectorFactoryTest.java @@ -5,8 +5,6 @@ import com.yahoo.jdisc.Metric; import com.yahoo.jdisc.http.ConnectorConfig; import com.yahoo.jdisc.http.ServerConfig; import com.yahoo.jdisc.http.ssl.impl.ConfiguredSslContextFactoryProvider; -import jakarta.servlet.http.HttpServletRequest; -import jakarta.servlet.http.HttpServletResponse; import org.eclipse.jetty.server.Request; import org.eclipse.jetty.server.Server; import org.eclipse.jetty.server.handler.AbstractHandler; @@ -14,6 +12,8 @@ import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; import java.io.IOException; import java.util.Map; diff --git a/container-core/src/test/java/com/yahoo/jdisc/http/server/jetty/ErrorResponseContentCreatorTest.java b/container-core/src/test/java/com/yahoo/jdisc/http/server/jetty/ErrorResponseContentCreatorTest.java index fdb9f2226de..8b18c8cf09d 100644 --- a/container-core/src/test/java/com/yahoo/jdisc/http/server/jetty/ErrorResponseContentCreatorTest.java +++ b/container-core/src/test/java/com/yahoo/jdisc/http/server/jetty/ErrorResponseContentCreatorTest.java @@ -2,9 +2,9 @@ package com.yahoo.jdisc.http.server.jetty; -import jakarta.servlet.http.HttpServletResponse; import org.junit.jupiter.api.Test; +import javax.servlet.http.HttpServletResponse; import java.nio.charset.StandardCharsets; import static org.junit.jupiter.api.Assertions.assertEquals; diff --git a/container-core/src/test/java/com/yahoo/jdisc/http/server/jetty/HttpRequestFactoryTest.java b/container-core/src/test/java/com/yahoo/jdisc/http/server/jetty/HttpRequestFactoryTest.java index e4b82db5b9f..a23a3505bcb 100644 --- a/container-core/src/test/java/com/yahoo/jdisc/http/server/jetty/HttpRequestFactoryTest.java +++ b/container-core/src/test/java/com/yahoo/jdisc/http/server/jetty/HttpRequestFactoryTest.java @@ -8,17 +8,15 @@ import com.yahoo.jdisc.Response; import com.yahoo.jdisc.handler.RequestHandler; import com.yahoo.jdisc.http.HttpRequest; import com.yahoo.jdisc.service.CurrentContainer; -import jakarta.servlet.http.HttpServletRequest; import org.junit.jupiter.api.Test; +import javax.servlet.http.HttpServletRequest; import java.net.URI; import static org.hamcrest.CoreMatchers.equalTo; import static org.hamcrest.CoreMatchers.is; import static org.hamcrest.MatcherAssert.assertThat; -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertNull; -import static org.junit.jupiter.api.Assertions.fail; +import static org.junit.jupiter.api.Assertions.*; /** * @author Steinar Knutsen diff --git a/container-core/src/test/java/com/yahoo/jdisc/http/server/jetty/HttpResponseStatisticsCollectorTest.java b/container-core/src/test/java/com/yahoo/jdisc/http/server/jetty/HttpResponseStatisticsCollectorTest.java index 502702ccf35..165659389ec 100644 --- a/container-core/src/test/java/com/yahoo/jdisc/http/server/jetty/HttpResponseStatisticsCollectorTest.java +++ b/container-core/src/test/java/com/yahoo/jdisc/http/server/jetty/HttpResponseStatisticsCollectorTest.java @@ -2,9 +2,6 @@ package com.yahoo.jdisc.http.server.jetty; import com.yahoo.jdisc.http.server.jetty.HttpResponseStatisticsCollector.StatisticsEntry; -import jakarta.servlet.ServletException; -import jakarta.servlet.http.HttpServletRequest; -import jakarta.servlet.http.HttpServletResponse; import org.eclipse.jetty.http.HttpFields; import org.eclipse.jetty.http.HttpURI; import org.eclipse.jetty.http.HttpVersion; @@ -13,7 +10,6 @@ import org.eclipse.jetty.http.MetaData.Response; import org.eclipse.jetty.server.AbstractConnector; import org.eclipse.jetty.server.Connector; import org.eclipse.jetty.server.HttpChannel; -import org.eclipse.jetty.server.HttpChannelOverHttp; import org.eclipse.jetty.server.HttpConfiguration; import org.eclipse.jetty.server.HttpTransport; import org.eclipse.jetty.server.Request; @@ -23,6 +19,9 @@ import org.eclipse.jetty.util.Callback; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; +import javax.servlet.ServletException; +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; import java.io.IOException; import java.nio.ByteBuffer; import java.util.List; @@ -165,8 +164,8 @@ public class HttpResponseStatisticsCollectorTest { } private Request testRequest(String scheme, int responseCode, String httpMethod, String path, com.yahoo.jdisc.Request.RequestType explicitRequestType) { - HttpChannel channel = new HttpChannelOverHttp(null, connector, new HttpConfiguration(), null, new DummyTransport()); - MetaData.Request metaData = new MetaData.Request(httpMethod, HttpURI.build(scheme + "://" + path), HttpVersion.HTTP_1_1, HttpFields.build()); + HttpChannel channel = new HttpChannel(connector, new HttpConfiguration(), null, new DummyTransport()); + MetaData.Request metaData = new MetaData.Request(httpMethod, new HttpURI(scheme + "://" + path), HttpVersion.HTTP_1_1, new HttpFields()); Request req = channel.getRequest(); if (explicitRequestType != null) req.setAttribute("requestType", explicitRequestType); @@ -193,7 +192,7 @@ public class HttpResponseStatisticsCollectorTest { private final class DummyTransport implements HttpTransport { @Override - public void send(MetaData.Request request, Response response, ByteBuffer byteBuffer, boolean b, Callback callback) { + public void send(Response info, boolean head, ByteBuffer content, boolean lastContent, Callback callback) { callback.succeeded(); } @@ -203,6 +202,11 @@ public class HttpResponseStatisticsCollectorTest { } @Override + public boolean isOptimizedForDirectBuffers() { + return false; + } + + @Override public void push(MetaData.Request request) { } diff --git a/container-core/src/test/java/com/yahoo/jdisc/http/server/jetty/HttpServerConformanceTest.java b/container-core/src/test/java/com/yahoo/jdisc/http/server/jetty/HttpServerConformanceTest.java index ae1a6494acd..7cce9f2a9ff 100644 --- a/container-core/src/test/java/com/yahoo/jdisc/http/server/jetty/HttpServerConformanceTest.java +++ b/container-core/src/test/java/com/yahoo/jdisc/http/server/jetty/HttpServerConformanceTest.java @@ -807,7 +807,6 @@ public class HttpServerConformanceTest extends ServerProviderConformanceTest { post.setProtocolVersion(HttpVersion.HTTP_1_1); request = post; } - request.addHeader("Connection", "close"); return executorService.submit(() -> httpClient.execute(request)); } diff --git a/container-core/src/test/java/com/yahoo/jdisc/http/server/jetty/HttpServerTest.java b/container-core/src/test/java/com/yahoo/jdisc/http/server/jetty/HttpServerTest.java index 39b6dcdc6d5..318067ac634 100644 --- a/container-core/src/test/java/com/yahoo/jdisc/http/server/jetty/HttpServerTest.java +++ b/container-core/src/test/java/com/yahoo/jdisc/http/server/jetty/HttpServerTest.java @@ -186,10 +186,9 @@ public class HttpServerTest { @Test void requireThatServerCanEchoCompressed() throws Exception { final JettyTestDriver driver = JettyTestDriver.newInstance(new EchoRequestHandler()); - try (SimpleHttpClient client = driver.newClient(true)) { - client.get("/status.html") - .expectStatusCode(is(OK)); - } + SimpleHttpClient client = driver.newClient(true); + client.get("/status.html") + .expectStatusCode(is(OK)); assertTrue(driver.close()); } @@ -533,9 +532,9 @@ public class HttpServerTest { .withTrustStore(certificateFile) .build(); - try (var c = new SimpleHttpClient(trustStoreOnlyCtx, driver.server().getListenPort(), false)) { - c.get("/dummy.html").expectStatusCode(is(UNAUTHORIZED)); - } + new SimpleHttpClient(trustStoreOnlyCtx, driver.server().getListenPort(), false) + .get("/dummy.html") + .expectStatusCode(is(UNAUTHORIZED)); assertTrue(driver.close()); } @@ -551,9 +550,9 @@ public class HttpServerTest { .withTrustStore(certificateFile) .build(); - try (var c = new SimpleHttpClient(trustStoreOnlyCtx, driver.server().getListenPort(), false)) { - c.get("/status.html").expectStatusCode(is(OK)); - } + new SimpleHttpClient(trustStoreOnlyCtx, driver.server().getListenPort(), false) + .get("/status.html") + .expectStatusCode(is(OK)); assertTrue(driver.close()); } diff --git a/container-core/src/test/java/com/yahoo/jdisc/http/server/jetty/ProxyProtocolTest.java b/container-core/src/test/java/com/yahoo/jdisc/http/server/jetty/ProxyProtocolTest.java index 6cd6f05933a..d4d6dcee957 100644 --- a/container-core/src/test/java/com/yahoo/jdisc/http/server/jetty/ProxyProtocolTest.java +++ b/container-core/src/test/java/com/yahoo/jdisc/http/server/jetty/ProxyProtocolTest.java @@ -12,8 +12,6 @@ import org.apache.hc.client5.http.ssl.NoopHostnameVerifier; import org.assertj.core.api.Assertions; import org.eclipse.jetty.client.HttpClient; import org.eclipse.jetty.client.api.ContentResponse; -import org.eclipse.jetty.client.http.HttpClientTransportOverHTTP; -import org.eclipse.jetty.io.ClientConnector; import org.eclipse.jetty.util.ssl.SslContextFactory; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.BeforeEach; @@ -187,15 +185,14 @@ class ProxyProtocolTest { // Using Jetty's http client as Apache httpclient does not support the proxy-protocol v1/v2. private static HttpClient createJettyHttpClient(Path certificateFile) throws Exception { - var ssl = new SslContextFactory.Client(); - ssl.setHostnameVerifier(NoopHostnameVerifier.INSTANCE); - ssl.setSslContext(new SslContextBuilder().withTrustStore(certificateFile).build()); - var connector = new ClientConnector(); - connector.setSslContextFactory(ssl); - HttpClient client = new HttpClient(new HttpClientTransportOverHTTP(connector)); - int timeout = 60 * 1000; - client.setConnectTimeout(timeout); - client.setIdleTimeout(timeout); + SslContextFactory.Client clientSslCtxFactory = new SslContextFactory.Client(); + clientSslCtxFactory.setHostnameVerifier(NoopHostnameVerifier.INSTANCE); + clientSslCtxFactory.setSslContext(new SslContextBuilder().withTrustStore(certificateFile).build()); + + HttpClient client = new HttpClient(clientSslCtxFactory); + client.setConnectTimeout(60*1000); + client.setStopTimeout(60*1000); + client.setIdleTimeout(60*1000); client.start(); return client; } diff --git a/container-dependencies-enforcer/pom.xml b/container-dependencies-enforcer/pom.xml index a88b7e1d577..8c420328fe2 100644 --- a/container-dependencies-enforcer/pom.xml +++ b/container-dependencies-enforcer/pom.xml @@ -60,35 +60,35 @@ <rules> <enforceDependencies implementation="com.yahoo.vespa.maven.plugin.enforcer.EnforceDependencies"> <allowed> - <include>*:*:*:jar:test</include> - <include>com.yahoo.vespa</include> - <include>aopalliance:aopalliance:[${aopalliance.version}]:jar:provided</include> - <include>com.fasterxml.jackson.core:jackson-annotations:[${jackson2.version}]:jar:provided</include> - <include>com.fasterxml.jackson.core:jackson-core:[${jackson2.version}]:jar:provided</include> - <include>com.fasterxml.jackson.core:jackson-databind:[${jackson-databind.version}]:jar:provided</include> - <include>com.fasterxml.jackson.datatype:jackson-datatype-jdk8:[${jackson2.version}]:jar:provided</include> - <include>com.fasterxml.jackson.datatype:jackson-datatype-jsr310:[${jackson2.version}]:jar:provided</include> + <include>*:*:*:test</include> + <include>com.yahoo.vespa:*:*:*</include> + <include>aopalliance:aopalliance:${aopalliance.version}:provided</include> + <include>com.fasterxml.jackson.core:jackson-annotations:${jackson2.version}:provided</include> + <include>com.fasterxml.jackson.core:jackson-core:${jackson2.version}:provided</include> + <include>com.fasterxml.jackson.core:jackson-databind:${jackson-databind.version}:provided</include> + <include>com.fasterxml.jackson.datatype:jackson-datatype-jdk8:${jackson2.version}:provided</include> + <include>com.fasterxml.jackson.datatype:jackson-datatype-jsr310:${jackson2.version}:provided</include> <!-- Guava with its internal dependencies --> - <include>com.google.guava:guava:[${guava.version}]:jar:provided</include> - <include>com.google.errorprone:error_prone_annotations:[2.2.0]:jar:provided</include> - <include>com.google.guava:failureaccess:[1.0.1]:jar:provided</include> - <include>com.google.j2objc:j2objc-annotations:[1.1]:jar:provided</include> + <include>com.google.guava:guava:${guava.version}:provided</include> + <include>com.google.errorprone:error_prone_annotations:2.2.0:provided</include> + <include>com.google.guava:failureaccess:1.0.1:provided</include> + <include>com.google.j2objc:j2objc-annotations:1.1:provided</include> - <include>com.google.inject:guice:[${guice.version}]:jar:provided:no_aop</include> - <include>com.sun.activation:javax.activation:[1.2.0]:jar:provided</include> - <include>com.sun.xml.bind:jaxb-core:[${jaxb.version}]:jar:provided</include> - <include>com.sun.xml.bind:jaxb-impl:[${jaxb.version}]:jar:provided</include> - <include>commons-logging:commons-logging:[1.2]:jar:provided</include> - <include>javax.inject:javax.inject:[${javax.inject.version}]:jar:provided</include> - <include>javax.servlet:javax.servlet-api:[${javax.servlet-api.version}]:jar:provided</include> - <include>javax.ws.rs:javax.ws.rs-api:[${javax.ws.rs-api.version}]:jar:provided</include> - <include>javax.xml.bind:jaxb-api:[${jaxb.version}]:jar:provided</include> - <include>org.slf4j:jcl-over-slf4j:[${slf4j.version}]:jar:provided</include> - <include>org.slf4j:log4j-over-slf4j:[${slf4j.version}]:jar:provided</include> - <include>org.slf4j:slf4j-api:[${slf4j.version}]:jar:provided</include> - <include>org.slf4j:slf4j-jdk14:[${slf4j.version}]:jar:provided</include> - <include>xml-apis:xml-apis:[${xml-apis.version}]:jar:provided</include> + <include>com.google.inject:guice:jar:no_aop:${guice.version}:provided</include> + <include>com.sun.activation:javax.activation:1.2.0:provided</include> + <include>com.sun.xml.bind:jaxb-core:${jaxb.version}:provided</include> + <include>com.sun.xml.bind:jaxb-impl:${jaxb.version}:provided</include> + <include>commons-logging:commons-logging:1.2:provided</include> + <include>javax.inject:javax.inject:${javax.inject.version}:provided</include> + <include>javax.servlet:javax.servlet-api:${javax.servlet-api.version}:provided</include> + <include>javax.ws.rs:javax.ws.rs-api:${javax.ws.rs-api.version}:provided</include> + <include>javax.xml.bind:jaxb-api:${jaxb.version}:provided</include> + <include>org.slf4j:jcl-over-slf4j:${slf4j.version}:provided</include> + <include>org.slf4j:log4j-over-slf4j:${slf4j.version}:provided</include> + <include>org.slf4j:slf4j-api:${slf4j.version}:provided</include> + <include>org.slf4j:slf4j-jdk14:${slf4j.version}:provided</include> + <include>xml-apis:xml-apis:${xml-apis.version}:provided</include> </allowed> </enforceDependencies> </rules> diff --git a/container-dev/pom.xml b/container-dev/pom.xml index 9bbb5591fbf..0c88531a248 100644 --- a/container-dev/pom.xml +++ b/container-dev/pom.xml @@ -98,8 +98,8 @@ <!-- START JETTY embedded jars --> <exclusion> - <groupId>org.eclipse.jetty.http2</groupId> - <artifactId>http2-common</artifactId> + <groupId>org.eclipse.jetty.alpn</groupId> + <artifactId>alpn-api</artifactId> </exclusion> <exclusion> <groupId>org.eclipse.jetty.http2</groupId> @@ -111,19 +111,11 @@ </exclusion> <exclusion> <groupId>org.eclipse.jetty</groupId> - <artifactId>jetty-alpn-server</artifactId> - </exclusion> - <exclusion> - <groupId>org.eclipse.jetty</groupId> <artifactId>jetty-client</artifactId> </exclusion> <exclusion> <groupId>org.eclipse.jetty</groupId> - <artifactId>jetty-http</artifactId> - </exclusion> - <exclusion> - <groupId>org.eclipse.jetty</groupId> - <artifactId>jetty-io</artifactId> + <artifactId>jetty-continuation</artifactId> </exclusion> <exclusion> <groupId>org.eclipse.jetty</groupId> @@ -139,11 +131,7 @@ </exclusion> <exclusion> <groupId>org.eclipse.jetty</groupId> - <artifactId>jetty-util</artifactId> - </exclusion> - <exclusion> - <groupId>org.eclipse.jetty.toolchain</groupId> - <artifactId>jetty-jakarta-servlet-api</artifactId> + <artifactId>jetty-servlets</artifactId> </exclusion> <!-- END JETTY embedded jars --> </exclusions> diff --git a/container-test/pom.xml b/container-test/pom.xml index e0ff7f62e93..32a64a98b9e 100644 --- a/container-test/pom.xml +++ b/container-test/pom.xml @@ -113,8 +113,8 @@ <!-- START JETTY embedded jars --> <dependency> - <groupId>org.eclipse.jetty.http2</groupId> - <artifactId>http2-common</artifactId> + <groupId>org.eclipse.jetty.alpn</groupId> + <artifactId>alpn-api</artifactId> </dependency> <dependency> <groupId>org.eclipse.jetty.http2</groupId> @@ -126,19 +126,11 @@ </dependency> <dependency> <groupId>org.eclipse.jetty</groupId> - <artifactId>jetty-alpn-server</artifactId> - </dependency> - <dependency> - <groupId>org.eclipse.jetty</groupId> <artifactId>jetty-client</artifactId> </dependency> <dependency> <groupId>org.eclipse.jetty</groupId> - <artifactId>jetty-http</artifactId> - </dependency> - <dependency> - <groupId>org.eclipse.jetty</groupId> - <artifactId>jetty-io</artifactId> + <artifactId>jetty-continuation</artifactId> </dependency> <dependency> <groupId>org.eclipse.jetty</groupId> @@ -154,11 +146,7 @@ </dependency> <dependency> <groupId>org.eclipse.jetty</groupId> - <artifactId>jetty-util</artifactId> - </dependency> - <dependency> - <groupId>org.eclipse.jetty.toolchain</groupId> - <artifactId>jetty-jakarta-servlet-api</artifactId> + <artifactId>jetty-servlets</artifactId> </dependency> <!-- END JETTY embedded jars --> </dependencies> diff --git a/flags/src/main/java/com/yahoo/vespa/flags/Flags.java b/flags/src/main/java/com/yahoo/vespa/flags/Flags.java index 300a3548a94..8626f2b6291 100644 --- a/flags/src/main/java/com/yahoo/vespa/flags/Flags.java +++ b/flags/src/main/java/com/yahoo/vespa/flags/Flags.java @@ -412,7 +412,7 @@ public class Flags { "Takes effect immediately"); public static final UnboundBooleanFlag REPORT_CORES_VIA_CFG = defineFeatureFlag( - "report-cores-via-cfg", false, + "report-cores-via-cfg", true, List.of("hakonhall"), "2022-11-01", "2022-12-01", "If true, report core dumps to the config server instead of directly to the panic app.", "Takes effect on the next tick.", diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/coredump/CoredumpHandler.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/coredump/CoredumpHandler.java index b54eae0b276..3a61f8b2619 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/coredump/CoredumpHandler.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/coredump/CoredumpHandler.java @@ -20,7 +20,10 @@ import com.yahoo.vespa.hosted.node.admin.container.metrics.Metrics; import com.yahoo.vespa.hosted.node.admin.maintenance.sync.ZstdCompressingInputStream; import com.yahoo.vespa.hosted.node.admin.nodeadmin.ConvergenceException; import com.yahoo.vespa.hosted.node.admin.nodeagent.NodeAgentContext; +import com.yahoo.vespa.hosted.node.admin.task.util.file.FileDeleter; import com.yahoo.vespa.hosted.node.admin.task.util.file.FileFinder; +import com.yahoo.vespa.hosted.node.admin.task.util.file.FileMover; +import com.yahoo.vespa.hosted.node.admin.task.util.file.MakeDirectory; import com.yahoo.vespa.hosted.node.admin.task.util.file.UnixPath; import com.yahoo.vespa.hosted.node.admin.task.util.fs.ContainerPath; @@ -39,6 +42,7 @@ import java.util.List; import java.util.Map; import java.util.Optional; import java.util.UUID; +import java.util.function.Predicate; import java.util.function.Supplier; import java.util.logging.Logger; import java.util.regex.Pattern; @@ -126,7 +130,7 @@ public class CoredumpHandler { } // Check if we have already started to process a core dump or we can enqueue a new core one - getCoredumpToProcess(containerCrashPath, containerProcessingPath) + getCoredumpToProcess(context, containerCrashPath, containerProcessingPath) .ifPresent(path -> { if (reportCoresViaCfgFlag.with(FetchVector.Dimension.NODE_TYPE, context.nodeType().name()).value()) { processAndReportSingleCoreDump2(context, path, dockerImage); @@ -137,12 +141,12 @@ public class CoredumpHandler { } /** @return path to directory inside processing directory that contains a core dump file to process */ - Optional<ContainerPath> getCoredumpToProcess(ContainerPath containerCrashPath, ContainerPath containerProcessingPath) { + Optional<ContainerPath> getCoredumpToProcess(NodeAgentContext context, ContainerPath containerCrashPath, ContainerPath containerProcessingPath) { return FileFinder.directories(containerProcessingPath).stream() .map(FileFinder.FileAttributes::path) .findAny() .map(ContainerPath.class::cast) - .or(() -> enqueueCoredump(containerCrashPath, containerProcessingPath)); + .or(() -> enqueueCoredump(context, containerCrashPath, containerProcessingPath)); } /** @@ -154,9 +158,19 @@ public class CoredumpHandler { * * @return path to directory inside processing directory which contains the enqueued core dump file */ - Optional<ContainerPath> enqueueCoredump(ContainerPath containerCrashPath, ContainerPath containerProcessingPath) { + Optional<ContainerPath> enqueueCoredump(NodeAgentContext context, ContainerPath containerCrashPath, ContainerPath containerProcessingPath) { + Predicate<String> isCoreDump = filename -> !HS_ERR_PATTERN.matcher(filename).matches(); + List<Path> toProcess = FileFinder.files(containerCrashPath) - .match(this::isReadyForProcessing) + .match(attributes -> { + if (isReadyForProcessing(attributes)) { + return true; + } else { + if (isCoreDump.test(attributes.filename())) + context.log(logger, attributes.path() + " is still being written"); + return false; + } + }) .maxDepth(1) .stream() .sorted(Comparator.comparing(FileFinder.FileAttributes::lastModifiedTime)) @@ -164,19 +178,20 @@ public class CoredumpHandler { .toList(); int coredumpIndex = IntStream.range(0, toProcess.size()) - .filter(i -> !HS_ERR_PATTERN.matcher(toProcess.get(i).getFileName().toString()).matches()) + .filter(i -> isCoreDump.test(toProcess.get(i).getFileName().toString())) .findFirst() .orElse(-1); // Either there are no files in crash directory, or all the files are hs_err files. if (coredumpIndex == -1) return Optional.empty(); - ContainerPath enqueuedDir = (ContainerPath) uncheck(() -> Files.createDirectories(containerProcessingPath.resolve(coredumpIdSupplier.get()))); + ContainerPath enqueuedDir = containerProcessingPath.resolve(coredumpIdSupplier.get()); + new MakeDirectory(enqueuedDir).createParents().converge(context); IntStream.range(0, coredumpIndex + 1) .forEach(i -> { Path path = toProcess.get(i); String prefix = i == coredumpIndex ? COREDUMP_FILENAME_PREFIX : ""; - uncheck(() -> Files.move(path, enqueuedDir.resolve(prefix + path.getFileName()))); + new FileMover(path, enqueuedDir.resolve(prefix + path.getFileName())).converge(context); }); return Optional.of(enqueuedDir); } @@ -258,12 +273,13 @@ public class CoredumpHandler { } catch (IOException e) { throw new UncheckedIOException(e); } - uncheck(() -> Files.delete(coreFile)); + new FileDeleter(coreFile).converge(context); Path newCoredumpDirectory = doneCoredumpsPath.resolve(context.containerName().asString()); - uncheck(() -> Files.createDirectories(newCoredumpDirectory)); + new MakeDirectory(newCoredumpDirectory).createParents().converge(context); // Files.move() does not support moving non-empty directories across providers, move using host paths - uncheck(() -> Files.move(coredumpDirectory.pathOnHost(), newCoredumpDirectory.resolve(coredumpDirectory.getFileName().toString()))); + new FileMover(coredumpDirectory.pathOnHost(), newCoredumpDirectory.resolve(coredumpDirectory.getFileName().toString())) + .converge(context); } ContainerPath findCoredumpFileInProcessingDirectory(ContainerPath coredumpProccessingDirectory) { @@ -348,8 +364,8 @@ public class CoredumpHandler { String coreDumpId = coreDumpDirectory.getFileName().toString(); cores.report(context.hostname(), coreDumpId, metadata); + context.log(logger, "Core dump reported: " + coreDumpId); finishProcessing(context, coreDumpDirectory, sharedCoreKey); - context.log(logger, "Successfully reported core dump " + coreDumpId); } private CoreDumpMetadata gatherMetadata(NodeAgentContext context, ContainerPath coreDumpDirectory) { diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/FileDeleter.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/FileDeleter.java index 6f3f0c06344..92dc34d5e8b 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/FileDeleter.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/FileDeleter.java @@ -26,7 +26,7 @@ public class FileDeleter { public boolean converge(TaskContext context) { boolean deleted = uncheck(() -> Files.deleteIfExists(path)); if (deleted) { - context.recordSystemModification(logger, "Deleted file or directory " + path); + context.recordSystemModification(logger, "Deleted " + path); } return deleted; diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/FileMover.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/FileMover.java new file mode 100644 index 00000000000..a5ba78e524e --- /dev/null +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/FileMover.java @@ -0,0 +1,55 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.node.admin.task.util.file; + +import com.yahoo.vespa.hosted.node.admin.component.TaskContext; + +import java.nio.file.CopyOption; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.StandardCopyOption; +import java.util.HashSet; +import java.util.Set; +import java.util.logging.Logger; + +import static com.yahoo.yolean.Exceptions.uncheck; + +/** + * Utility for idempotent move of (any type of) file. + * + * @author hakonhall + */ +public class FileMover { + private static final Logger logger = Logger.getLogger(FileMover.class.getName()); + + private final Path source; + private final Path destination; + private final Set<CopyOption> moveOptions = new HashSet<>(); + + public FileMover(Path source, Path destination) { + this.source = source; + this.destination = destination; + } + + public FileMover replaceExisting() { + moveOptions.add(StandardCopyOption.REPLACE_EXISTING); + return this; + } + + public FileMover atomic() { + moveOptions.add(StandardCopyOption.ATOMIC_MOVE); + return this; + } + + /** + * Move file. + * + * @return false if the source doesn't exist while the destination do. + * @see Files#move(Path, Path, CopyOption...) Files.move() + */ + public boolean converge(TaskContext context) { + if (!Files.exists(source) && Files.exists(destination)) return false; + uncheck(() -> Files.move(source, destination, moveOptions.toArray(CopyOption[]::new))); + context.recordSystemModification(logger, "Moved " + source + " to " + destination); + return true; + } +} diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/coredump/CoredumpHandlerTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/coredump/CoredumpHandlerTest.java index 1d53f0974ab..235007d7499 100644 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/coredump/CoredumpHandlerTest.java +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/coredump/CoredumpHandlerTest.java @@ -1,6 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.node.admin.maintenance.coredump; +import com.yahoo.security.KeyId; import com.yahoo.security.SealedSharedKey; import com.yahoo.security.SecretSharedKey; import com.yahoo.test.ManualClock; @@ -77,7 +78,7 @@ public class CoredumpHandlerTest { createFileAged(crashPath.resolve("bash.core.431"), Duration.ZERO); assertFolderContents(crashPath, "bash.core.431"); - Optional<ContainerPath> enqueuedPath = coredumpHandler.enqueueCoredump(crashPath, processingDir); + Optional<ContainerPath> enqueuedPath = coredumpHandler.enqueueCoredump(context, crashPath, processingDir); assertEquals(Optional.empty(), enqueuedPath); // bash.core.431 finished writing... and 2 more have since been written @@ -86,7 +87,7 @@ public class CoredumpHandlerTest { createFileAged(crashPath.resolve("vespa-slobrok.core.673"), Duration.ofMinutes(5)); when(coredumpIdSupplier.get()).thenReturn("id-123").thenReturn("id-321"); - enqueuedPath = coredumpHandler.enqueueCoredump(crashPath, processingDir); + enqueuedPath = coredumpHandler.enqueueCoredump(context, crashPath, processingDir); assertEquals(Optional.of(processingDir.resolve("id-123")), enqueuedPath); assertFolderContents(crashPath, "bash.core.431", "vespa-slobrok.core.673"); assertFolderContents(processingDir, "id-123"); @@ -94,7 +95,7 @@ public class CoredumpHandlerTest { verify(coredumpIdSupplier, times(1)).get(); // Enqueue another - enqueuedPath = coredumpHandler.enqueueCoredump(crashPath, processingDir); + enqueuedPath = coredumpHandler.enqueueCoredump(context, crashPath, processingDir); assertEquals(Optional.of(processingDir.resolve("id-321")), enqueuedPath); assertFolderContents(crashPath, "bash.core.431"); assertFolderContents(processingDir, "id-123", "id-321"); @@ -116,7 +117,7 @@ public class CoredumpHandlerTest { createFileAged(crashPath.resolve("hs_err_pid2421.log"), Duration.ofSeconds(550)); when(coredumpIdSupplier.get()).thenReturn("id-123").thenReturn("id-321"); - Optional<ContainerPath> enqueuedPath = coredumpHandler.enqueueCoredump(crashPath, processingDir); + Optional<ContainerPath> enqueuedPath = coredumpHandler.enqueueCoredump(context, crashPath, processingDir); assertEquals(Optional.of(processingDir.resolve("id-123")), enqueuedPath); assertFolderContents(crashPath, "hs_err_pid69.log", "java.core.69"); assertFolderContents(processingDir, "id-123"); @@ -128,7 +129,7 @@ public class CoredumpHandlerTest { ContainerPath processingDir = context.paths().of("/some/other/processing"); // Initially there are no core dumps - Optional<ContainerPath> enqueuedPath = coredumpHandler.enqueueCoredump(containerCrashPath, processingDir); + Optional<ContainerPath> enqueuedPath = coredumpHandler.enqueueCoredump(context, containerCrashPath, processingDir); assertEquals(Optional.empty(), enqueuedPath); // 3 core dumps occur @@ -138,11 +139,11 @@ public class CoredumpHandlerTest { createFileAged(containerCrashPath.resolve("vespa-slobrok.core.673"), Duration.ofMinutes(5)); when(coredumpIdSupplier.get()).thenReturn("id-123"); - enqueuedPath = coredumpHandler.getCoredumpToProcess(containerCrashPath, processingDir); + enqueuedPath = coredumpHandler.getCoredumpToProcess(context, containerCrashPath, processingDir); assertEquals(Optional.of(processingDir.resolve("id-123")), enqueuedPath); // Running this again wont enqueue new core dumps as we are still processing the one enqueued previously - enqueuedPath = coredumpHandler.getCoredumpToProcess(containerCrashPath, processingDir); + enqueuedPath = coredumpHandler.getCoredumpToProcess(context, containerCrashPath, processingDir); assertEquals(Optional.of(processingDir.resolve("id-123")), enqueuedPath); verify(coredumpIdSupplier, times(1)).get(); } @@ -303,7 +304,7 @@ public class CoredumpHandlerTest { private static SecretSharedKey makeFixedSecretSharedKey() { byte[] keyBytes = bytesOf("very secret yes!"); // 128 bits var secretKey = new SecretKeySpec(keyBytes, "AES"); - byte[] keyId = bytesOf("the shiniest key"); + var keyId = KeyId.ofString("the shiniest key"); // We don't parse any of these fields in the test, so just use dummy contents. byte[] enc = bytesOf("hello world"); byte[] ciphertext = bytesOf("imaginary ciphertext"); diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/file/FileMoverTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/file/FileMoverTest.java new file mode 100644 index 00000000000..5eb02dfc7fa --- /dev/null +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/file/FileMoverTest.java @@ -0,0 +1,73 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.node.admin.task.util.file; + +import com.yahoo.vespa.hosted.node.admin.component.TaskContext; +import com.yahoo.vespa.test.file.TestFileSystem; +import org.junit.jupiter.api.Test; + +import java.io.IOException; +import java.io.UncheckedIOException; +import java.nio.file.FileAlreadyExistsException; +import java.nio.file.FileSystem; +import java.nio.file.NoSuchFileException; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.junit.jupiter.api.Assertions.fail; +import static org.mockito.Mockito.mock; + +/** + * @author hakonhall + */ +class FileMoverTest { + private final FileSystem fileSystem = TestFileSystem.create(); + private final TaskContext context = mock(TaskContext.class); + private final UnixPath source = new UnixPath(fileSystem.getPath("/from/source")); + private final UnixPath destination = new UnixPath(fileSystem.getPath("/to/destination")); + private final FileMover mover = new FileMover(source.toPath(), destination.toPath()); + + @Test + void movingRegularFile() { + assertConvergeThrows(() -> mover.converge(context), NoSuchFileException.class, "/from/source"); + + source.createParents().writeUtf8File("content"); + assertConvergeThrows(() -> mover.converge(context), NoSuchFileException.class, "/to/destination"); + + destination.createParents(); + assertTrue(mover.converge(context)); + assertFalse(source.exists()); + assertTrue(destination.exists()); + assertEquals("content", destination.readUtf8File()); + + assertFalse(mover.converge(context)); + + source.writeUtf8File("content 2"); + assertConvergeThrows(() -> mover.converge(context), FileAlreadyExistsException.class, "/to/destination"); + + mover.replaceExisting(); + assertTrue(mover.converge(context)); + + source.writeUtf8File("content 3"); + destination.deleteIfExists(); + destination.createDirectory(); + assertTrue(mover.converge(context)); + } + + private void assertConvergeThrows(Runnable runnable, Class<?> expectedRootExceptionClass, String expectedMessage) { + try { + runnable.run(); + fail(); + } catch (Throwable t) { + Throwable rootCause = t; + do { + Throwable cause = rootCause.getCause(); + if (cause == null) break; + rootCause = cause; + } while (true); + + assertTrue(expectedRootExceptionClass.isInstance(rootCause), "Unexpected root cause: " + rootCause); + assertEquals(expectedMessage, rootCause.getMessage()); + } + } +}
\ No newline at end of file diff --git a/parent/pom.xml b/parent/pom.xml index 29da1b59155..549d66e37a0 100644 --- a/parent/pom.xml +++ b/parent/pom.xml @@ -442,6 +442,7 @@ </build> </profile> </profiles> + <dependencyManagement> <dependencies> <!-- Please keep this list lexically sorted by groupId, then artifactId.--> @@ -649,6 +650,11 @@ </dependency> <dependency> <groupId>org.apache.commons</groupId> + <artifactId>commons-text</artifactId> + <version>${commons-text.version}</version> + </dependency> + <dependency> + <groupId>org.apache.commons</groupId> <artifactId>commons-math3</artifactId> <version>${commons.math3.version}</version> </dependency> @@ -891,9 +897,9 @@ <version>${eclipse-collections.version}</version> </dependency> <dependency> - <groupId>org.eclipse.jetty.http2</groupId> - <artifactId>http2-common</artifactId> - <version>${jetty.version}</version> + <groupId>org.eclipse.jetty.alpn</groupId> + <artifactId>alpn-api</artifactId> + <version>${jetty-alpn.version}</version> </dependency> <dependency> <groupId>org.eclipse.jetty.http2</groupId> @@ -907,37 +913,32 @@ </dependency> <dependency> <groupId>org.eclipse.jetty</groupId> - <artifactId>jetty-alpn-server</artifactId> - <version>${jetty.version}</version> - </dependency> - <dependency> - <groupId>org.eclipse.jetty</groupId> <artifactId>jetty-client</artifactId> <version>${jetty.version}</version> </dependency> <dependency> <groupId>org.eclipse.jetty</groupId> - <artifactId>jetty-http</artifactId> + <artifactId>jetty-continuation</artifactId> <version>${jetty.version}</version> </dependency> <dependency> <groupId>org.eclipse.jetty</groupId> - <artifactId>jetty-io</artifactId> + <artifactId>jetty-server</artifactId> <version>${jetty.version}</version> </dependency> <dependency> <groupId>org.eclipse.jetty</groupId> - <artifactId>jetty-jmx</artifactId> + <artifactId>jetty-servlet</artifactId> <version>${jetty.version}</version> </dependency> <dependency> <groupId>org.eclipse.jetty</groupId> - <artifactId>jetty-server</artifactId> + <artifactId>jetty-servlets</artifactId> <version>${jetty.version}</version> </dependency> <dependency> <groupId>org.eclipse.jetty</groupId> - <artifactId>jetty-servlet</artifactId> + <artifactId>jetty-jmx</artifactId> <version>${jetty.version}</version> </dependency> <dependency> @@ -946,11 +947,6 @@ <version>${jetty.version}</version> </dependency> <dependency> - <groupId>org.eclipse.jetty.toolchain</groupId> - <artifactId>jetty-jakarta-servlet-api</artifactId> - <version>${jetty-servlet-api.version}</version> - </dependency> - <dependency> <groupId>org.glassfish.jaxb</groupId> <artifactId>jaxb-runtime</artifactId> <version>2.3.2</version> <!-- 2.3.3 has a BROKEN manifest --> @@ -1094,6 +1090,89 @@ <artifactId>json-smart</artifactId> <version>${json-smart.version}</version> </dependency> + <dependency> + <!-- Force fresh woodstox-core without security issue hadoop-3.3.4 --> + <groupId>com.fasterxml.woodstox</groupId> + <artifactId>woodstox-core</artifactId> + <version>${woodstox.version}</version> + </dependency> + <dependency> + <!-- Force fresh jersey-json without security issue hadoop-3.3.4 --> + <groupId>com.sun.jersey</groupId> + <artifactId>jersey-json</artifactId> + <version>${sun-jersey-json.version}</version> + </dependency> + <dependency> + <!-- Force fresh jettison without security issue hadoop-3.3.4 --> + <groupId>org.codehaus.jettison</groupId> + <artifactId>jettison</artifactId> + <version>${jettison.version}</version> + </dependency> + <dependency> + <!-- Transitive dependencies from pig-0.16 up-to-date --> + <groupId>tomcat</groupId> + <artifactId>jasper-runtime</artifactId> + <version>${tomcat-jasper.version}</version> + </dependency> + <dependency> + <!-- Transitive dependencies from pig-0.16 up-to-date --> + <groupId>tomcat</groupId> + <artifactId>jasper-compiler</artifactId> + <version>${tomcat-jasper.version}</version> + </dependency> + <!-- Hadoop dependencies --> + <dependency> + <groupId>org.apache.hadoop</groupId> + <artifactId>hadoop-common</artifactId> + <version>${hadoop.version}</version> + <exclusions> + <exclusion> + <groupId>log4j</groupId> + <artifactId>log4j</artifactId> + </exclusion> + <exclusion> + <groupId>org.slf4j</groupId> + <artifactId>slf4j-log4j12</artifactId> + </exclusion> + </exclusions> + </dependency> + <dependency> + <groupId>org.apache.hadoop</groupId> + <artifactId>hadoop-client</artifactId> + <version>${hadoop.version}</version> + <exclusions> + <exclusion> + <groupId>log4j</groupId> + <artifactId>log4j</artifactId> + </exclusion> + <exclusion> + <groupId>org.slf4j</groupId> + <artifactId>slf4j-log4j12</artifactId> + </exclusion> + </exclusions> + </dependency> + <dependency> + <groupId>org.apache.pig</groupId> + <artifactId>pig</artifactId> + <version>${pig.version}</version> + <classifier>h2</classifier> + </dependency> + <dependency> + <!-- Hadoop test dependency --> + <groupId>org.apache.hadoop</groupId> + <artifactId>hadoop-minicluster</artifactId> + <version>${hadoop.version}</version> + <exclusions> + <exclusion> + <groupId>log4j</groupId> + <artifactId>log4j</artifactId> + </exclusion> + <exclusion> + <groupId>org.slf4j</groupId> + <artifactId>slf4j-log4j12</artifactId> + </exclusion> + </exclusions> + </dependency> </dependencies> </dependencyManagement> @@ -1122,14 +1201,18 @@ <bouncycastle.version>1.72</bouncycastle.version> <curator.version>5.3.0</curator.version> <commons-codec.version>1.15</commons-codec.version> + <commons-text.version>1.10.0</commons-text.version> <commons.math3.version>3.6.1</commons.math3.version> <eclipse-collections.version>11.0.0</eclipse-collections.version> <felix.version>7.0.1</felix.version> <felix.log.version>1.0.1</felix.log.version> <findbugs.version>3.0.2</findbugs.version> <!-- Should be kept in sync with guava --> + <groovy.version>3.0.13</groovy.version> + <hadoop.version>3.3.4</hadoop.version> <hdrhistogram.version>2.1.12</hdrhistogram.version> - <jetty.version>11.0.12</jetty.version> - <jetty-servlet-api.version>5.0.2</jetty-servlet-api.version> + <jettison.version>1.5.1</jettison.version> + <jetty.version>9.4.49.v20220914</jetty.version> + <jetty-alpn.version>1.1.3.v20160715</jetty-alpn.version> <jjwt.version>0.11.2</jjwt.version> <jna.version>5.11.0</jna.version> <json-smart.version>2.4.8</json-smart.version> @@ -1159,11 +1242,15 @@ <onnxruntime.version>1.12.1</onnxruntime.version> <!-- WARNING: sync cloud-tenant-base-dependencies-enforcer/pom.xml --> <org.json.version>20220320</org.json.version> <org.lz4.version>1.8.0</org.lz4.version> + <pig.version>0.16.0</pig.version> <prometheus.client.version>0.6.0</prometheus.client.version> <protobuf.version>3.21.7</protobuf.version> <spifly.version>1.3.5</spifly.version> + <sun-jersey-json.version>1.19.4</sun-jersey-json.version> <surefire.version>2.22.2</surefire.version> + <tomcat-jasper.version>5.5.23</tomcat-jasper.version> <wiremock.version>2.34.0</wiremock.version> + <woodstox.version>6.4.0</woodstox.version> <zookeeper.client.version>3.8.0</zookeeper.client.version> <doclint>all</doclint> diff --git a/searchlib/src/tests/attribute/tensorattribute/tensorattribute_test.cpp b/searchlib/src/tests/attribute/tensorattribute/tensorattribute_test.cpp index 9127c4b59fc..791ce80f62a 100644 --- a/searchlib/src/tests/attribute/tensorattribute/tensorattribute_test.cpp +++ b/searchlib/src/tests/attribute/tensorattribute/tensorattribute_test.cpp @@ -198,7 +198,7 @@ public: size_t memory_usage_cnt() const { return _memory_usage_cnt; } void add_document(uint32_t docid) override { - auto vector = _vectors.get_vector(docid).typify<double>(); + auto vector = _vectors.get_vector(docid, 0).typify<double>(); _adds.emplace_back(docid, DoubleVector(vector.begin(), vector.end())); } std::unique_ptr<PrepareResult> prepare_add_document(uint32_t docid, @@ -214,11 +214,11 @@ public: auto* mock_result = dynamic_cast<MockPrepareResult*>(prepare_result.get()); assert(mock_result); EXPECT_EQUAL(docid, mock_result->docid); - auto vector = _vectors.get_vector(docid).typify<double>(); + auto vector = _vectors.get_vector(docid, 0).typify<double>(); _complete_adds.emplace_back(docid, DoubleVector(vector.begin(), vector.end())); } void remove_document(uint32_t docid) override { - auto vector = _vectors.get_vector(docid).typify<double>(); + auto vector = _vectors.get_vector(docid, 0).typify<double>(); _removes.emplace_back(docid, DoubleVector(vector.begin(), vector.end())); } void assign_generation(generation_t current_gen) override { diff --git a/searchlib/src/tests/tensor/direct_tensor_store/direct_tensor_store_test.cpp b/searchlib/src/tests/tensor/direct_tensor_store/direct_tensor_store_test.cpp index cb9fa8522a8..cf0656fc919 100644 --- a/searchlib/src/tests/tensor/direct_tensor_store/direct_tensor_store_test.cpp +++ b/searchlib/src/tests/tensor/direct_tensor_store/direct_tensor_store_test.cpp @@ -2,22 +2,23 @@ #include <vespa/searchlib/tensor/direct_tensor_store.h> #include <vespa/vespalib/gtest/gtest.h> -#include <vespa/eval/eval/simple_value.h> +#include <vespa/eval/eval/fast_value.h> #include <vespa/eval/eval/tensor_spec.h> #include <vespa/eval/eval/value.h> +#include <vespa/eval/eval/value_codec.h> #include <vespa/vespalib/datastore/datastore.hpp> using namespace search::tensor; using vespalib::datastore::EntryRef; -using vespalib::eval::SimpleValue; +using vespalib::eval::FastValueBuilderFactory; using vespalib::eval::TensorSpec; using vespalib::eval::Value; using vespalib::eval::ValueType; using vespalib::eval::TypedCells; using vespalib::MemoryUsage; -vespalib::string tensor_spec("tensor(x{})"); +vespalib::string tensor_type_spec("tensor(x{})"); class MockBigTensor : public Value { @@ -41,21 +42,21 @@ public: Value::UP make_tensor(const TensorSpec& spec) { - auto value = SimpleValue::from_spec(spec); + auto value = value_from_spec(spec, FastValueBuilderFactory::get()); return std::make_unique<MockBigTensor>(std::move(value)); } Value::UP make_tensor(double value) { - return make_tensor(TensorSpec(tensor_spec).add({{"x", "a"}}, value)); + return make_tensor(TensorSpec(tensor_type_spec).add({{"x", "a"}}, value)); } class DirectTensorStoreTest : public ::testing::Test { public: DirectTensorStore store; - DirectTensorStoreTest() : store() {} + DirectTensorStoreTest() : store(ValueType::from_spec(tensor_type_spec)) {} virtual ~DirectTensorStoreTest() { store.reclaim_all_memory(); @@ -124,5 +125,25 @@ TEST_F(DirectTensorStoreTest, move_on_compact_allocates_new_entry_and_leaves_old EXPECT_GT(mem_2.usedBytes(), mem_1.usedBytes() + tensor_mem_usage.allocatedBytes()); } +TEST_F(DirectTensorStoreTest, get_typed_cells) +{ + auto tensor_spec = TensorSpec(tensor_type_spec).add({{"x", "a"}}, 4.5).add({{"x", "b"}}, 5.5).add({{"x", "c"}}, 6.5).add({{"x", "d"}}, 7.5); + auto tensor = value_from_spec(tensor_spec, FastValueBuilderFactory::get()); + auto ref = store.store_tensor(std::move(tensor)); + std::vector<double> values; + for (uint32_t subspace = 0; subspace < 4; ++subspace) { + auto cells = store.get_typed_cells(ref, subspace).typify<double>(); + EXPECT_EQ(1, cells.size()); + values.emplace_back(cells[0]); + } + EXPECT_EQ((std::vector<double>{4.5, 5.5, 6.5, 7.5}), values); + for (auto tref : { ref, EntryRef() }) { + auto subspace = tref.valid() ? 4 : 0; + auto cells = store.get_typed_cells(tref, subspace).typify<double>(); + EXPECT_EQ(1, cells.size()); + EXPECT_EQ(0.0, cells[0]); + } +} + GTEST_MAIN_RUN_ALL_TESTS() diff --git a/searchlib/src/tests/tensor/hnsw_index/hnsw_index_test.cpp b/searchlib/src/tests/tensor/hnsw_index/hnsw_index_test.cpp index 3a50601abbd..8d3d389090b 100644 --- a/searchlib/src/tests/tensor/hnsw_index/hnsw_index_test.cpp +++ b/searchlib/src/tests/tensor/hnsw_index/hnsw_index_test.cpp @@ -43,7 +43,8 @@ public: _vectors[docid] = vec; return *this; } - vespalib::eval::TypedCells get_vector(uint32_t docid) const override { + vespalib::eval::TypedCells get_vector(uint32_t docid, uint32_t subspace) const override { + (void) subspace; ArrayRef ref(_vectors[docid]); return vespalib::eval::TypedCells(ref); } @@ -138,7 +139,7 @@ public: } void expect_top_3(uint32_t docid, std::vector<uint32_t> exp_hits) { uint32_t k = 3; - auto qv = vectors.get_vector(docid); + auto qv = vectors.get_vector(docid, 0); auto rv = index->top_k_candidates(qv, k, global_filter->ptr_if_active()).peek(); std::sort(rv.begin(), rv.end(), LesserDistance()); size_t idx = 0; @@ -158,7 +159,7 @@ public: check_with_distance_threshold(docid); } void check_with_distance_threshold(uint32_t docid) { - auto qv = vectors.get_vector(docid); + auto qv = vectors.get_vector(docid, 0); uint32_t k = 3; auto rv = index->top_k_candidates(qv, k, global_filter->ptr_if_active()).peek(); std::sort(rv.begin(), rv.end(), LesserDistance()); @@ -712,7 +713,7 @@ public: UP prepare_add(uint32_t docid, uint32_t max_level = 0) { level_generator->level = max_level; vespalib::GenerationHandler::Guard dummy; - auto vector = vectors.get_vector(docid); + auto vector = vectors.get_vector(docid, 0); return index->prepare_add_document(docid, vector, dummy); } void complete_add(uint32_t docid, UP up) { diff --git a/searchlib/src/tests/tensor/hnsw_index/stress_hnsw_mt.cpp b/searchlib/src/tests/tensor/hnsw_index/stress_hnsw_mt.cpp index 81b56909d57..c5c88d2eeff 100644 --- a/searchlib/src/tests/tensor/hnsw_index/stress_hnsw_mt.cpp +++ b/searchlib/src/tests/tensor/hnsw_index/stress_hnsw_mt.cpp @@ -110,8 +110,9 @@ public: memcpy(&_vectors[docid], vec.cbegin(), sizeof(MallocPointVector)); return *this; } - vespalib::eval::TypedCells get_vector(uint32_t docid) const override { + vespalib::eval::TypedCells get_vector(uint32_t docid, uint32_t subspace) const override { assert(docid < NUM_POSSIBLE_DOCS); + (void) subspace; ConstVectorRef ref(_vectors[docid]); return vespalib::eval::TypedCells(ref); } diff --git a/searchlib/src/tests/tensor/tensor_buffer_store/tensor_buffer_store_test.cpp b/searchlib/src/tests/tensor/tensor_buffer_store/tensor_buffer_store_test.cpp index 3bbb6cd334e..05e40200167 100644 --- a/searchlib/src/tests/tensor/tensor_buffer_store/tensor_buffer_store_test.cpp +++ b/searchlib/src/tests/tensor/tensor_buffer_store/tensor_buffer_store_test.cpp @@ -1,14 +1,15 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include <vespa/searchlib/tensor/tensor_buffer_store.h> -#include <vespa/eval/eval/simple_value.h> +#include <vespa/eval/eval/fast_value.h> #include <vespa/eval/eval/tensor_spec.h> #include <vespa/eval/eval/value.h> +#include <vespa/eval/eval/value_codec.h> #include <vespa/vespalib/gtest/gtest.h> using search::tensor::TensorBufferStore; using vespalib::datastore::EntryRef; -using vespalib::eval::SimpleValue; +using vespalib::eval::FastValueBuilderFactory; using vespalib::eval::TensorSpec; using vespalib::eval::Value; using vespalib::eval::ValueType; @@ -52,7 +53,7 @@ TensorBufferStoreTest::store_tensor(const Value& tensor) EntryRef TensorBufferStoreTest::store_tensor(const TensorSpec& spec) { - auto tensor = SimpleValue::from_spec(spec); + auto tensor = value_from_spec(spec, FastValueBuilderFactory::get()); return store_tensor(*tensor); } @@ -161,4 +162,22 @@ TEST_F(TensorBufferStoreTest, stored_tensor_can_be_encoded_and_stored_as_encoded } } +TEST_F(TensorBufferStoreTest, get_typed_cells) +{ + auto ref = store_tensor(tensor_specs.back()); + std::vector<double> values; + for (uint32_t subspace = 0; subspace < 4; ++subspace) { + auto cells = _store.get_typed_cells(ref, subspace).typify<double>(); + EXPECT_EQ(1, cells.size()); + values.emplace_back(cells[0]); + } + EXPECT_EQ((std::vector<double>{4.5, 5.5, 6.5, 7.5}), values); + for (auto tref : { ref, EntryRef() }) { + auto subspace = tref.valid() ? 4 : 0; + auto cells = _store.get_typed_cells(tref, subspace).typify<double>(); + EXPECT_EQ(1, cells.size()); + EXPECT_EQ(0.0, cells[0]); + } +} + GTEST_MAIN_RUN_ALL_TESTS() diff --git a/searchlib/src/vespa/searchlib/tensor/CMakeLists.txt b/searchlib/src/vespa/searchlib/tensor/CMakeLists.txt index 4d98e7d59a8..2e8b619b2c4 100644 --- a/searchlib/src/vespa/searchlib/tensor/CMakeLists.txt +++ b/searchlib/src/vespa/searchlib/tensor/CMakeLists.txt @@ -11,6 +11,7 @@ vespa_add_library(searchlib_tensor OBJECT direct_tensor_store.cpp distance_calculator.cpp distance_function_factory.cpp + empty_subspace.cpp euclidean_distance.cpp geo_degrees_distance.cpp hamming_distance.cpp diff --git a/searchlib/src/vespa/searchlib/tensor/dense_tensor_attribute.cpp b/searchlib/src/vespa/searchlib/tensor/dense_tensor_attribute.cpp index 08a511c65bf..e69a6d925e9 100644 --- a/searchlib/src/vespa/searchlib/tensor/dense_tensor_attribute.cpp +++ b/searchlib/src/vespa/searchlib/tensor/dense_tensor_attribute.cpp @@ -93,9 +93,9 @@ DenseTensorAttribute::get_state(const vespalib::slime::Inserter& inserter) const } vespalib::eval::TypedCells -DenseTensorAttribute::get_vector(uint32_t docid) const +DenseTensorAttribute::get_vector(uint32_t docid, uint32_t subspace) const { - EntryRef ref = acquire_entry_ref(docid); + EntryRef ref = (subspace == 0) ? acquire_entry_ref(docid) : EntryRef(); return _denseTensorStore.get_typed_cells(ref); } diff --git a/searchlib/src/vespa/searchlib/tensor/dense_tensor_attribute.h b/searchlib/src/vespa/searchlib/tensor/dense_tensor_attribute.h index 58c68c71f7b..dc398b7ec6f 100644 --- a/searchlib/src/vespa/searchlib/tensor/dense_tensor_attribute.h +++ b/searchlib/src/vespa/searchlib/tensor/dense_tensor_attribute.h @@ -4,7 +4,6 @@ #include "default_nearest_neighbor_index_factory.h" #include "dense_tensor_store.h" -#include "doc_vector_access.h" #include "tensor_attribute.h" #include "typed_cells_comparator.h" #include <memory> @@ -17,7 +16,7 @@ class NearestNeighborIndex; * Attribute vector class used to store dense tensors for all * documents in memory. */ -class DenseTensorAttribute : public TensorAttribute, public DocVectorAccess { +class DenseTensorAttribute : public TensorAttribute { private: DenseTensorStore _denseTensorStore; TypedCellsComparator _comp; @@ -35,7 +34,7 @@ public: void get_state(const vespalib::slime::Inserter& inserter) const override; // Implements DocVectorAccess - vespalib::eval::TypedCells get_vector(uint32_t docid) const override; + vespalib::eval::TypedCells get_vector(uint32_t docid, uint32_t subspace) const override; }; } diff --git a/searchlib/src/vespa/searchlib/tensor/dense_tensor_store.cpp b/searchlib/src/vespa/searchlib/tensor/dense_tensor_store.cpp index c1ba8dba382..c7327422f81 100644 --- a/searchlib/src/vespa/searchlib/tensor/dense_tensor_store.cpp +++ b/searchlib/src/vespa/searchlib/tensor/dense_tensor_store.cpp @@ -79,9 +79,8 @@ DenseTensorStore::DenseTensorStore(const ValueType &type, std::shared_ptr<vespal _tensorSizeCalc(type), _bufferType(_tensorSizeCalc, std::move(allocator)), _type(type), - _emptySpace() + _empty(type) { - _emptySpace.resize(getBufSize(), 0); _store.addType(&_bufferType); _store.init_primary_buffers(); _store.enableFreeLists(); diff --git a/searchlib/src/vespa/searchlib/tensor/dense_tensor_store.h b/searchlib/src/vespa/searchlib/tensor/dense_tensor_store.h index b23886c3ac3..9d0ad6536b0 100644 --- a/searchlib/src/vespa/searchlib/tensor/dense_tensor_store.h +++ b/searchlib/src/vespa/searchlib/tensor/dense_tensor_store.h @@ -3,6 +3,7 @@ #pragma once #include "tensor_store.h" +#include "empty_subspace.h" #include <vespa/eval/eval/value_type.h> #include <vespa/eval/eval/typed_cells.h> #include <vespa/vespalib/datastore/datastore.h> @@ -50,7 +51,7 @@ private: TensorSizeCalc _tensorSizeCalc; BufferType _bufferType; ValueType _type; // type of dense tensor - std::vector<char> _emptySpace; + EmptySubspace _empty; public: DenseTensorStore(const ValueType &type, std::shared_ptr<vespalib::alloc::MemoryAllocator> allocator); ~DenseTensorStore() override; @@ -74,7 +75,10 @@ public: DenseTensorStore* as_dense() override; vespalib::eval::TypedCells get_typed_cells(EntryRef ref) const { - return vespalib::eval::TypedCells(ref.valid() ? getRawBuffer(ref) : &_emptySpace[0], + if (!ref.valid()) { + return _empty.cells(); + } + return vespalib::eval::TypedCells(getRawBuffer(ref), _type.cell_type(), getNumCells()); } // The following method is meant to be used only for unit tests. diff --git a/searchlib/src/vespa/searchlib/tensor/direct_tensor_attribute.cpp b/searchlib/src/vespa/searchlib/tensor/direct_tensor_attribute.cpp index c30b48f8b40..f1dd7238805 100644 --- a/searchlib/src/vespa/searchlib/tensor/direct_tensor_attribute.cpp +++ b/searchlib/src/vespa/searchlib/tensor/direct_tensor_attribute.cpp @@ -3,13 +3,15 @@ #include "direct_tensor_attribute.h" #include <vespa/eval/eval/fast_value.h> #include <vespa/eval/eval/value.h> +#include <vespa/searchcommon/attribute/config.h> using vespalib::eval::FastValueBuilderFactory; namespace search::tensor { DirectTensorAttribute::DirectTensorAttribute(stringref name, const Config &cfg) - : TensorAttribute(name, cfg, _direct_store) + : TensorAttribute(name, cfg, _direct_store), + _direct_store(cfg.tensorType()) { } @@ -71,4 +73,11 @@ DirectTensorAttribute::get_tensor_ref(DocId docId) const return *ptr; } +vespalib::eval::TypedCells +DirectTensorAttribute::get_vector(uint32_t docid, uint32_t subspace) const +{ + EntryRef ref = acquire_entry_ref(docid); + return _direct_store.get_typed_cells(ref, subspace); +} + } // namespace diff --git a/searchlib/src/vespa/searchlib/tensor/direct_tensor_attribute.h b/searchlib/src/vespa/searchlib/tensor/direct_tensor_attribute.h index 1e929ad6933..98c34ec1060 100644 --- a/searchlib/src/vespa/searchlib/tensor/direct_tensor_attribute.h +++ b/searchlib/src/vespa/searchlib/tensor/direct_tensor_attribute.h @@ -23,6 +23,9 @@ public: void set_tensor(DocId docId, std::unique_ptr<vespalib::eval::Value> tensor); const vespalib::eval::Value &get_tensor_ref(DocId docId) const override; bool supports_get_tensor_ref() const override { return true; } + + // Implements DocVectorAccess + vespalib::eval::TypedCells get_vector(uint32_t docid, uint32_t subspace) const override; }; } // namespace search::tensor diff --git a/searchlib/src/vespa/searchlib/tensor/direct_tensor_store.cpp b/searchlib/src/vespa/searchlib/tensor/direct_tensor_store.cpp index 013e7dedeba..cfc70cddb5c 100644 --- a/searchlib/src/vespa/searchlib/tensor/direct_tensor_store.cpp +++ b/searchlib/src/vespa/searchlib/tensor/direct_tensor_store.cpp @@ -50,9 +50,10 @@ DirectTensorStore::add_entry(TensorSP tensor) return ref; } -DirectTensorStore::DirectTensorStore() +DirectTensorStore::DirectTensorStore(const vespalib::eval::ValueType& tensor_type) : TensorStore(_tensor_store), - _tensor_store(std::make_unique<TensorBufferType>()) + _tensor_store(std::make_unique<TensorBufferType>()), + _empty(tensor_type) { _tensor_store.enableFreeLists(); } diff --git a/searchlib/src/vespa/searchlib/tensor/direct_tensor_store.h b/searchlib/src/vespa/searchlib/tensor/direct_tensor_store.h index c55dda5646a..e11c215d44c 100644 --- a/searchlib/src/vespa/searchlib/tensor/direct_tensor_store.h +++ b/searchlib/src/vespa/searchlib/tensor/direct_tensor_store.h @@ -3,6 +3,8 @@ #pragma once #include "tensor_store.h" +#include "empty_subspace.h" +#include <vespa/eval/eval/value.h> #include <vespa/vespalib/datastore/datastore.h> namespace vespalib::eval { struct Value; } @@ -32,11 +34,12 @@ private: }; TensorStoreType _tensor_store; + EmptySubspace _empty; EntryRef add_entry(TensorSP tensor); public: - DirectTensorStore(); + DirectTensorStore(const vespalib::eval::ValueType& tensor_type); ~DirectTensorStore() override; using RefType = TensorStoreType::RefType; @@ -56,6 +59,17 @@ public: EntryRef store_encoded_tensor(vespalib::nbostream& encoded) override; std::unique_ptr<vespalib::eval::Value> get_tensor(EntryRef ref) const override; bool encode_stored_tensor(EntryRef ref, vespalib::nbostream& target) const override; + vespalib::eval::TypedCells get_typed_cells(EntryRef ref, uint32_t subspace) const { + auto tensor = get_tensor_ptr(ref); + if (tensor == nullptr || subspace >= tensor->index().size()) { + return _empty.cells(); + } + auto cells = tensor->cells(); + auto type = tensor->type(); + auto data = static_cast<const char *>(cells.data); + auto dense_subspace_size = type.dense_subspace_size(); + return vespalib::eval::TypedCells(data + vespalib::eval::CellTypeUtils::mem_size(type.cell_type(), subspace * dense_subspace_size), cells.type, dense_subspace_size); + } }; } diff --git a/searchlib/src/vespa/searchlib/tensor/doc_vector_access.h b/searchlib/src/vespa/searchlib/tensor/doc_vector_access.h index a581fdc20a7..7a64c1d28ae 100644 --- a/searchlib/src/vespa/searchlib/tensor/doc_vector_access.h +++ b/searchlib/src/vespa/searchlib/tensor/doc_vector_access.h @@ -15,7 +15,7 @@ namespace search::tensor { class DocVectorAccess { public: virtual ~DocVectorAccess() {} - virtual vespalib::eval::TypedCells get_vector(uint32_t docid) const = 0; + virtual vespalib::eval::TypedCells get_vector(uint32_t docid, uint32_t subspace) const = 0; }; } diff --git a/searchlib/src/vespa/searchlib/tensor/empty_subspace.cpp b/searchlib/src/vespa/searchlib/tensor/empty_subspace.cpp new file mode 100644 index 00000000000..c225a6082f5 --- /dev/null +++ b/searchlib/src/vespa/searchlib/tensor/empty_subspace.cpp @@ -0,0 +1,20 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include "empty_subspace.h" +#include <vespa/eval/eval/value_type.h> + +namespace search::tensor { + +EmptySubspace::EmptySubspace(const vespalib::eval::ValueType& type) + : _empty_space(), + _cells() +{ + auto dense_subspace_size = type.dense_subspace_size(); + auto cell_type = type.cell_type(); + _empty_space.resize(vespalib::eval::CellTypeUtils::mem_size(cell_type, dense_subspace_size), 0); + _cells = vespalib::eval::TypedCells(&_empty_space[0], cell_type, dense_subspace_size); +} + +EmptySubspace::~EmptySubspace() = default; + +} diff --git a/searchlib/src/vespa/searchlib/tensor/empty_subspace.h b/searchlib/src/vespa/searchlib/tensor/empty_subspace.h new file mode 100644 index 00000000000..8e58e35712f --- /dev/null +++ b/searchlib/src/vespa/searchlib/tensor/empty_subspace.h @@ -0,0 +1,26 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#pragma once + +#include <vespa/eval/eval/typed_cells.h> +#include <vector> + +namespace vespalib::eval { class ValueType; } + +namespace search::tensor { + +/* + * Class containg an empty subspace, used as a bad fallback when we cannot + * get a real subspace. + */ +class EmptySubspace +{ + std::vector<char> _empty_space; + vespalib::eval::TypedCells _cells; +public: + EmptySubspace(const vespalib::eval::ValueType& type); + ~EmptySubspace(); + const vespalib::eval::TypedCells& cells() const noexcept { return _cells; } +}; + +} diff --git a/searchlib/src/vespa/searchlib/tensor/hnsw_index.h b/searchlib/src/vespa/searchlib/tensor/hnsw_index.h index 1833eec3909..9a05fe223c5 100644 --- a/searchlib/src/vespa/searchlib/tensor/hnsw_index.h +++ b/searchlib/src/vespa/searchlib/tensor/hnsw_index.h @@ -132,10 +132,10 @@ protected: inline TypedCells get_vector(uint32_t nodeid) const { uint32_t docid = get_docid(nodeid); - return _vectors.get_vector(docid); + return _vectors.get_vector(docid, 0); } inline TypedCells get_vector_by_docid(uint32_t docid) const { - return _vectors.get_vector(docid); + return _vectors.get_vector(docid, 0); } double calc_distance(uint32_t lhs_nodeid, uint32_t rhs_nodeid) const; diff --git a/searchlib/src/vespa/searchlib/tensor/serialized_fast_value_attribute.cpp b/searchlib/src/vespa/searchlib/tensor/serialized_fast_value_attribute.cpp index 9cb50c43983..3ae592e6e6f 100644 --- a/searchlib/src/vespa/searchlib/tensor/serialized_fast_value_attribute.cpp +++ b/searchlib/src/vespa/searchlib/tensor/serialized_fast_value_attribute.cpp @@ -26,4 +26,11 @@ SerializedFastValueAttribute::~SerializedFastValueAttribute() _tensorStore.reclaim_all_memory(); } +vespalib::eval::TypedCells +SerializedFastValueAttribute::get_vector(uint32_t docid, uint32_t subspace) const +{ + EntryRef ref = acquire_entry_ref(docid); + return _tensorBufferStore.get_typed_cells(ref, subspace); +} + } diff --git a/searchlib/src/vespa/searchlib/tensor/serialized_fast_value_attribute.h b/searchlib/src/vespa/searchlib/tensor/serialized_fast_value_attribute.h index ab2f2dc43bf..6215dbbc461 100644 --- a/searchlib/src/vespa/searchlib/tensor/serialized_fast_value_attribute.h +++ b/searchlib/src/vespa/searchlib/tensor/serialized_fast_value_attribute.h @@ -21,6 +21,9 @@ class SerializedFastValueAttribute : public TensorAttribute { public: SerializedFastValueAttribute(vespalib::stringref baseFileName, const Config &cfg); ~SerializedFastValueAttribute() override; + + // Implements DocVectorAccess + vespalib::eval::TypedCells get_vector(uint32_t docid, uint32_t subspace) const override; }; } diff --git a/searchlib/src/vespa/searchlib/tensor/tensor_attribute.h b/searchlib/src/vespa/searchlib/tensor/tensor_attribute.h index f87c97acaad..a91bcd80731 100644 --- a/searchlib/src/vespa/searchlib/tensor/tensor_attribute.h +++ b/searchlib/src/vespa/searchlib/tensor/tensor_attribute.h @@ -3,6 +3,7 @@ #pragma once #include "i_tensor_attribute.h" +#include "doc_vector_access.h" #include "prepare_result.h" #include "tensor_store.h" #include <vespa/searchlib/attribute/not_implemented_attribute.h> @@ -16,8 +17,7 @@ namespace search::tensor { /** * Attribute vector class used to store tensors for all documents in memory. */ -class TensorAttribute : public NotImplementedAttribute, public ITensorAttribute -{ +class TensorAttribute : public NotImplementedAttribute, public ITensorAttribute, public DocVectorAccess { protected: using AtomicEntryRef = vespalib::datastore::AtomicEntryRef; using EntryRef = TensorStore::EntryRef; diff --git a/searchlib/src/vespa/searchlib/tensor/tensor_buffer_operations.cpp b/searchlib/src/vespa/searchlib/tensor/tensor_buffer_operations.cpp index d13d3f24b5b..3a861552d80 100644 --- a/searchlib/src/vespa/searchlib/tensor/tensor_buffer_operations.cpp +++ b/searchlib/src/vespa/searchlib/tensor/tensor_buffer_operations.cpp @@ -74,7 +74,8 @@ TensorBufferOperations::TensorBufferOperations(const vespalib::eval::ValueType& _dense_subspace_size(tensor_type.dense_subspace_size()), _cell_type(tensor_type.cell_type()), _addr(_num_mapped_dimensions), - _addr_refs() + _addr_refs(), + _empty(tensor_type) { _addr_refs.reserve(_addr.size()); for (auto& label : _addr) { diff --git a/searchlib/src/vespa/searchlib/tensor/tensor_buffer_operations.h b/searchlib/src/vespa/searchlib/tensor/tensor_buffer_operations.h index 7b6d089f8f2..963e9b99920 100644 --- a/searchlib/src/vespa/searchlib/tensor/tensor_buffer_operations.h +++ b/searchlib/src/vespa/searchlib/tensor/tensor_buffer_operations.h @@ -2,7 +2,7 @@ #pragma once -#include <vespa/eval/eval/cell_type.h> +#include "empty_subspace.h" #include <vespa/vespalib/datastore/aligner.h> #include <vespa/vespalib/util/arrayref.h> #include <vespa/vespalib/util/string_id.h> @@ -45,6 +45,7 @@ class TensorBufferOperations vespalib::eval::CellType _cell_type; std::vector<vespalib::string_id> _addr; std::vector<vespalib::string_id*> _addr_refs; + EmptySubspace _empty; using Aligner = vespalib::datastore::Aligner<vespalib::datastore::dynamic_alignment>; @@ -99,6 +100,18 @@ public: void reclaim_labels(vespalib::ArrayRef<char> buf) const; // Serialize stored tensor to target (used when saving attribute) void encode_stored_tensor(vespalib::ConstArrayRef<char> buf, const vespalib::eval::ValueType& type, vespalib::nbostream& target) const; + vespalib::eval::TypedCells get_empty_subspace() const noexcept { + return _empty.cells(); + } + vespalib::eval::TypedCells get_typed_cells(vespalib::ConstArrayRef<char> buf, uint32_t subspace) const { + auto num_subspaces = get_num_subspaces(buf); + if (subspace >= num_subspaces) { + return _empty.cells(); + } + auto cells_mem_size = get_cells_mem_size(num_subspaces); + auto aligner = select_aligner(cells_mem_size); + return vespalib::eval::TypedCells(buf.data() + get_cells_offset(num_subspaces, aligner) + get_cells_mem_size(subspace), _cell_type, _dense_subspace_size); + } }; } diff --git a/searchlib/src/vespa/searchlib/tensor/tensor_buffer_store.h b/searchlib/src/vespa/searchlib/tensor/tensor_buffer_store.h index 1b5520233e1..071e238d5cc 100644 --- a/searchlib/src/vespa/searchlib/tensor/tensor_buffer_store.h +++ b/searchlib/src/vespa/searchlib/tensor/tensor_buffer_store.h @@ -34,6 +34,13 @@ public: EntryRef store_encoded_tensor(vespalib::nbostream& encoded) override; std::unique_ptr<vespalib::eval::Value> get_tensor(EntryRef ref) const override; bool encode_stored_tensor(EntryRef ref, vespalib::nbostream& target) const override; + vespalib::eval::TypedCells get_typed_cells(EntryRef ref, uint32_t subspace) const { + if (!ref.valid()) { + return _ops.get_empty_subspace(); + } + auto buf = _array_store.get(ref); + return _ops.get_typed_cells(buf, subspace); + } }; } diff --git a/security-utils/src/main/java/com/yahoo/security/KeyId.java b/security-utils/src/main/java/com/yahoo/security/KeyId.java new file mode 100644 index 00000000000..08e137eff03 --- /dev/null +++ b/security-utils/src/main/java/com/yahoo/security/KeyId.java @@ -0,0 +1,89 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.security; + +import java.util.Arrays; +import java.util.Objects; + +import static com.yahoo.security.ArrayUtils.fromUtf8Bytes; +import static com.yahoo.security.ArrayUtils.toUtf8Bytes; + +/** + * Represents a named key ID comprising an arbitrary (but length-limited) + * sequence of valid UTF-8 bytes. + * + * @author vekterli + */ +public class KeyId { + + // Max length MUST be possible to fit in an unsigned byte; see SealedSharedKey token encoding/decoding. + public static final int MAX_KEY_ID_UTF8_LENGTH = 255; + + private final byte[] keyIdBytes; + + private KeyId(byte[] keyIdBytes) { + if (keyIdBytes.length > MAX_KEY_ID_UTF8_LENGTH) { + throw new IllegalArgumentException("Key ID is too large to be encoded (max is %d, got %d)" + .formatted(MAX_KEY_ID_UTF8_LENGTH, keyIdBytes.length)); + } + verifyByteStringRoundtripsAsValidUtf8(keyIdBytes); + this.keyIdBytes = keyIdBytes; + } + + /** + * Construct a KeyId containing the given sequence of bytes. + * + * @param keyIdBytes array of valid UTF-8 bytes. May be zero-length, but not null. + * Note: to avoid accidental mutations, the key bytes are deep-copied. + * @return a new KeyId instance + */ + public static KeyId ofBytes(byte[] keyIdBytes) { + Objects.requireNonNull(keyIdBytes); + return new KeyId(Arrays.copyOf(keyIdBytes, keyIdBytes.length)); + } + + /** + * Construct a KeyId containing the UTF-8 byte representation of the given string. + * + * @param keyId a string whose UTF-8 byte representation will be the key ID. May be + * zero-length but not null. + * @return a new KeyId instance + */ + public static KeyId ofString(String keyId) { + Objects.requireNonNull(keyId); + return new KeyId(toUtf8Bytes(keyId)); + } + + /** + * @return the raw backing byte array. <strong>Must therefore not be mutated.</strong> + */ + public byte[] asBytes() { return keyIdBytes; } + + public String asString() { return fromUtf8Bytes(keyIdBytes); } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + KeyId keyId = (KeyId) o; + return Arrays.equals(keyIdBytes, keyId.keyIdBytes); + } + + @Override + public int hashCode() { + return Arrays.hashCode(keyIdBytes); + } + + @Override + public String toString() { + return "KeyId(%s)".formatted(asString()); + } + + private static void verifyByteStringRoundtripsAsValidUtf8(byte[] byteStr) { + String asStr = fromUtf8Bytes(byteStr); // Replaces bad chars with a placeholder + byte[] asBytes = toUtf8Bytes(asStr); + if (!Arrays.equals(byteStr, asBytes)) { + throw new IllegalArgumentException("Key ID is not valid normalized UTF-8"); + } + } + +} diff --git a/security-utils/src/main/java/com/yahoo/security/SealedSharedKey.java b/security-utils/src/main/java/com/yahoo/security/SealedSharedKey.java index fe1be85539c..a921b3baf87 100644 --- a/security-utils/src/main/java/com/yahoo/security/SealedSharedKey.java +++ b/security-utils/src/main/java/com/yahoo/security/SealedSharedKey.java @@ -19,20 +19,14 @@ import static com.yahoo.security.ArrayUtils.toUtf8Bytes; * This token representation is expected to be used as a convenient serialization * form when communicating shared keys. */ -public record SealedSharedKey(byte[] keyId, byte[] enc, byte[] ciphertext) { +public record SealedSharedKey(KeyId keyId, byte[] enc, byte[] ciphertext) { /** Current encoding version of opaque sealed key tokens. Must be less than 256. */ public static final int CURRENT_TOKEN_VERSION = 1; - public static final int MAX_KEY_ID_UTF8_LENGTH = 255; /** Encryption context for v1 tokens is always a 32-byte X25519 public key */ public static final int MAX_ENC_CONTEXT_LENGTH = 255; public SealedSharedKey { - if (keyId.length > MAX_KEY_ID_UTF8_LENGTH) { - throw new IllegalArgumentException("Key ID is too large to be encoded (max is %d, got %d)" - .formatted(MAX_KEY_ID_UTF8_LENGTH, keyId.length)); - } - verifyByteStringRoundtripsAsValidUtf8(keyId); if (enc.length > MAX_ENC_CONTEXT_LENGTH) { throw new IllegalArgumentException("Encryption context is too large to be encoded (max is %d, got %d)" .formatted(MAX_ENC_CONTEXT_LENGTH, enc.length)); @@ -44,11 +38,12 @@ public record SealedSharedKey(byte[] keyId, byte[] enc, byte[] ciphertext) { * reconstruct the SealedSharedKey instance when passed verbatim to fromTokenString(). */ public String toTokenString() { + byte[] keyIdBytes = keyId.asBytes(); // u8 token version || u8 length(key id) || key id || u8 length(enc) || enc || ciphertext - ByteBuffer encoded = ByteBuffer.allocate(1 + 1 + keyId.length + 1 + enc.length + ciphertext.length); + ByteBuffer encoded = ByteBuffer.allocate(1 + 1 + keyIdBytes.length + 1 + enc.length + ciphertext.length); encoded.put((byte)CURRENT_TOKEN_VERSION); - encoded.put((byte)keyId.length); - encoded.put(keyId); + encoded.put((byte)keyIdBytes.length); + encoded.put(keyIdBytes); encoded.put((byte)enc.length); encoded.put(enc); encoded.put(ciphertext); @@ -76,24 +71,15 @@ public record SealedSharedKey(byte[] keyId, byte[] enc, byte[] ciphertext) { .formatted(CURRENT_TOKEN_VERSION, version)); } int keyIdLen = Byte.toUnsignedInt(decoded.get()); - byte[] keyId = new byte[keyIdLen]; - decoded.get(keyId); - verifyByteStringRoundtripsAsValidUtf8(keyId); + byte[] keyIdBytes = new byte[keyIdLen]; + decoded.get(keyIdBytes); int encLen = Byte.toUnsignedInt(decoded.get()); byte[] enc = new byte[encLen]; decoded.get(enc); byte[] ciphertext = new byte[decoded.remaining()]; decoded.get(ciphertext); - return new SealedSharedKey(keyId, enc, ciphertext); - } - - private static void verifyByteStringRoundtripsAsValidUtf8(byte[] byteStr) { - String asStr = fromUtf8Bytes(byteStr); // Replaces bad chars with a placeholder - byte[] asBytes = toUtf8Bytes(asStr); - if (!Arrays.equals(byteStr, asBytes)) { - throw new IllegalArgumentException("Key ID is not valid normalized UTF-8"); - } + return new SealedSharedKey(KeyId.ofBytes(keyIdBytes), enc, ciphertext); } public int tokenVersion() { return CURRENT_TOKEN_VERSION; } diff --git a/security-utils/src/main/java/com/yahoo/security/SharedKeyGenerator.java b/security-utils/src/main/java/com/yahoo/security/SharedKeyGenerator.java index 47936dab114..8a1a7dd3688 100644 --- a/security-utils/src/main/java/com/yahoo/security/SharedKeyGenerator.java +++ b/security-utils/src/main/java/com/yahoo/security/SharedKeyGenerator.java @@ -60,17 +60,17 @@ public class SharedKeyGenerator { } } - public static SecretSharedKey generateForReceiverPublicKey(PublicKey receiverPublicKey, byte[] keyId) { + public static SecretSharedKey generateForReceiverPublicKey(PublicKey receiverPublicKey, KeyId keyId) { var secretKey = generateRandomSecretAesKey(); // We protect the integrity of the key ID by passing it as AAD. - var sealed = HPKE.sealBase((XECPublicKey) receiverPublicKey, EMPTY_BYTES, keyId, secretKey.getEncoded()); + var sealed = HPKE.sealBase((XECPublicKey) receiverPublicKey, EMPTY_BYTES, keyId.asBytes(), secretKey.getEncoded()); var sealedSharedKey = new SealedSharedKey(keyId, sealed.enc(), sealed.ciphertext()); return new SecretSharedKey(secretKey, sealedSharedKey); } public static SecretSharedKey fromSealedKey(SealedSharedKey sealedKey, PrivateKey receiverPrivateKey) { byte[] secretKeyBytes = HPKE.openBase(sealedKey.enc(), (XECPrivateKey) receiverPrivateKey, - EMPTY_BYTES, sealedKey.keyId(), sealedKey.ciphertext()); + EMPTY_BYTES, sealedKey.keyId().asBytes(), sealedKey.ciphertext()); return new SecretSharedKey(new SecretKeySpec(secretKeyBytes, "AES"), sealedKey); } diff --git a/security-utils/src/test/java/com/yahoo/security/KeyIdTest.java b/security-utils/src/test/java/com/yahoo/security/KeyIdTest.java new file mode 100644 index 00000000000..27b9572f5cd --- /dev/null +++ b/security-utils/src/test/java/com/yahoo/security/KeyIdTest.java @@ -0,0 +1,82 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.security; + +import org.junit.jupiter.api.Test; + +import java.util.Arrays; + +import static org.junit.jupiter.api.Assertions.assertArrayEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; + +/** + * @author vekterli + */ +public class KeyIdTest { + + @Test + void equality_predicated_on_key_id_byte_string() { + var id0s = KeyId.ofString(""); + var id1s = KeyId.ofString("1"); + var id2s = KeyId.ofString("12"); + assertEquals(id0s, id0s); + assertEquals(id1s, id1s); + assertEquals(id2s, id2s); + assertNotEquals(id0s, id1s); + assertNotEquals(id1s, id0s); + assertNotEquals(id1s, id2s); + assertNotEquals(id0s, id2s); + var id0b = KeyId.ofBytes(new byte[0]); + var id1b = KeyId.ofBytes(new byte[]{ '1' }); + var id2b = KeyId.ofBytes(new byte[]{ '1', '2' }); + assertEquals(id0s, id0b); + assertEquals(id1s, id1b); + assertEquals(id2s, id2b); + } + + @Test + void accessors_return_expected_values() { + byte[] fooBytes = new byte[]{'f','o','o'}; + byte[] barBytes = new byte[]{'b','a','r'}; + + var id1 = KeyId.ofString("foo"); + assertEquals("foo", id1.asString()); + assertArrayEquals(fooBytes, id1.asBytes()); + + var id2 = KeyId.ofBytes(barBytes); + assertEquals("bar", id2.asString()); + assertArrayEquals(barBytes, id2.asBytes()); + } + + @Test + void key_id_bytes_are_deep_copied_when_constructed_from_raw_byte_array() { + byte[] keyBytes = new byte[]{'f','o','o'}; + byte[] expected = Arrays.copyOf(keyBytes, keyBytes.length); + var id = KeyId.ofBytes(keyBytes); + keyBytes[0] = 'b'; + assertArrayEquals(expected, id.asBytes()); + } + + @Test + void can_construct_largest_possible_key_id() { + byte[] okIdBytes = new byte[KeyId.MAX_KEY_ID_UTF8_LENGTH]; + Arrays.fill(okIdBytes, (byte)'A'); + var okId = KeyId.ofBytes(okIdBytes); + assertArrayEquals(okIdBytes, okId.asBytes()); + } + + @Test + void too_big_key_id_throws() { + byte[] tooBigIdBytes = new byte[KeyId.MAX_KEY_ID_UTF8_LENGTH + 1]; + Arrays.fill(tooBigIdBytes, (byte)'A'); + assertThrows(IllegalArgumentException.class, () -> KeyId.ofBytes(tooBigIdBytes)); + } + + @Test + void malformed_utf8_key_id_is_rejected_on_construction() { + byte[] malformedIdBytes = new byte[]{ (byte)0xC0 }; // First part of a 2-byte continuation without trailing byte + assertThrows(IllegalArgumentException.class, () -> KeyId.ofBytes(malformedIdBytes)); + } + +} diff --git a/security-utils/src/test/java/com/yahoo/security/SharedKeyTest.java b/security-utils/src/test/java/com/yahoo/security/SharedKeyTest.java index f635121eda0..74b4ca0854b 100644 --- a/security-utils/src/test/java/com/yahoo/security/SharedKeyTest.java +++ b/security-utils/src/test/java/com/yahoo/security/SharedKeyTest.java @@ -21,7 +21,7 @@ import static org.junit.jupiter.api.Assertions.assertThrows; public class SharedKeyTest { - private static final byte[] KEY_ID_1 = new byte[] {1}; + private static final KeyId KEY_ID_1 = KeyId.ofString("1"); @Test void generated_secret_key_is_128_bit_aes() { @@ -49,7 +49,7 @@ public class SharedKeyTest { void token_v1_representation_is_stable() { var receiverPrivate = KeyUtils.fromBase64EncodedX25519PrivateKey("4qGcntygFn_a3uqeBa1PbDlygQ-cpOuNznTPIz9ftWE"); var receiverPublic = KeyUtils.fromBase64EncodedX25519PublicKey( "ROAH_S862tNMpbJ49lu1dPXFCPHFIXZK30pSrMZEmEg"); - byte[] keyId = toUtf8Bytes("my key ID"); + var keyId = KeyId.ofString("my key ID"); // Token generated for the above receiver public key, with the below expected shared secret (in hex) var publicToken = "AQlteSBrZXkgSUQgAtTxJJdmv3eUoW5Z3NJSdZ3poKPEkW0SJOGQXP6CaC5XfyAVoUlK_NyYIMsJKyNYKU6WmagZpVG2zQGFJoqiFA"; @@ -58,7 +58,7 @@ public class SharedKeyTest { var theirSealed = SealedSharedKey.fromTokenString(publicToken); var theirShared = SharedKeyGenerator.fromSealedKey(theirSealed, receiverPrivate); - assertArrayEquals(keyId, theirSealed.keyId()); + assertEquals(keyId, theirSealed.keyId()); assertEquals(expectedSharedSecret, hex(theirShared.secretKey().getEncoded())); } @@ -73,21 +73,21 @@ public class SharedKeyTest { @Test void token_carries_opaque_key_id_bytes_as_metadata() { - byte[] keyId = toUtf8Bytes("hello key id world"); + var keyId = KeyId.ofString("hello key id world"); var keyPair = KeyUtils.generateX25519KeyPair(); var myShared = SharedKeyGenerator.generateForReceiverPublicKey(keyPair.getPublic(), keyId); var publicToken = myShared.sealedSharedKey().toTokenString(); var theirShared = SealedSharedKey.fromTokenString(publicToken); - assertArrayEquals(theirShared.keyId(), keyId); + assertEquals(theirShared.keyId(), keyId); } @Test void key_id_integrity_is_protected_by_aad() { - byte[] goodId = toUtf8Bytes("my key 1"); - var keyPair = KeyUtils.generateX25519KeyPair(); - var myShared = SharedKeyGenerator.generateForReceiverPublicKey(keyPair.getPublic(), goodId); - var mySealed = myShared.sealedSharedKey(); - byte[] badId = toUtf8Bytes("my key 2"); + var goodId = KeyId.ofString("my key 1"); + var keyPair = KeyUtils.generateX25519KeyPair(); + var myShared = SharedKeyGenerator.generateForReceiverPublicKey(keyPair.getPublic(), goodId); + var mySealed = myShared.sealedSharedKey(); + var badId = KeyId.ofString("my key 2"); var tamperedShared = new SealedSharedKey(badId, mySealed.enc(), mySealed.ciphertext()); // Should not be able to unseal the token since the AAD auth tag won't be correct @@ -96,44 +96,32 @@ public class SharedKeyTest { } @Test - void key_id_encoding_size_is_bounded() { - byte[] okId = new byte[SealedSharedKey.MAX_KEY_ID_UTF8_LENGTH]; - Arrays.fill(okId, (byte)'A'); + void can_encode_and_decode_largest_possible_key_id() { + byte[] okIdBytes = new byte[KeyId.MAX_KEY_ID_UTF8_LENGTH]; + Arrays.fill(okIdBytes, (byte)'A'); + var okId = KeyId.ofBytes(okIdBytes); var keyPair = KeyUtils.generateX25519KeyPair(); var myShared = SharedKeyGenerator.generateForReceiverPublicKey(keyPair.getPublic(), okId); - assertArrayEquals(okId, myShared.sealedSharedKey().keyId()); + assertEquals(okId, myShared.sealedSharedKey().keyId()); var asToken = myShared.sealedSharedKey().toTokenString(); var decoded = SealedSharedKey.fromTokenString(asToken); - assertArrayEquals(okId, decoded.keyId()); - - byte[] tooBigId = new byte[SealedSharedKey.MAX_KEY_ID_UTF8_LENGTH + 1]; - Arrays.fill(tooBigId, (byte)'A'); - assertThrows(IllegalArgumentException.class, - () -> SharedKeyGenerator.generateForReceiverPublicKey(keyPair.getPublic(), tooBigId)); - } - - @Test - void malformed_utf8_key_id_is_rejected_on_construction() { - byte[] malformedId = new byte[]{ (byte)0xC0 }; // First part of a 2-byte continuation without trailing byte - var keyPair = KeyUtils.generateX25519KeyPair(); - assertThrows(IllegalArgumentException.class, - () -> SharedKeyGenerator.generateForReceiverPublicKey(keyPair.getPublic(), malformedId)); + assertEquals(okId, decoded.keyId()); } // TODO make this test less implementation specific if possible... @Test void malformed_utf8_key_id_is_rejected_on_parsing() { - byte[] goodId = new byte[] { (byte)'A' }; - var keyPair = KeyUtils.generateX25519KeyPair(); - var myShared = SharedKeyGenerator.generateForReceiverPublicKey(keyPair.getPublic(), goodId); + var goodId = KeyId.ofBytes(new byte[] { (byte)'A' }); + var keyPair = KeyUtils.generateX25519KeyPair(); + var myShared = SharedKeyGenerator.generateForReceiverPublicKey(keyPair.getPublic(), goodId); // token header is u8 version || u8 key id length || key id bytes ... - // Since the key ID is only 1 bytes long, patch it with a bad UTF-8 value (see above test) + // Since the key ID is only 1 bytes long, patch it with a bad UTF-8 value byte[] tokenBytes = Base64.getUrlDecoder().decode(myShared.sealedSharedKey().toTokenString()); - tokenBytes[2] = (byte)0xC0; - var tokenStr = Base64.getUrlEncoder().encodeToString(tokenBytes); - assertThrows(IllegalArgumentException.class, () -> SealedSharedKey.fromTokenString(tokenStr)); + tokenBytes[2] = (byte)0xC0; // First part of a 2-byte continuation without trailing byte + var patchedTokenStr = Base64.getUrlEncoder().encodeToString(tokenBytes); + assertThrows(IllegalArgumentException.class, () -> SealedSharedKey.fromTokenString(patchedTokenStr)); } static byte[] streamEncryptString(String data, SecretSharedKey secretSharedKey) throws IOException { diff --git a/vespa-enforcer-extensions/src/main/java/com/yahoo/vespa/maven/plugin/enforcer/EnforceDependencies.java b/vespa-enforcer-extensions/src/main/java/com/yahoo/vespa/maven/plugin/enforcer/EnforceDependencies.java index 154ba9db790..4ff363a3e4e 100644 --- a/vespa-enforcer-extensions/src/main/java/com/yahoo/vespa/maven/plugin/enforcer/EnforceDependencies.java +++ b/vespa-enforcer-extensions/src/main/java/com/yahoo/vespa/maven/plugin/enforcer/EnforceDependencies.java @@ -2,10 +2,7 @@ package com.yahoo.vespa.maven.plugin.enforcer; import org.apache.maven.artifact.Artifact; -import org.apache.maven.artifact.versioning.ArtifactVersion; -import org.apache.maven.artifact.versioning.DefaultArtifactVersion; -import org.apache.maven.artifact.versioning.InvalidVersionSpecificationException; -import org.apache.maven.artifact.versioning.VersionRange; +import org.apache.maven.artifact.DefaultArtifact; import org.apache.maven.enforcer.rule.api.EnforcerRule; import org.apache.maven.enforcer.rule.api.EnforcerRuleException; import org.apache.maven.enforcer.rule.api.EnforcerRuleHelper; @@ -104,38 +101,41 @@ public class EnforceDependencies implements EnforcerRule { return children; } - // Similar rule matching to bannedDependencies + /** + * Rules follows same structure as {@link DefaultArtifact#toString} output. + * Allowed variants: + * - {@code <groupId>:<artifactId>:<version>:<scope>} + * - {@code <groupId>:<artifactId>:<type>:<version>:<scope>} + * - {@code <groupId>:<artifactId>:<type>:<classifier>:<version>:<scope>} + */ private static boolean matches(Artifact dependency, String rule) throws EnforcerRuleException { String[] segments = rule.split(":"); - if (segments.length < 1 || segments.length > 6) throw new EnforcerRuleException("Invalid rule: " + rule); - if (!segmentMatches(dependency.getGroupId(), segments[0])) return false; - if (segments.length > 1 && !segmentMatches(dependency.getArtifactId(), segments[1])) return false; - if (segments.length > 2 && !versionMatches(dependency.getVersion(), segments[2])) return false; - if (segments.length > 3 && !segmentMatches(dependency.getType(), segments[3])) return false; - if (segments.length > 4 && !segmentMatches(dependency.getScope(), segments[4])) return false; - if (segments.length > 5 && dependency.hasClassifier() && !segmentMatches(dependency.getClassifier(), segments[5])) - return false; - return true; + if (segments.length < 4 || segments.length > 6) throw new EnforcerRuleException("Invalid rule: " + rule); + int index = 0; + if (!segmentMatches(dependency.getGroupId(), segments[index++])) return false; + if (!segmentMatches(dependency.getArtifactId(), segments[index++])) return false; + if (segments.length >= 5) { + if (!segmentMatches(dependency.getType(), segments[index++])) return false; + } + if (segments.length == 6) { + if (!dependency.hasClassifier() || !segmentMatches(dependency.getClassifier(), segments[index++])) return false; + } else { + if (dependency.hasClassifier()) return false; + } + if (!versionMatches(dependency.getVersion(), segments[index++])) return false; + return segmentMatches(dependency.getScope(), segments[index]); } + /** Matches simple glob like patterns using '?' and '*' */ private static boolean segmentMatches(String value, String segmentPattern) { String regex = segmentPattern - .replace(".", "\\.").replace("*", ".*").replace(":", "\\:").replace('?', '.') - .replace("[", "\\[").replace("]", "\\]").replace("(", "\\(").replace(")", "\\)"); + .replace(".", "\\.").replace("*", ".*").replace(":", "\\:").replace('?', '.').replace("(", "\\(") + .replace(")", "\\)"); return Pattern.matches(regex, value); } - private static boolean versionMatches(String rawVersion, String segmentPattern) throws EnforcerRuleException { - try { - if (segmentMatches(rawVersion, segmentPattern)) return true; - VersionRange allowedRange = VersionRange.createFromVersionSpec(segmentPattern); - ArtifactVersion version = new DefaultArtifactVersion(rawVersion); - ArtifactVersion recommended = allowedRange.getRecommendedVersion(); - if (recommended == null) return allowedRange.containsVersion(version); - return recommended.compareTo(version) <= 0; - } catch (InvalidVersionSpecificationException e) { - throw new EnforcerRuleException(e.getMessage(), e); - } + private static boolean versionMatches(String rawVersion, String segmentPattern) { + return segmentMatches(rawVersion, segmentPattern); } public void setAllowed(List<String> allowed) { this.allowedDependencies = allowed; } diff --git a/vespa-enforcer-extensions/src/test/java/com/yahoo/vespa/maven/plugin/enforcer/EnforceDependenciesTest.java b/vespa-enforcer-extensions/src/test/java/com/yahoo/vespa/maven/plugin/enforcer/EnforceDependenciesTest.java index 0dcbe595121..e8eec1dbf89 100644 --- a/vespa-enforcer-extensions/src/test/java/com/yahoo/vespa/maven/plugin/enforcer/EnforceDependenciesTest.java +++ b/vespa-enforcer-extensions/src/test/java/com/yahoo/vespa/maven/plugin/enforcer/EnforceDependenciesTest.java @@ -23,8 +23,8 @@ class EnforceDependenciesTest { artifact("com.yahoo.vespa", "container-core", "8.0.0", "provided"), artifact("com.yahoo.vespa", "testutils", "8.0.0", "test")); Set<String> rules = Set.of( - "com.yahoo.vespa:container-core:*:jar:provided", - "com.yahoo.vespa:*:*:jar:test"); + "com.yahoo.vespa:container-core:jar:*:provided", + "com.yahoo.vespa:*:jar:*:test"); assertDoesNotThrow(() -> EnforceDependencies.validateDependencies(dependencies, rules, true)); } @@ -33,7 +33,7 @@ class EnforceDependenciesTest { Set<Artifact> dependencies = Set.of( artifact("com.yahoo.vespa", "container-core", "8.0.0", "provided"), artifact("com.yahoo.vespa", "testutils", "8.0.0", "test")); - Set<String> rules = Set.of("com.yahoo.vespa:*:*:jar:test"); + Set<String> rules = Set.of("com.yahoo.vespa:*:jar:*:test"); EnforcerRuleException exception = assertThrows( EnforcerRuleException.class, () -> EnforceDependencies.validateDependencies(dependencies, rules, true)); @@ -51,8 +51,8 @@ class EnforceDependenciesTest { Set<Artifact> dependencies = Set.of( artifact("com.yahoo.vespa", "testutils", "8.0.0", "test")); Set<String> rules = Set.of( - "com.yahoo.vespa:container-core:*:jar:provided", - "com.yahoo.vespa:*:*:jar:test"); + "com.yahoo.vespa:container-core:jar:*:provided", + "com.yahoo.vespa:*:jar:*:test"); EnforcerRuleException exception = assertThrows( EnforcerRuleException.class, () -> EnforceDependencies.validateDependencies(dependencies, rules, true)); @@ -60,7 +60,7 @@ class EnforceDependenciesTest { """ Vespa dependency enforcer failed: Rules not matching any dependency: - - com.yahoo.vespa:container-core:*:jar:provided + - com.yahoo.vespa:container-core:jar:*:provided """; assertEquals(expectedErrorMessage, exception.getMessage()); } @@ -70,7 +70,7 @@ class EnforceDependenciesTest { Set<Artifact> dependencies = Set.of( artifact("com.yahoo.vespa", "testutils", "8.0.0", "test")); Set<String> rules = Set.of( - "com.yahoo.vespa:testutils:[7.0.0]:jar:test"); + "com.yahoo.vespa:testutils:jar:7.0.0:test"); EnforcerRuleException exception = assertThrows( EnforcerRuleException.class, () -> EnforceDependencies.validateDependencies(dependencies, rules, true)); @@ -80,7 +80,7 @@ class EnforceDependenciesTest { Dependencies not matching any rule: - com.yahoo.vespa:testutils:jar:8.0.0:test Rules not matching any dependency: - - com.yahoo.vespa:testutils:[7.0.0]:jar:test + - com.yahoo.vespa:testutils:jar:7.0.0:test """; assertEquals(expectedErrorMessage, exception.getMessage()); } @@ -90,7 +90,7 @@ class EnforceDependenciesTest { Set<Artifact> dependencies = Set.of( artifact("com.yahoo.vespa", "testutils", "8.0.0", "test")); Set<String> rules = Set.of( - "com.yahoo.vespa:testutils:8.0.0:jar:provided"); + "com.yahoo.vespa:testutils:jar:8.0.0:provided"); EnforcerRuleException exception = assertThrows( EnforcerRuleException.class, () -> EnforceDependencies.validateDependencies(dependencies, rules, true)); @@ -100,14 +100,48 @@ class EnforceDependenciesTest { Dependencies not matching any rule: - com.yahoo.vespa:testutils:jar:8.0.0:test Rules not matching any dependency: - - com.yahoo.vespa:testutils:8.0.0:jar:provided + - com.yahoo.vespa:testutils:jar:8.0.0:provided + """; + assertEquals(expectedErrorMessage, exception.getMessage()); + } + + @Test + void matches_shorter_rule_variant_without_type() { + Set<Artifact> dependencies = Set.of( + artifact("com.yahoo.vespa", "testutils", "8.0.0", "test")); + assertDoesNotThrow(() -> EnforceDependencies.validateDependencies( + dependencies, Set.of("com.yahoo.vespa:testutils:jar:8.0.0:test"), true)); + assertDoesNotThrow(() -> EnforceDependencies.validateDependencies( + dependencies, Set.of("com.yahoo.vespa:testutils:8.0.0:test"), true)); + } + + @Test + void matches_artifact_with_classifier() { + Set<Artifact> dependencies = Set.of( + artifact("com.google.inject", "guice", "4.2.3", "provided", "no_aop")); + assertDoesNotThrow(() -> EnforceDependencies.validateDependencies( + dependencies, Set.of("com.google.inject:guice:jar:no_aop:4.2.3:provided"), true)); + EnforcerRuleException exception = assertThrows( + EnforcerRuleException.class, + () -> EnforceDependencies.validateDependencies( + dependencies, Set.of("com.google.inject:guice:4.2.3:provided"), true)); + String expectedErrorMessage = + """ + Vespa dependency enforcer failed: + Dependencies not matching any rule: + - com.google.inject:guice:jar:no_aop:4.2.3:provided + Rules not matching any dependency: + - com.google.inject:guice:4.2.3:provided """; assertEquals(expectedErrorMessage, exception.getMessage()); } private static Artifact artifact(String groupId, String artifactId, String version, String scope) { + return artifact(groupId, artifactId, version, scope, null); + } + private static Artifact artifact(String groupId, String artifactId, String version, String scope, String classifier) { return new DefaultArtifact( - groupId, artifactId, version, scope, "jar", /*classifier*/null, new DefaultArtifactHandler("jar")); + groupId, artifactId, version, scope, "jar", classifier, new DefaultArtifactHandler("jar")); } }
\ No newline at end of file diff --git a/vespa-hadoop/pom.xml b/vespa-hadoop/pom.xml index 42ee990e727..43f7c17967d 100644 --- a/vespa-hadoop/pom.xml +++ b/vespa-hadoop/pom.xml @@ -17,171 +17,29 @@ <properties> <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding> - <hadoop.version>3.3.4</hadoop.version> - <woodstox.version>6.4.0</woodstox.version> - <sun-jersey-json.version>1.19.4</sun-jersey-json.version> - <jettison.version>1.5.1</jettison.version> - <pig.version>0.16.0</pig.version> - <tomcat-jasper.version>5.5.23</tomcat-jasper.version> - <jetty9.version>9.4.49.v20220914</jetty9.version> </properties> - <dependencyManagement> - <!-- Force 9.4.x version of transitive Jetty dependencies. Required by hadoop libraries --> - <dependencies> - <dependency> - <groupId>org.eclipse.jetty</groupId> - <artifactId>jetty-alpn-client</artifactId> - <version>${jetty9.version}</version> - </dependency> - <dependency> - <groupId>org.eclipse.jetty</groupId> - <artifactId>jetty-client</artifactId> - <version>${jetty9.version}</version> - </dependency> - <dependency> - <groupId>org.eclipse.jetty</groupId> - <artifactId>jetty-http</artifactId> - <version>${jetty9.version}</version> - </dependency> - <dependency> - <groupId>org.eclipse.jetty</groupId> - <artifactId>jetty-io</artifactId> - <version>${jetty9.version}</version> - </dependency> - <dependency> - <groupId>org.eclipse.jetty</groupId> - <artifactId>jetty-security</artifactId> - <version>${jetty9.version}</version> - </dependency> - <dependency> - <groupId>org.eclipse.jetty</groupId> - <artifactId>jetty-server</artifactId> - <version>${jetty9.version}</version> - </dependency> - <dependency> - <groupId>org.eclipse.jetty</groupId> - <artifactId>jetty-servlet</artifactId> - <version>${jetty9.version}</version> - </dependency> - <dependency> - <groupId>org.eclipse.jetty</groupId> - <artifactId>jetty-util</artifactId> - <version>${jetty9.version}</version> - </dependency> - <dependency> - <groupId>org.eclipse.jetty</groupId> - <artifactId>jetty-webapp</artifactId> - <version>${jetty9.version}</version> - </dependency> - <dependency> - <groupId>org.eclipse.jetty</groupId> - <artifactId>jetty-xml</artifactId> - <version>${jetty9.version}</version> - </dependency> - <dependency> - <groupId>org.eclipse.jetty.websocket</groupId> - <artifactId>jetty-websocket-api</artifactId> - <version>${jetty9.version}</version> - </dependency> - <dependency> - <groupId>org.eclipse.jetty.websocket</groupId> - <artifactId>jetty-websocket-client</artifactId> - <version>${jetty9.version}</version> - </dependency> - <dependency> - <groupId>org.eclipse.jetty.websocket</groupId> - <artifactId>jetty-websocket-common</artifactId> - <version>${jetty9.version}</version> - </dependency> - <!-- Force fresh woodstox-core without security issue --> - <dependency> - <groupId>com.fasterxml.woodstox</groupId> - <artifactId>woodstox-core</artifactId> - <version>${woodstox.version}</version> - </dependency> - <!-- Force fresh jersey-json without security issue --> - <dependency> - <groupId>com.sun.jersey</groupId> - <artifactId>jersey-json</artifactId> - <version>${sun-jersey-json.version}</version> - </dependency> - <!-- Force fresh jettison without security issue --> - <dependency> - <groupId>org.codehaus.jettison</groupId> - <artifactId>jettison</artifactId> - <version>${jettison.version}</version> - </dependency> - <!-- Below versions are to bring transitive dependencies from pig up-to-date --> - <dependency> - <groupId>tomcat</groupId> - <artifactId>jasper-runtime</artifactId> - <version>${tomcat-jasper.version}</version> - </dependency> - <dependency> - <groupId>tomcat</groupId> - <artifactId>jasper-compiler</artifactId> - <version>${tomcat-jasper.version}</version> - </dependency> - <dependency> - <groupId>org.codehaus.groovy</groupId> - <artifactId>groovy-all</artifactId> - <version>3.0.13</version> - <type>pom</type> - </dependency> - </dependencies> - </dependencyManagement> - <dependencies> <!-- Hadoop dependencies --> <dependency> <groupId>org.apache.hadoop</groupId> <artifactId>hadoop-common</artifactId> - <version>${hadoop.version}</version> <scope>provided</scope> - <exclusions> - <exclusion> - <groupId>log4j</groupId> - <artifactId>log4j</artifactId> - </exclusion> - <exclusion> - <groupId>org.slf4j</groupId> - <artifactId>slf4j-log4j12</artifactId> - </exclusion> - </exclusions> </dependency> <dependency> <groupId>org.apache.hadoop</groupId> <artifactId>hadoop-client</artifactId> - <version>${hadoop.version}</version> <scope>provided</scope> - <exclusions> - <exclusion> - <groupId>log4j</groupId> - <artifactId>log4j</artifactId> - </exclusion> - <exclusion> - <groupId>org.slf4j</groupId> - <artifactId>slf4j-log4j12</artifactId> - </exclusion> - </exclusions> </dependency> <dependency> <groupId>org.apache.pig</groupId> <artifactId>pig</artifactId> - <version>${pig.version}</version> <classifier>h2</classifier> <scope>provided</scope> </dependency> <!-- These are inherited from parent. Needed for correct versions on Hadoop. --> <dependency> - <groupId>commons-cli</groupId> - <artifactId>commons-cli</artifactId> - <version>1.2</version> - <scope>provided</scope> - </dependency> - <dependency> <groupId>org.apache.httpcomponents</groupId> <artifactId>httpclient</artifactId> <scope>compile</scope> @@ -196,18 +54,7 @@ <dependency> <groupId>org.apache.hadoop</groupId> <artifactId>hadoop-minicluster</artifactId> - <version>${hadoop.version}</version> <scope>test</scope> - <exclusions> - <exclusion> - <groupId>log4j</groupId> - <artifactId>log4j</artifactId> - </exclusion> - <exclusion> - <groupId>org.slf4j</groupId> - <artifactId>slf4j-log4j12</artifactId> - </exclusion> - </exclusions> </dependency> <dependency> <!-- This is a HACK due to hadoop relying on mockito in NameNodeAdapter, but not providing it. Brum, brum !! --> diff --git a/vespaclient-java/src/main/java/com/yahoo/vespa/security/tool/crypto/DecryptTool.java b/vespaclient-java/src/main/java/com/yahoo/vespa/security/tool/crypto/DecryptTool.java index af59784bfe0..fc485eb92f2 100644 --- a/vespaclient-java/src/main/java/com/yahoo/vespa/security/tool/crypto/DecryptTool.java +++ b/vespaclient-java/src/main/java/com/yahoo/vespa/security/tool/crypto/DecryptTool.java @@ -1,6 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.security.tool.crypto; +import com.yahoo.security.KeyId; import com.yahoo.security.KeyUtils; import com.yahoo.security.SealedSharedKey; import com.yahoo.security.SharedKeyGenerator; @@ -95,8 +96,8 @@ public class DecryptTool implements Tool { var tokenString = CliUtils.optionOrThrow(arguments, TOKEN_OPTION); var sealedSharedKey = SealedSharedKey.fromTokenString(tokenString.strip()); if (maybeKeyId.isPresent()) { - byte[] myKeyIdBytes = toUtf8Bytes(maybeKeyId.get()); - if (!Arrays.equals(myKeyIdBytes, sealedSharedKey.keyId())) { + var myKeyId = KeyId.ofString(maybeKeyId.get()); + if (!myKeyId.equals(sealedSharedKey.keyId())) { // Don't include raw key bytes array verbatim in message (may contain control chars etc). throw new IllegalArgumentException("Key ID specified with --key-id does not match key ID " + "used when generating the supplied token"); diff --git a/vespaclient-java/src/main/java/com/yahoo/vespa/security/tool/crypto/EncryptTool.java b/vespaclient-java/src/main/java/com/yahoo/vespa/security/tool/crypto/EncryptTool.java index cb16151c9b6..737bade400f 100644 --- a/vespaclient-java/src/main/java/com/yahoo/vespa/security/tool/crypto/EncryptTool.java +++ b/vespaclient-java/src/main/java/com/yahoo/vespa/security/tool/crypto/EncryptTool.java @@ -1,6 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.security.tool.crypto; +import com.yahoo.security.KeyId; import com.yahoo.security.KeyUtils; import com.yahoo.security.SharedKeyGenerator; import com.yahoo.vespa.security.tool.CliUtils; @@ -79,7 +80,7 @@ public class EncryptTool implements Tool { var outputPath = Paths.get(CliUtils.optionOrThrow(arguments, OUTPUT_FILE_OPTION)); var recipientPubKey = KeyUtils.fromBase64EncodedX25519PublicKey(CliUtils.optionOrThrow(arguments, RECIPIENT_PUBLIC_KEY_OPTION).strip()); - var keyId = toUtf8Bytes(CliUtils.optionOrThrow(arguments, KEY_ID_OPTION)); + var keyId = KeyId.ofString(CliUtils.optionOrThrow(arguments, KEY_ID_OPTION)); var shared = SharedKeyGenerator.generateForReceiverPublicKey(recipientPubKey, keyId); var cipher = SharedKeyGenerator.makeAesGcmEncryptionCipher(shared); diff --git a/vespaclient-java/src/main/java/com/yahoo/vespa/security/tool/crypto/TokenInfoTool.java b/vespaclient-java/src/main/java/com/yahoo/vespa/security/tool/crypto/TokenInfoTool.java index dc597e9301f..5b9f97fd430 100644 --- a/vespaclient-java/src/main/java/com/yahoo/vespa/security/tool/crypto/TokenInfoTool.java +++ b/vespaclient-java/src/main/java/com/yahoo/vespa/security/tool/crypto/TokenInfoTool.java @@ -47,7 +47,7 @@ public class TokenInfoTool implements Tool { var stdOut = invocation.stdOut(); stdOut.format("Version: %d\n", token.tokenVersion()); - stdOut.format("Key ID: %s (%s)\n", StringUtilities.escape(fromUtf8Bytes(token.keyId())), hex(token.keyId())); + stdOut.format("Key ID: %s (%s)\n", StringUtilities.escape(token.keyId().asString()), hex(token.keyId().asBytes())); stdOut.format("HPKE enc: %s\n", hex(token.enc())); stdOut.format("HPKE ciphertext: %s\n", hex(token.ciphertext())); diff --git a/vespalib/src/tests/coro/lazy/lazy_test.cpp b/vespalib/src/tests/coro/lazy/lazy_test.cpp index b838152249e..ead5e8e6427 100644 --- a/vespalib/src/tests/coro/lazy/lazy_test.cpp +++ b/vespalib/src/tests/coro/lazy/lazy_test.cpp @@ -2,37 +2,21 @@ #include <vespa/vespalib/coro/lazy.h> #include <vespa/vespalib/coro/sync_wait.h> +#include <vespa/vespalib/coro/schedule.h> +#include <vespa/vespalib/util/size_literals.h> #include <vespa/vespalib/util/require.h> +#include <vespa/vespalib/util/threadstackexecutor.h> #include <vespa/vespalib/gtest/gtest.h> #include <mutex> #include <thread> +using vespalib::Executor; using vespalib::coro::Lazy; +using vespalib::coro::ScheduleFailedException; +using vespalib::coro::schedule; using vespalib::coro::sync_wait; - -std::mutex thread_lock; -std::vector<std::thread> threads; -struct JoinThreads { - ~JoinThreads() { - for (auto &thread: threads) { - thread.join(); - } - threads.clear(); - } -}; - -auto run_in_other_thread() { - struct awaiter { - bool await_ready() const noexcept { return false; } - void await_suspend(std::coroutine_handle<> handle) const { - auto guard = std::lock_guard(thread_lock); - threads.push_back(std::thread(handle)); - } - void await_resume() const noexcept {} - }; - return awaiter(); -} +using vespalib::coro::try_schedule; Lazy<int> make_lazy(int value) { co_return value; @@ -68,9 +52,17 @@ Lazy<T> forward_value(Lazy<T> value) { } template <typename T> -Lazy<T> switch_thread(Lazy<T> value) { +Lazy<std::pair<bool,T>> try_schedule_on(Executor &executor, Lazy<T> value) { std::cerr << "switching from thread " << std::this_thread::get_id() << std::endl; - co_await run_in_other_thread(); + bool accepted = co_await try_schedule(executor); + std::cerr << "........... to thread " << std::this_thread::get_id() << std::endl; + co_return std::make_pair(accepted, co_await value); +} + +template <typename T> +Lazy<T> schedule_on(Executor &executor, Lazy<T> value) { + std::cerr << "switching from thread " << std::this_thread::get_id() << std::endl; + co_await schedule(executor); std::cerr << "........... to thread " << std::this_thread::get_id() << std::endl; co_return co_await value; } @@ -107,15 +99,28 @@ TEST(LazyTest, extract_rvalue_from_lazy_in_sync_wait) { } TEST(LazyTest, calculate_result_in_another_thread) { - JoinThreads thread_guard; - auto result = sync_wait(switch_thread(make_lazy(7))); - EXPECT_EQ(result, 7); + vespalib::ThreadStackExecutor executor(1, 128_Ki); + auto result = sync_wait(try_schedule_on(executor, make_lazy(7))); + EXPECT_EQ(result.first, true); + EXPECT_EQ(result.second, 7); + auto result2 = sync_wait(schedule_on(executor, make_lazy(8))); + EXPECT_EQ(result2, 8); } TEST(LazyTest, exceptions_are_propagated) { - JoinThreads thread_guard; - auto lazy = switch_thread(forward_value(will_throw())); + vespalib::ThreadStackExecutor executor(1, 128_Ki); + auto lazy = try_schedule_on(executor, forward_value(will_throw())); EXPECT_THROW(sync_wait(lazy), vespalib::RequireFailedException); } +TEST(LazyTest, not_able_to_switch_thread_if_executor_is_shut_down) { + vespalib::ThreadStackExecutor executor(1, 128_Ki); + executor.shutdown(); + auto result = sync_wait(try_schedule_on(executor, make_lazy(7))); + EXPECT_EQ(result.first, false); + EXPECT_EQ(result.second, 7); + auto lazy = schedule_on(executor, make_lazy(8)); + EXPECT_THROW(sync_wait(lazy), ScheduleFailedException); +} + GTEST_MAIN_RUN_ALL_TESTS() diff --git a/vespalib/src/vespa/vespalib/coro/schedule.h b/vespalib/src/vespa/vespalib/coro/schedule.h new file mode 100644 index 00000000000..6dfa5b9536c --- /dev/null +++ b/vespalib/src/vespa/vespalib/coro/schedule.h @@ -0,0 +1,76 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#pragma once + +#include <vespa/vespalib/util/executor.h> +#include <coroutine> +#include <exception> + +namespace vespalib::coro { + +struct ScheduleFailedException : std::runtime_error { + using std::runtime_error::runtime_error; +}; + +// Schedule the current coroutine on the given executor. Throws an +// exception if the request was rejected by the executor. + +auto schedule(Executor &executor) { + struct [[nodiscard]] awaiter { + Executor &executor; + awaiter(Executor &executor_in) + : executor(executor_in) {} + bool await_ready() const noexcept { return false; } + void await_suspend(std::coroutine_handle<> handle) { + struct ResumeTask : Executor::Task { + std::coroutine_handle<> handle; + ResumeTask(std::coroutine_handle<> handle_in) + : handle(handle_in) {} + void run() override { handle.resume(); } + }; + Executor::Task::UP task = std::make_unique<ResumeTask>(handle); + task = executor.execute(std::move(task)); + if (task) { + throw ScheduleFailedException("rejected by executor"); + } + } + void await_resume() const noexcept {} + }; + return awaiter(executor); +} + +// Try to schedule the current coroutine on the given executor. If the +// awaiter returns true, the coroutine is now run by the executor. If +// the awaiter returns false, the request was rejected by the executor +// and the coroutine is still running in our original context. + +auto try_schedule(Executor &executor) { + struct [[nodiscard]] awaiter { + Executor &executor; + bool accepted; + awaiter(Executor &executor_in) + : executor(executor_in), accepted(true) {} + bool await_ready() const noexcept { return false; } + bool await_suspend(std::coroutine_handle<> handle) { + struct ResumeTask : Executor::Task { + std::coroutine_handle<> handle; + ResumeTask(std::coroutine_handle<> handle_in) + : handle(handle_in) {} + void run() override { handle.resume(); } + }; + Executor::Task::UP task = std::make_unique<ResumeTask>(handle); + task = executor.execute(std::move(task)); + if (task) { + // need to start with accepted == true to avoid race + // with handle.resume() from executor thread before + // await_suspend has returned. + accepted = false; + } + return accepted; + } + [[nodiscard]] bool await_resume() const noexcept { return accepted; } + }; + return awaiter(executor); +} + +} |