diff options
33 files changed, 200 insertions, 204 deletions
diff --git a/client/go/internal/cli/cmd/prod.go b/client/go/internal/cli/cmd/prod.go index 0e00597221a..f322bd667df 100644 --- a/client/go/internal/cli/cmd/prod.go +++ b/client/go/internal/cli/cmd/prod.go @@ -105,21 +105,17 @@ https://cloud.vespa.ai/en/reference/deployment`, func newProdSubmitCmd(cli *CLI) *cobra.Command { return &cobra.Command{ Use: "submit", - Short: "Submit your application for production deployment", - Long: `Submit your application for production deployment. + Short: "Submit an application for production deployment", + Long: `Submit an application for production deployment. -This commands uploads your application package to Vespa Cloud and deploys it to +This commands uploads an application package to Vespa Cloud and deploys it to the production zones specified in deployment.xml. -Nodes are allocated to your application according to resources specified in +Nodes are allocated to the application according to resources specified in services.xml. -While submitting an application from a local development environment is -supported, it's strongly recommended that production deployments are performed -by a continuous build system. - For more information about production deployments in Vespa Cloud see: -https://cloud.vespa.ai/en/getting-to-production +https://cloud.vespa.ai/en/production-deployment https://cloud.vespa.ai/en/automated-deployments`, DisableAutoGenTag: true, SilenceUsage: true, diff --git a/config-model/src/main/java/com/yahoo/schema/processing/IndexingValidation.java b/config-model/src/main/java/com/yahoo/schema/processing/IndexingValidation.java index d8c1fb3125f..3c7e9b4066f 100644 --- a/config-model/src/main/java/com/yahoo/schema/processing/IndexingValidation.java +++ b/config-model/src/main/java/com/yahoo/schema/processing/IndexingValidation.java @@ -62,7 +62,7 @@ public class IndexingValidation extends Processor { final Set<String> prevNames = new HashSet<>(); @Override - protected ExpressionConverter branch() { + public ExpressionConverter branch() { MyConverter ret = new MyConverter(); ret.outputs.addAll(outputs); ret.prevNames.addAll(prevNames); diff --git a/config-model/src/test/derived/indexswitches/ilscripts.cfg b/config-model/src/test/derived/indexswitches/ilscripts.cfg index 77ac18e3261..5cda0a9fdc7 100644 --- a/config-model/src/test/derived/indexswitches/ilscripts.cfg +++ b/config-model/src/test/derived/indexswitches/ilscripts.cfg @@ -4,7 +4,7 @@ ilscript[].doctype "indexswitches" ilscript[].docfield[] "title" ilscript[].docfield[] "descr" ilscript[].docfield[] "source_src" -ilscript[].content[] "clear_state | guard { input source_src | switch { case \"theweb\": input source_src | tokenize normalize | summary source | index source; case \"amg\": input source_src | tokenize normalize | summary source; default: input source_src . \" partner\" | tokenize normalize | summary source | index source; }; }" +ilscript[].content[] "clear_state | guard { input source_src | switch { case \"amg\": input source_src | tokenize normalize | summary source; case \"theweb\": input source_src | tokenize normalize | summary source | index source; default: input source_src . \" partner\" | tokenize normalize | summary source | index source; }; }" ilscript[].content[] "clear_state | guard { input title | tokenize normalize stem:\"BEST\" | summary title | index title; }" ilscript[].content[] "clear_state | guard { input descr | tokenize normalize stem:\"BEST\" | summary descr | index descr; }" ilscript[].content[] "input source_src | passthrough source_src" diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/auditlog/AuditLog.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/auditlog/AuditLog.java index aa6e3b0c44d..cbd0f685d80 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/auditlog/AuditLog.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/auditlog/AuditLog.java @@ -49,7 +49,7 @@ public record AuditLog(List<Entry> entries) { public record Entry(Instant at, String principal, Method method, String resource, Optional<String> data, Client client) implements Comparable<Entry> { - private final static int maxDataLength = 1024; + final static int maxDataLength = 1024; private final static Comparator<Entry> comparator = Comparator.comparing(Entry::at).reversed(); public Entry(Instant at, Client client, String principal, Method method, String resource, byte[] data) { diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/auditlog/AuditLogger.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/auditlog/AuditLogger.java index 033cd0a52c9..13b3d9d170f 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/auditlog/AuditLogger.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/auditlog/AuditLogger.java @@ -4,11 +4,12 @@ package com.yahoo.vespa.hosted.controller.auditlog; import com.yahoo.container.jdisc.HttpRequest; import com.yahoo.jdisc.http.HttpHeaders; import com.yahoo.transaction.Mutex; +import com.yahoo.vespa.hosted.controller.auditlog.AuditLog.Entry; import com.yahoo.vespa.hosted.controller.persistence.CuratorDb; import java.io.ByteArrayInputStream; -import java.io.IOException; -import java.io.UncheckedIOException; +import java.io.InputStream; +import java.io.SequenceInputStream; import java.net.URI; import java.security.Principal; import java.time.Clock; @@ -17,6 +18,9 @@ import java.time.Instant; import java.util.Objects; import java.util.Optional; +import static com.yahoo.yolean.Exceptions.uncheck; +import static java.util.Objects.requireNonNullElse; + /** * This provides read and write operations for the audit log. * @@ -58,14 +62,8 @@ public class AuditLogger { "misconfiguration and should not happen"); } - byte[] data = new byte[0]; - try { - if (request.getData() != null) { - data = request.getData().readAllBytes(); - } - } catch (IOException e) { - throw new UncheckedIOException(e); - } + InputStream requestData = requireNonNullElse(request.getData(), InputStream.nullInputStream()); + byte[] data = uncheck(() -> requestData.readNBytes(Entry.maxDataLength)); AuditLog.Entry.Client client = parseClient(request); Instant now = clock.instant(); @@ -80,7 +78,9 @@ public class AuditLogger { } // Create a new input stream to allow callers to consume request body - return new HttpRequest(request.getJDiscRequest(), new ByteArrayInputStream(data), request.propertyMap()); + return new HttpRequest(request.getJDiscRequest(), + new SequenceInputStream(new ByteArrayInputStream(data), requestData), + request.propertyMap()); } private static AuditLog.Entry.Client parseClient(HttpRequest request) { diff --git a/default_build_settings.cmake b/default_build_settings.cmake index c82bb91accc..2785a98a396 100644 --- a/default_build_settings.cmake +++ b/default_build_settings.cmake @@ -4,11 +4,11 @@ include(VespaExtendedDefaultBuildSettings OPTIONAL) function(setup_vespa_default_build_settings_darwin) message("-- Setting up default build settings for darwin") - set(DEFAULT_EXTRA_LINK_DIRECTORY "${VESPA_DEPS_PREFIX}/lib" "/usr/local/opt/bison/lib" "/usr/local/opt/flex/lib" "/usr/local/opt/icu4c/lib" "/usr/local/opt/openssl@1.1/lib" "/usr/local/opt/openblas/lib") - list(APPEND DEFAULT_EXTRA_LINK_DIRECTORY "/usr/local/lib") + set(DEFAULT_EXTRA_LINK_DIRECTORY "${VESPA_DEPS_PREFIX}/lib" "${VESPA_HOMEBREW_PREFIX}/opt/bison/lib" "${VESPA_HOMEBREW_PREFIX}/opt/flex/lib" "${VESPA_HOMEBREW_PREFIX}/opt/icu4c/lib" "${VESPA_HOMEBREW_PREFIX}/opt/openssl@1.1/lib" "${VESPA_HOMEBREW_PREFIX}/opt/openblas/lib") + list(APPEND DEFAULT_EXTRA_LINK_DIRECTORY "${VESPA_HOMEBREW_PREFIX}/lib") set(DEFAULT_EXTRA_LINK_DIRECTORY "${DEFAULT_EXTRA_LINK_DIRECTORY}" PARENT_SCOPE) - set(DEFAULT_EXTRA_INCLUDE_DIRECTORY "${VESPA_DEPS_PREFIX}/include" "/usr/local/opt/flex/include" "/usr/local/opt/icu4c/include" "/usr/local/opt/openssl@1.1/include" "/usr/local/opt/openblas/include") - list(APPEND DEFAULT_EXTRA_INCLUDE_DIRECTORY "/usr/local/include") + set(DEFAULT_EXTRA_INCLUDE_DIRECTORY "${VESPA_DEPS_PREFIX}/include" "${VESPA_HOMEBREW_PREFIX}/opt/flex/include" "${VESPA_HOMEBREW_PREFIX}/opt/icu4c/include" "${VESPA_HOMEBREW_PREFIX}/opt/openssl@1.1/include" "${VESPA_HOMEBREW_PREFIX}/opt/openblas/include") + list(APPEND DEFAULT_EXTRA_INCLUDE_DIRECTORY "${VESPA_HOMEBREW_PREFIX}/include") set(DEFAULT_EXTRA_INCLUDE_DIRECTORY "${DEFAULT_EXTRA_INCLUDE_DIRECTORY}" PARENT_SCOPE) endfunction() @@ -84,7 +84,7 @@ endfunction() function(vespa_use_default_cmake_prefix_path) set(DEFAULT_CMAKE_PREFIX_PATH ${VESPA_DEPS_PREFIX}) if (APPLE) - list(APPEND DEFAULT_CMAKE_PREFIX_PATH "/usr/local/opt/bison" "/usr/local/opt/flex" "/usr/local/opt/openssl@1.1" "/usr/local/opt/openblas" "/usr/local/opt/icu4c") + list(APPEND DEFAULT_CMAKE_PREFIX_PATH "${VESPA_HOMEBREW_PREFIX}/opt/bison" "${VESPA_HOMEBREW_PREFIX}/opt/flex" "${VESPA_HOMEBREW_PREFIX}/opt/openssl@1.1" "${VESPA_HOMEBREW_PREFIX}/opt/openblas" "${VESPA_HOMEBREW_PREFIX}/opt/icu4c") endif() message("-- DEFAULT_CMAKE_PREFIX_PATH is ${DEFAULT_CMAKE_PREFIX_PATH}") if(NOT DEFINED CMAKE_PREFIX_PATH) @@ -203,8 +203,8 @@ function(vespa_use_default_cxx_compiler) unset(DEFAULT_CMAKE_CXX_COMPILER) if(NOT DEFINED VESPA_COMPILER_VARIANT OR VESPA_COMPILER_VARIANT STREQUAL "gcc") if(APPLE) - set(DEFAULT_CMAKE_C_COMPILER "/usr/local/bin/gcc-12") - set(DEFAULT_CMAKE_CXX_COMPILER "/usr/local/bin/g++-12") + set(DEFAULT_CMAKE_C_COMPILER "${VESPA_HOMEBREW_PREFIX}/bin/gcc-12") + set(DEFAULT_CMAKE_CXX_COMPILER "${VESPA_HOMEBREW_PREFIX}/bin/g++-12") elseif(VESPA_OS_DISTRO_COMBINED STREQUAL "amzn 2") set(DEFAULT_CMAKE_C_COMPILER "/usr/bin/gcc10-gcc") set(DEFAULT_CMAKE_CXX_COMPILER "/usr/bin/gcc10-g++") @@ -215,8 +215,8 @@ function(vespa_use_default_cxx_compiler) endif() elseif(VESPA_COMPILER_VARIANT STREQUAL "clang") if(APPLE) - set(DEFAULT_CMAKE_C_COMPILER, "/usr/local/opt/llvm/bin/clang") - set(DEFAULT_CMAKE_CXX_COMPILER "/usr/local/opt/llvm/bin/clang++") + set(DEFAULT_CMAKE_C_COMPILER, "${VESPA_HOMEBREW_PREFIX}/opt/llvm/bin/clang") + set(DEFAULT_CMAKE_CXX_COMPILER "${VESPA_HOMEBREW_PREFIX}/opt/llvm/bin/clang++") elseif(EXISTS "/usr/bin/clang" AND EXISTS "/usr/bin/clang++") set(DEFAULT_CMAKE_C_COMPILER "/usr/bin/clang") set(DEFAULT_CMAKE_CXX_COMPILER "/usr/bin/clang++") diff --git a/functions.cmake b/functions.cmake index 7fa0b0db954..7f217867314 100644 --- a/functions.cmake +++ b/functions.cmake @@ -746,6 +746,14 @@ function(vespa_detect_build_platform) elseif(APPLE) set(OS_DISTRO "darwin") set(OS_DISTRO_VERSION ${CMAKE_SYSTEM_VERSION}) + if(EXISTS "/opt/homebrew/bin/brew") + set(VESPA_HOMEBREW_PREFIX "/opt/homebrew") + elseif(EXISTS "/usr/local/bin/brew") + set(VESPA_HOMEBREW_PREFIX "/usr/local") + else() + message(FATAL_ERROR "-- Cannot determine homebrew prefix") + endif() + set(VESPA_HOMEBREW_PREFIX ${VESPA_HOMEBREW_PREFIX} PARENT_SCOPE) endif() if(OS_DISTRO) set(VESPA_OS_DISTRO ${OS_DISTRO} PARENT_SCOPE) diff --git a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/ExpressionConverter.java b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/ExpressionConverter.java index ccad9d6d08b..231f6fb7598 100644 --- a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/ExpressionConverter.java +++ b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/ExpressionConverter.java @@ -1,127 +1,22 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.indexinglanguage; -import com.yahoo.collections.Pair; -import com.yahoo.vespa.indexinglanguage.expressions.*; - -import java.lang.reflect.InvocationTargetException; -import java.util.HashMap; -import java.util.LinkedList; -import java.util.List; -import java.util.Map; +import com.yahoo.vespa.indexinglanguage.expressions.Expression; /** * @author Simon Thoresen Hult */ -@SuppressWarnings({ "UnusedDeclaration" }) public abstract class ExpressionConverter implements Cloneable { - public final Expression convert(Expression exp) { - if (exp == null) { - return null; - } - if (shouldConvert(exp)) { - return doConvert(exp); - } - if (!(exp instanceof CompositeExpression)) { - return exp; - } - try { - // The class.getMethod here takes 8% of the cpu time in reading the SSBE application package - // TODO: Implement double dispatch through visitor instead? - return (Expression)ExpressionConverter.class.getMethod("innerConvert", exp.getClass()).invoke(this, exp); - } catch (IllegalAccessException | NoSuchMethodException e) { - throw new UnsupportedOperationException(exp.getClass().getName(), e); - } catch (InvocationTargetException e) { - Throwable t = e.getTargetException(); - throw t instanceof RuntimeException ? (RuntimeException)t : new RuntimeException(t); - } - } - - public Expression innerConvert(ArithmeticExpression exp) { - return new ArithmeticExpression(convert(exp.getLeftHandSide()), - exp.getOperator(), - convert(exp.getRightHandSide())); - } - - public Expression innerConvert(CatExpression exp) { - List<Expression> lst = new LinkedList<>(); - for (Expression innerExp : exp) { - Expression next = convert(innerExp); - if (next != null) { - lst.add(next); - } - } - return new CatExpression(lst); - } - - public Expression innerConvert(ChoiceExpression exp) { - var convertedInnerExpressions = exp.asList().stream().map(inner -> convert(inner)).toList(); - return new ChoiceExpression(convertedInnerExpressions); - } - - public Expression innerConvert(ForEachExpression exp) { - return new ForEachExpression(convert(exp.getInnerExpression())); - } - - public Expression innerConvert(GuardExpression exp) { - return new GuardExpression(convert(exp.getInnerExpression())); - } - - public Expression innerConvert(IfThenExpression exp) { - return new IfThenExpression(branch().convert(exp.getLeftHandSide()), - exp.getComparator(), - branch().convert(exp.getRightHandSide()), - branch().convert(exp.getIfTrueExpression()), - branch().convert(exp.getIfFalseExpression())); - } - - public Expression innerConvert(ParenthesisExpression exp) { - return new ParenthesisExpression(convert(exp.getInnerExpression())); - } - - public Expression innerConvert(ScriptExpression exp) { - List<StatementExpression> lst = new LinkedList<>(); - for (Expression innerExp : exp) { - StatementExpression next = (StatementExpression)branch().convert(innerExp); - if (next != null) { - lst.add(next); - } - } - return new ScriptExpression(lst); - } - - public Expression innerConvert(SelectInputExpression exp) { - List<Pair<String, Expression>> cases = new LinkedList<>(); - for (Pair<String, Expression> pair : exp.getCases()) { - cases.add(new Pair<>(pair.getFirst(), branch().convert(pair.getSecond()))); - } - return new SelectInputExpression(cases); - } - - public Expression innerConvert(StatementExpression exp) { - List<Expression> lst = new LinkedList<>(); - for (Expression innerExp : exp) { - Expression next = convert(innerExp); - if (next != null) { - lst.add(next); - } - } - return new StatementExpression(lst); - } - - public Expression innerConvert(SwitchExpression exp) { - Map<String, Expression> cases = new HashMap<>(); - for (Map.Entry<String, Expression> entry : exp.getCases().entrySet()) { - Expression next = branch().convert(entry.getValue()); - if (next != null) { - cases.put(entry.getKey(), next); - } - } - return new SwitchExpression(cases, branch().convert(exp.getDefaultExpression())); + public final Expression convert(Expression expression) { + if (expression == null) return null; + if (shouldConvert(expression)) + return doConvert(expression); + else + return expression.convertChildren(this); } - protected ExpressionConverter branch() { + public ExpressionConverter branch() { return this; } diff --git a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/ValueTransformProvider.java b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/ValueTransformProvider.java index bbd8c5ebcb8..623f940b06b 100644 --- a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/ValueTransformProvider.java +++ b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/ValueTransformProvider.java @@ -20,7 +20,7 @@ public abstract class ValueTransformProvider extends ExpressionConverter { } @Override - protected final ExpressionConverter branch() { + public final ExpressionConverter branch() { return clone(); } diff --git a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/ArithmeticExpression.java b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/ArithmeticExpression.java index e4bc2dae965..b7ee444975f 100644 --- a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/ArithmeticExpression.java +++ b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/ArithmeticExpression.java @@ -4,6 +4,7 @@ package com.yahoo.vespa.indexinglanguage.expressions; import com.yahoo.document.DataType; import com.yahoo.document.NumericDataType; import com.yahoo.document.datatypes.*; +import com.yahoo.vespa.indexinglanguage.ExpressionConverter; import com.yahoo.vespa.objects.ObjectOperation; import com.yahoo.vespa.objects.ObjectPredicate; @@ -55,6 +56,12 @@ public final class ArithmeticExpression extends CompositeExpression { this.rhs = rhs; } + @Override + public ArithmeticExpression convertChildren(ExpressionConverter converter) { + // TODO: branch()? + return new ArithmeticExpression(converter.convert(lhs), op, converter.convert(rhs)); + } + public Expression getLeftHandSide() { return lhs; } diff --git a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/CatExpression.java b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/CatExpression.java index 4c14c633fbf..564ab015e10 100644 --- a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/CatExpression.java +++ b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/CatExpression.java @@ -9,6 +9,7 @@ import com.yahoo.document.datatypes.Array; import com.yahoo.document.datatypes.FieldValue; import com.yahoo.document.datatypes.StringFieldValue; import com.yahoo.document.datatypes.WeightedSet; +import com.yahoo.vespa.indexinglanguage.ExpressionConverter; import java.util.*; @@ -26,6 +27,11 @@ public final class CatExpression extends ExpressionList<Expression> { } @Override + public CatExpression convertChildren(ExpressionConverter converter) { + return new CatExpression(convertChildList(converter)); + } + + @Override protected void doExecute(ExecutionContext context) { FieldValue input = context.getValue(); DataType inputType = input != null ? input.getDataType() : null; diff --git a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/ChoiceExpression.java b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/ChoiceExpression.java index 4f83cbfdd8c..86826770828 100644 --- a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/ChoiceExpression.java +++ b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/ChoiceExpression.java @@ -3,6 +3,7 @@ package com.yahoo.vespa.indexinglanguage.expressions; import com.yahoo.document.DataType; import com.yahoo.document.datatypes.FieldValue; +import com.yahoo.vespa.indexinglanguage.ExpressionConverter; import java.util.Arrays; import java.util.Collection; @@ -32,6 +33,11 @@ public class ChoiceExpression extends ExpressionList<Expression> { } @Override + public ChoiceExpression convertChildren(ExpressionConverter converter) { + return new ChoiceExpression(asList().stream().map(choice -> converter.branch().convert(choice)).toList()); + } + + @Override protected void doExecute(ExecutionContext context) { FieldValue input = context.getValue(); for (Expression expression : this) { diff --git a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/CompositeExpression.java b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/CompositeExpression.java index 27e5524f4ad..8c00aad6bb0 100644 --- a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/CompositeExpression.java +++ b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/CompositeExpression.java @@ -2,12 +2,16 @@ package com.yahoo.vespa.indexinglanguage.expressions; import com.yahoo.document.DataType; +import com.yahoo.vespa.indexinglanguage.ExpressionConverter; /** * @author Simon Thoresen Hult */ public abstract class CompositeExpression extends Expression { + @Override + public abstract CompositeExpression convertChildren(ExpressionConverter converter); + protected CompositeExpression(DataType inputType) { super(inputType); } diff --git a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/Expression.java b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/Expression.java index bf8201ee7ee..f498b871096 100644 --- a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/Expression.java +++ b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/Expression.java @@ -35,6 +35,12 @@ public abstract class Expression extends Selectable { this.inputType = inputType; } + /** + * Returns an expression where the children of this has been converted using the given converter. + * This default implementation returns this as it has no children. + */ + public Expression convertChildren(ExpressionConverter converter) { return this; } + /** Sets the document type and field the statement this expression is part of will write to */ public void setStatementOutput(DocumentType documentType, Field field) {} diff --git a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/ExpressionList.java b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/ExpressionList.java index e2ff1de7126..57de66f80a0 100644 --- a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/ExpressionList.java +++ b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/ExpressionList.java @@ -4,6 +4,7 @@ package com.yahoo.vespa.indexinglanguage.expressions; import com.yahoo.document.DataType; import com.yahoo.document.DocumentType; import com.yahoo.document.Field; +import com.yahoo.vespa.indexinglanguage.ExpressionConverter; import com.yahoo.vespa.objects.ObjectOperation; import com.yahoo.vespa.objects.ObjectPredicate; @@ -11,6 +12,7 @@ import java.util.Collections; import java.util.Iterator; import java.util.LinkedList; import java.util.List; +import java.util.Objects; /** * @author Simon Thoresen Hult @@ -26,6 +28,10 @@ public abstract class ExpressionList<T extends Expression> extends CompositeExpr } } + protected List<Expression> convertChildList(ExpressionConverter converter) { + return asList().stream().map(converter::convert).filter(Objects::nonNull).toList(); + } + @Override public void setStatementOutput(DocumentType documentType, Field field) { for (Expression expression : expressions) diff --git a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/ForEachExpression.java b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/ForEachExpression.java index 0f3a445bcb9..3053a391823 100644 --- a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/ForEachExpression.java +++ b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/ForEachExpression.java @@ -6,6 +6,7 @@ import com.yahoo.document.datatypes.Array; import com.yahoo.document.datatypes.FieldValue; import com.yahoo.document.datatypes.Struct; import com.yahoo.document.datatypes.WeightedSet; +import com.yahoo.vespa.indexinglanguage.ExpressionConverter; import com.yahoo.vespa.indexinglanguage.FieldValueConverter; import com.yahoo.vespa.objects.ObjectOperation; import com.yahoo.vespa.objects.ObjectPredicate; @@ -27,6 +28,11 @@ public final class ForEachExpression extends CompositeExpression { } @Override + public ForEachExpression convertChildren(ExpressionConverter converter) { + return new ForEachExpression(converter.convert(exp)); + } + + @Override public void setStatementOutput(DocumentType documentType, Field field) { exp.setStatementOutput(documentType, field); } diff --git a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/GuardExpression.java b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/GuardExpression.java index da7cfcdcaee..38a05c3056c 100644 --- a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/GuardExpression.java +++ b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/GuardExpression.java @@ -4,6 +4,7 @@ package com.yahoo.vespa.indexinglanguage.expressions; import com.yahoo.document.DataType; import com.yahoo.document.DocumentType; import com.yahoo.document.Field; +import com.yahoo.vespa.indexinglanguage.ExpressionConverter; import com.yahoo.vespa.indexinglanguage.ExpressionVisitor; import com.yahoo.vespa.indexinglanguage.UpdateAdapter; import com.yahoo.vespa.objects.ObjectOperation; @@ -28,6 +29,11 @@ public final class GuardExpression extends CompositeExpression { } @Override + public GuardExpression convertChildren(ExpressionConverter converter) { + return new GuardExpression(converter.convert(exp)); + } + + @Override public void setStatementOutput(DocumentType documentType, Field field) { exp.setStatementOutput(documentType, field); } diff --git a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/IfThenExpression.java b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/IfThenExpression.java index 8a29c8e8645..f05795aa234 100644 --- a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/IfThenExpression.java +++ b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/IfThenExpression.java @@ -6,6 +6,7 @@ import com.yahoo.document.DocumentType; import com.yahoo.document.Field; import com.yahoo.document.datatypes.FieldValue; import com.yahoo.document.datatypes.NumericFieldValue; +import com.yahoo.vespa.indexinglanguage.ExpressionConverter; import com.yahoo.vespa.objects.ObjectOperation; import com.yahoo.vespa.objects.ObjectPredicate; @@ -57,6 +58,15 @@ public final class IfThenExpression extends CompositeExpression { } @Override + public IfThenExpression convertChildren(ExpressionConverter converter) { + return new IfThenExpression(converter.branch().convert(lhs), + cmp, + converter.branch().convert(rhs), + converter.branch().convert(ifTrue), + converter.branch().convert(ifFalse)); + } + + @Override public void setStatementOutput(DocumentType documentType, Field field) { lhs.setStatementOutput(documentType, field); rhs.setStatementOutput(documentType, field); diff --git a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/ParenthesisExpression.java b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/ParenthesisExpression.java index 60b059f3ef1..6e476f5f7e4 100644 --- a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/ParenthesisExpression.java +++ b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/ParenthesisExpression.java @@ -4,6 +4,7 @@ package com.yahoo.vespa.indexinglanguage.expressions; import com.yahoo.document.DataType; import com.yahoo.document.DocumentType; import com.yahoo.document.Field; +import com.yahoo.vespa.indexinglanguage.ExpressionConverter; import com.yahoo.vespa.objects.ObjectOperation; import com.yahoo.vespa.objects.ObjectPredicate; @@ -24,6 +25,11 @@ public class ParenthesisExpression extends CompositeExpression { } @Override + public ParenthesisExpression convertChildren(ExpressionConverter converter) { + return new ParenthesisExpression(converter.convert(innerExp)); + } + + @Override public void setStatementOutput(DocumentType documentType, Field field) { innerExp.setStatementOutput(documentType, field); } diff --git a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/ScriptExpression.java b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/ScriptExpression.java index f0c37960a99..1a640c9924e 100644 --- a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/ScriptExpression.java +++ b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/ScriptExpression.java @@ -6,6 +6,7 @@ import com.yahoo.document.datatypes.FieldValue; import com.yahoo.language.Linguistics; import com.yahoo.language.process.Embedder; import com.yahoo.language.simple.SimpleLinguistics; +import com.yahoo.vespa.indexinglanguage.ExpressionConverter; import com.yahoo.vespa.indexinglanguage.ScriptParser; import com.yahoo.vespa.indexinglanguage.ScriptParserContext; import com.yahoo.vespa.indexinglanguage.parser.IndexingInput; @@ -17,6 +18,7 @@ import java.util.Collections; import java.util.Iterator; import java.util.List; import java.util.Map; +import java.util.Objects; /** * @author Simon Thoresen Hult @@ -36,6 +38,14 @@ public final class ScriptExpression extends ExpressionList<StatementExpression> } @Override + public ScriptExpression convertChildren(ExpressionConverter converter) { + return new ScriptExpression(asList().stream() + .map(child -> (StatementExpression)converter.branch().convert(child)) + .filter(Objects::nonNull) + .toList()); + } + + @Override protected void doExecute(ExecutionContext context) { FieldValue input = context.getValue(); for (StatementExpression statement : this) { diff --git a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/SelectInputExpression.java b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/SelectInputExpression.java index 212b60525f9..bb8111f358e 100644 --- a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/SelectInputExpression.java +++ b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/SelectInputExpression.java @@ -6,10 +6,12 @@ import com.yahoo.document.DataType; import com.yahoo.document.DocumentType; import com.yahoo.document.Field; import com.yahoo.document.datatypes.FieldValue; +import com.yahoo.vespa.indexinglanguage.ExpressionConverter; import com.yahoo.vespa.objects.ObjectOperation; import com.yahoo.vespa.objects.ObjectPredicate; import java.util.Arrays; +import java.util.Collection; import java.util.Collections; import java.util.List; @@ -32,6 +34,13 @@ public final class SelectInputExpression extends CompositeExpression { } @Override + public SelectInputExpression convertChildren(ExpressionConverter converter) { + return new SelectInputExpression(cases.stream() + .map(c -> new Pair<>(c.getFirst(), converter.branch().convert(c.getSecond()))) + .toList()); + } + + @Override public void setStatementOutput(DocumentType documentType, Field field) { for (var casePair : cases) casePair.getSecond().setStatementOutput(documentType, field); diff --git a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/StatementExpression.java b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/StatementExpression.java index 8516ddb883d..75f206ef47d 100644 --- a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/StatementExpression.java +++ b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/StatementExpression.java @@ -5,6 +5,7 @@ import com.yahoo.document.DataType; import com.yahoo.language.Linguistics; import com.yahoo.language.process.Embedder; import com.yahoo.language.simple.SimpleLinguistics; +import com.yahoo.vespa.indexinglanguage.ExpressionConverter; import com.yahoo.vespa.indexinglanguage.ScriptParser; import com.yahoo.vespa.indexinglanguage.ScriptParserContext; import com.yahoo.vespa.indexinglanguage.parser.IndexingInput; @@ -15,6 +16,7 @@ import java.util.Iterator; import java.util.LinkedList; import java.util.List; import java.util.Map; +import java.util.Objects; import java.util.stream.Collectors; /** @@ -45,6 +47,14 @@ public final class StatementExpression extends ExpressionList<Expression> { public List<String> getInputFields() { return inputFields; } @Override + public StatementExpression convertChildren(ExpressionConverter converter) { + return new StatementExpression(asList().stream() + .map(child -> converter.convert(child)) + .filter(Objects::nonNull) + .toList()); + } + + @Override protected void doExecute(ExecutionContext context) { for (Expression expression : this) { context.execute(expression); diff --git a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/SwitchExpression.java b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/SwitchExpression.java index 86913d8c1ba..c7cf7066483 100644 --- a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/SwitchExpression.java +++ b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/SwitchExpression.java @@ -7,6 +7,7 @@ import com.yahoo.document.Field; import com.yahoo.document.datatypes.FieldValue; import com.yahoo.document.datatypes.StringFieldValue; import com.yahoo.text.StringUtilities; +import com.yahoo.vespa.indexinglanguage.ExpressionConverter; import com.yahoo.vespa.objects.ObjectOperation; import com.yahoo.vespa.objects.ObjectPredicate; @@ -32,6 +33,17 @@ public final class SwitchExpression extends CompositeExpression { this.cases.putAll(cases); } + @Override + public SwitchExpression convertChildren(ExpressionConverter converter) { + var convertedCases = new LinkedHashMap<String, Expression>(); + for (var entry : cases.entrySet()) { + var converted = converter.branch().convert(entry.getValue()); + if (converted != null) + convertedCases.put(entry.getKey(), converted); + } + return new SwitchExpression(convertedCases, converter.branch().convert(defaultExp)); + } + public boolean isEmpty() { return defaultExp == null && cases.isEmpty(); } diff --git a/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/ExpressionConverterTestCase.java b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/ExpressionConverterTestCase.java index f1e1be0ae41..8aeaa084e1b 100644 --- a/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/ExpressionConverterTestCase.java +++ b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/ExpressionConverterTestCase.java @@ -2,7 +2,6 @@ package com.yahoo.vespa.indexinglanguage; import com.yahoo.collections.Pair; -import com.yahoo.document.DataType; import com.yahoo.document.datatypes.IntegerFieldValue; import com.yahoo.language.simple.SimpleLinguistics; import com.yahoo.vespa.indexinglanguage.expressions.ArithmeticExpression; @@ -11,9 +10,7 @@ import com.yahoo.vespa.indexinglanguage.expressions.Base64DecodeExpression; import com.yahoo.vespa.indexinglanguage.expressions.Base64EncodeExpression; import com.yahoo.vespa.indexinglanguage.expressions.CatExpression; import com.yahoo.vespa.indexinglanguage.expressions.ClearStateExpression; -import com.yahoo.vespa.indexinglanguage.expressions.CompositeExpression; import com.yahoo.vespa.indexinglanguage.expressions.EchoExpression; -import com.yahoo.vespa.indexinglanguage.expressions.ExecutionContext; import com.yahoo.vespa.indexinglanguage.expressions.Expression; import com.yahoo.vespa.indexinglanguage.expressions.ForEachExpression; import com.yahoo.vespa.indexinglanguage.expressions.GetFieldExpression; @@ -54,7 +51,6 @@ import com.yahoo.vespa.indexinglanguage.expressions.ToStringExpression; import com.yahoo.vespa.indexinglanguage.expressions.ToWsetExpression; import com.yahoo.vespa.indexinglanguage.expressions.TokenizeExpression; import com.yahoo.vespa.indexinglanguage.expressions.TrimExpression; -import com.yahoo.vespa.indexinglanguage.expressions.VerificationContext; import com.yahoo.vespa.indexinglanguage.expressions.ZCurveExpression; import com.yahoo.vespa.indexinglanguage.linguistics.AnnotatorConfig; import org.junit.Test; @@ -73,7 +69,6 @@ import static org.junit.Assert.fail; */ public class ExpressionConverterTestCase { - @SuppressWarnings("unchecked") @Test public void requireThatAllExpressionTypesCanBeTraversed() { assertConvertable(new ArithmeticExpression(new InputExpression("foo"), ArithmeticExpression.Operator.ADD, @@ -167,16 +162,6 @@ public class ExpressionConverterTestCase { } @Test - public void requireThatUnknownCompositeThrows() { - try { - new MyTraverser().convert(new MyComposite()); - fail(); - } catch (UnsupportedOperationException e) { - assertEquals(NoSuchMethodException.class, e.getCause().getClass()); - } - } - - @Test public void requireThatConversionExceptionCanBeThrown() { final RuntimeException expectedCause = new RuntimeException(); try { @@ -254,24 +239,4 @@ public class ExpressionConverterTestCase { } } - private static class MyComposite extends CompositeExpression { - - MyComposite() { - super(null); - } - @Override - protected void doExecute(ExecutionContext context) { - - } - - @Override - protected void doVerify(VerificationContext context) { - - } - - @Override - public DataType createdOutputType() { - return null; - } - } } diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/cores/CoreDumpMetadata.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/cores/CoreDumpMetadata.java index 7367a254b4a..eceb83a6cba 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/cores/CoreDumpMetadata.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/cores/CoreDumpMetadata.java @@ -12,6 +12,9 @@ import java.util.Optional; * @author hakonhall */ public class CoreDumpMetadata { + public enum Type { CORE_DUMP, JVM_HEAP, OOM } + + private Type type; private String binPath; private List<String> backtrace; private List<String> backtraceAllThreads; @@ -24,30 +27,33 @@ public class CoreDumpMetadata { public CoreDumpMetadata() {} - public Optional<String> binPath() { return Optional.ofNullable(binPath); }; - public Optional<List<String>> backtrace() { return Optional.ofNullable(backtrace); }; - public Optional<List<String>> backtraceAllThreads() { return Optional.ofNullable(backtraceAllThreads); }; - public Optional<Path> coredumpPath() { return Optional.ofNullable(coreDumpPath); }; + public Optional<Type> type() { return Optional.ofNullable(type); } + public Optional<String> binPath() { return Optional.ofNullable(binPath); } + public Optional<List<String>> backtrace() { return Optional.ofNullable(backtrace); } + public Optional<List<String>> backtraceAllThreads() { return Optional.ofNullable(backtraceAllThreads); } + public Optional<Path> coredumpPath() { return Optional.ofNullable(coreDumpPath); } public Optional<String> decryptionToken() { return Optional.ofNullable(decryptionToken); } - public Optional<String> kernelVersion() { return Optional.ofNullable(kernelVersion); }; - public Optional<String> cpuMicrocodeVersion() { return Optional.ofNullable(cpuMicrocodeVersion); }; - public Optional<DockerImage> dockerImage() { return Optional.ofNullable(dockerImage); }; - public Optional<String> vespaVersion() { return Optional.ofNullable(vespaVersion); }; + public Optional<String> kernelVersion() { return Optional.ofNullable(kernelVersion); } + public Optional<String> cpuMicrocodeVersion() { return Optional.ofNullable(cpuMicrocodeVersion); } + public Optional<DockerImage> dockerImage() { return Optional.ofNullable(dockerImage); } + public Optional<String> vespaVersion() { return Optional.ofNullable(vespaVersion); } - public CoreDumpMetadata setBinPath(String binPath) { this.binPath = binPath; return this; }; - public CoreDumpMetadata setBacktrace(List<String> backtrace) { this.backtrace = backtrace; return this; }; - public CoreDumpMetadata setBacktraceAllThreads(List<String> backtraceAllThreads) { this.backtraceAllThreads = backtraceAllThreads; return this; }; - public CoreDumpMetadata setCoreDumpPath(Path coreDumpPath) { this.coreDumpPath = coreDumpPath; return this; }; + public CoreDumpMetadata setType(Type type) { this.type = type; return this; } + public CoreDumpMetadata setBinPath(String binPath) { this.binPath = binPath; return this; } + public CoreDumpMetadata setBacktrace(List<String> backtrace) { this.backtrace = backtrace; return this; } + public CoreDumpMetadata setBacktraceAllThreads(List<String> backtraceAllThreads) { this.backtraceAllThreads = backtraceAllThreads; return this; } + public CoreDumpMetadata setCoreDumpPath(Path coreDumpPath) { this.coreDumpPath = coreDumpPath; return this; } public CoreDumpMetadata setDecryptionToken(String decryptionToken) { this.decryptionToken = decryptionToken; return this; } - public CoreDumpMetadata setKernelVersion(String kernelVersion) { this.kernelVersion = kernelVersion; return this; }; - public CoreDumpMetadata setCpuMicrocodeVersion(String cpuMicrocodeVersion) { this.cpuMicrocodeVersion = cpuMicrocodeVersion; return this; }; - public CoreDumpMetadata setDockerImage(DockerImage dockerImage) { this.dockerImage = dockerImage; return this; }; - public CoreDumpMetadata setVespaVersion(String vespaVersion) { this.vespaVersion = vespaVersion; return this; }; + public CoreDumpMetadata setKernelVersion(String kernelVersion) { this.kernelVersion = kernelVersion; return this; } + public CoreDumpMetadata setCpuMicrocodeVersion(String cpuMicrocodeVersion) { this.cpuMicrocodeVersion = cpuMicrocodeVersion; return this; } + public CoreDumpMetadata setDockerImage(DockerImage dockerImage) { this.dockerImage = dockerImage; return this; } + public CoreDumpMetadata setVespaVersion(String vespaVersion) { this.vespaVersion = vespaVersion; return this; } @Override public String toString() { return "CoreDumpMetadata{" + - "binPath=" + binPath + + "type=" + type + + ", binPath=" + binPath + ", backtrace=" + backtrace + ", backtraceAllThreads=" + backtraceAllThreads + ", coreDumpPath=" + coreDumpPath + @@ -64,7 +70,8 @@ public class CoreDumpMetadata { if (this == o) return true; if (o == null || getClass() != o.getClass()) return false; CoreDumpMetadata metadata = (CoreDumpMetadata) o; - return Objects.equals(binPath, metadata.binPath) && + return type == metadata.type && + Objects.equals(binPath, metadata.binPath) && Objects.equals(backtrace, metadata.backtrace) && Objects.equals(backtraceAllThreads, metadata.backtraceAllThreads) && Objects.equals(coreDumpPath, metadata.coreDumpPath) && @@ -77,7 +84,7 @@ public class CoreDumpMetadata { @Override public int hashCode() { - return Objects.hash(binPath, backtrace, backtraceAllThreads, coreDumpPath, decryptionToken, kernelVersion, + return Objects.hash(type, binPath, backtrace, backtraceAllThreads, coreDumpPath, decryptionToken, kernelVersion, cpuMicrocodeVersion, dockerImage, vespaVersion); } } diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/cores/bindings/ReportCoreDumpRequest.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/cores/bindings/ReportCoreDumpRequest.java index 27cf28b8e1e..cf717aff787 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/cores/bindings/ReportCoreDumpRequest.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/cores/bindings/ReportCoreDumpRequest.java @@ -31,6 +31,7 @@ public class ReportCoreDumpRequest { public List<String> backtrace; public List<String> backtrace_all_threads; + public String type; public String bin_path; public String coredump_path; public String cpu_microcode_version; @@ -44,6 +45,7 @@ public class ReportCoreDumpRequest { /** Fill this from metadata and return this. */ @JsonIgnore public ReportCoreDumpRequest fillFrom(CoreDumpMetadata metadata) { + metadata.type().ifPresent(type -> this.type = type.name()); metadata.binPath().ifPresent(binPath -> this.bin_path = binPath); metadata.backtrace().ifPresent(backtrace -> this.backtrace = List.copyOf(backtrace)); metadata.backtraceAllThreads().ifPresent(backtraceAllThreads -> this.backtrace_all_threads = List.copyOf(backtraceAllThreads)); diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/coredump/CoreCollector.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/coredump/CoreCollector.java index 28773767d24..4761b6da421 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/coredump/CoreCollector.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/coredump/CoreCollector.java @@ -99,14 +99,15 @@ public class CoreCollector { var metadata = new CoreDumpMetadata(); if (JAVA_HEAP_DUMP_PATTERN.matcher(coredumpPath.getFileName().toString()).find()) { - metadata.setBinPath("java") + metadata.setType(CoreDumpMetadata.Type.JVM_HEAP) + .setBinPath("java") .setBacktrace(List.of("Heap dump, no backtrace available")); return metadata; } try { String binPath = readBinPath(context, coredumpPath); - metadata.setBinPath(binPath); + metadata.setType(CoreDumpMetadata.Type.CORE_DUMP).setBinPath(binPath); if (Path.of(binPath).getFileName().toString().equals("java")) { metadata.setBacktraceAllThreads(readJstack(context, coredumpPath, binPath)); diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/coredump/CoreCollectorTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/coredump/CoreCollectorTest.java index 4fa18c71da0..2ca9e084d96 100644 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/coredump/CoreCollectorTest.java +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/coredump/CoreCollectorTest.java @@ -143,6 +143,7 @@ public class CoreCollectorTest { String.join("\n", GDB_BACKTRACE)); var expected = new CoreDumpMetadata().setBinPath(TEST_BIN_PATH) + .setType(CoreDumpMetadata.Type.CORE_DUMP) .setBacktrace(GDB_BACKTRACE) .setBacktraceAllThreads(GDB_BACKTRACE); assertEquals(expected, coreCollector.collect(context, TEST_CORE_PATH)); @@ -156,7 +157,7 @@ public class CoreCollectorTest { mockExec(new String[]{GDB_PATH_RHEL8 + " -n -ex set print frame-arguments none -ex bt -batch /usr/bin/program /tmp/core.1234"}, "", "Failure"); - var expected = new CoreDumpMetadata().setBinPath(TEST_BIN_PATH); + var expected = new CoreDumpMetadata().setBinPath(TEST_BIN_PATH).setType(CoreDumpMetadata.Type.CORE_DUMP); assertEquals(expected, coreCollector.collect(context, TEST_CORE_PATH)); } @@ -174,6 +175,7 @@ public class CoreCollectorTest { jstack); var expected = new CoreDumpMetadata().setBinPath(jdkPath) + .setType(CoreDumpMetadata.Type.CORE_DUMP) .setBacktraceAllThreads(List.of(jstack)); assertEquals(expected, coreCollector.collect(context, TEST_CORE_PATH)); } @@ -181,6 +183,7 @@ public class CoreCollectorTest { @Test void metadata_for_java_heap_dump() { var expected = new CoreDumpMetadata().setBinPath("java") + .setType(CoreDumpMetadata.Type.JVM_HEAP) .setBacktrace(List.of("Heap dump, no backtrace available")); assertEquals(expected, coreCollector.collect(context, context.paths().of("/dump_java_pid123.hprof"))); diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/MetricsReporter.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/MetricsReporter.java index 5af74214648..e6cfe8ca6b5 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/MetricsReporter.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/MetricsReporter.java @@ -176,6 +176,9 @@ public class MetricsReporter extends NodeRepositoryMaintainer { boolean converged = currentVersion.isPresent() && currentVersion.get().equals(wantedVersion); metric.set("wantToChangeVespaVersion", converged ? 0 : 1, context); + if (node.cloudAccount().isEnclave(nodeRepository().zone())) { + metric.set("hasWireguardKey", node.wireguardPubKey().isPresent() ? 1 : 0, context); + } } else { context = getContext(Map.of("state", node.state().name(), "host", node.hostname())); diff --git a/searchlib/src/tests/attribute/attribute_test.cpp b/searchlib/src/tests/attribute/attribute_test.cpp index 91666eb29fd..d2d3ccaad23 100644 --- a/searchlib/src/tests/attribute/attribute_test.cpp +++ b/searchlib/src/tests/attribute/attribute_test.cpp @@ -2334,6 +2334,10 @@ AttributeTest::test_paged_attribute(const vespalib::string& name, const vespalib size_t rounded_size = vespalib::round_up_to_page_size(1); size_t lid_mapping_size = 1200; size_t sv_maxlid = 1200; + if (rounded_size == 16_Ki) { + lid_mapping_size = 4200; + sv_maxlid = 1300; + } if (rounded_size == 64_Ki) { lid_mapping_size = 17000; sv_maxlid = 1500; diff --git a/vespalib/src/tests/datastore/free_list/free_list_test.cpp b/vespalib/src/tests/datastore/free_list/free_list_test.cpp index 11c8bba4fe5..ec14d0dd28c 100644 --- a/vespalib/src/tests/datastore/free_list/free_list_test.cpp +++ b/vespalib/src/tests/datastore/free_list/free_list_test.cpp @@ -8,7 +8,6 @@ using namespace vespalib::datastore; using MyEntryRef = EntryRefT<8, 4>; -constexpr uint32_t array_size = 6; struct FreeListTest : public testing::Test { diff --git a/vespalib/src/tests/signalhandler/CMakeLists.txt b/vespalib/src/tests/signalhandler/CMakeLists.txt index 4f78eb2e82d..88be14f994f 100644 --- a/vespalib/src/tests/signalhandler/CMakeLists.txt +++ b/vespalib/src/tests/signalhandler/CMakeLists.txt @@ -5,6 +5,11 @@ vespa_add_library(vespalib_signalhandler_test_my_shared_library TEST DEPENDS vespalib ) + +# Don't convert call to jump when returning a value from a function with +# a compatible stack. +set_source_files_properties(my_shared_library.cpp PROPERTIES COMPILE_OPTIONS "-fno-optimize-sibling-calls") + vespa_add_executable(vespalib_signalhandler_test_app TEST SOURCES signalhandler_test.cpp diff --git a/vespalib/src/vespa/vespalib/util/CMakeLists.txt b/vespalib/src/vespa/vespalib/util/CMakeLists.txt index 663b3b65638..8ee3957af32 100644 --- a/vespalib/src/vespa/vespalib/util/CMakeLists.txt +++ b/vespalib/src/vespa/vespalib/util/CMakeLists.txt @@ -100,3 +100,7 @@ vespa_add_library(vespalib_vespalib_util OBJECT zstdcompressor.cpp DEPENDS ) + +# Don't convert call to jump when returning a value from a function with +# a compatible stack. +set_source_files_properties(signalhandler.cpp PROPERTIES COMPILE_OPTIONS "-fno-optimize-sibling-calls") |