diff options
156 files changed, 1277 insertions, 726 deletions
diff --git a/build_settings.cmake b/build_settings.cmake index ac4e86c2e81..c96a626ad0b 100644 --- a/build_settings.cmake +++ b/build_settings.cmake @@ -54,7 +54,12 @@ if ("${CMAKE_CXX_COMPILER_ID}" STREQUAL "Clang" OR "${CMAKE_CXX_COMPILER_ID}" ST endif() else() set(CXX_SPECIFIC_WARN_OPTS "-Wsuggest-override -Wnon-virtual-dtor -Wformat-security") - set(VESPA_ATOMIC_LIB "atomic") + if(VESPA_OS_DISTRO_COMBINED STREQUAL "centos 8" OR + VESPA_OS_DISTRO_COMBINED STREQUAL "rhel 8.1") + set(VESPA_ATOMIC_LIB "") + else() + set(VESPA_ATOMIC_LIB "atomic") + endif() set(VESPA_GCC_LIB "gcc") set(VESPA_STDCXX_FS_LIB "stdc++fs") endif() @@ -79,7 +84,11 @@ if(CMAKE_HOST_SYSTEM_NAME STREQUAL "Darwin") set(CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS} -latomic -ldl") endif() else() - set(CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS} -Wl,--build-id -latomic -ldl -Wl,-E") + if(VESPA_ATOMIC_LIB STREQUAL "") + set(CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS} -Wl,--build-id -ldl -Wl,-E") + else() + set(CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS} -Wl,--build-id -latomic -ldl -Wl,-E") + endif() endif() SET(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -rdynamic" ) diff --git a/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java b/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java index c406fa80434..86360d482a6 100644 --- a/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java +++ b/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java @@ -72,6 +72,12 @@ public interface ModelContext { double defaultTermwiseLimit(); // TODO Revisit in May or June 2020 + double threadPoolSizeFactor(); + + // TODO Revisit in May or June 2020 + double queueSizeFactor(); + + // TODO Revisit in May or June 2020 double defaultSoftStartSeconds(); // TODO Revisit in May or June 2020 diff --git a/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java b/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java index 99225beba4f..d799af36c3b 100644 --- a/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java +++ b/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java @@ -43,6 +43,8 @@ public class TestProperties implements ModelContext.Properties { private double topKProbability = 1.0; private double defaultTermwiseLimit = 1.0; private double softStartSeconds = 0.0; + private double threadPoolSizeFactor = 0.0; + private double queueSizeFactor = 0.0; private Optional<EndpointCertificateSecrets> endpointCertificateSecrets = Optional.empty(); private AthenzDomain athenzDomain; @@ -65,6 +67,16 @@ public class TestProperties implements ModelContext.Properties { @Override public double defaultTermwiseLimit() { return defaultTermwiseLimit; } @Override + public double threadPoolSizeFactor() { + return threadPoolSizeFactor; + } + + @Override + public double queueSizeFactor() { + return queueSizeFactor; + } + + @Override public double defaultSoftStartSeconds() { return softStartSeconds; } @@ -86,7 +98,15 @@ public class TestProperties implements ModelContext.Properties { this.softStartSeconds = softStartSeconds; return this; } + public TestProperties setThreadPoolSizeFactor(double threadPoolSizeFactor) { + this.threadPoolSizeFactor = threadPoolSizeFactor; + return this; + } + public TestProperties setQueueSizeFactor(double queueSizeFactor) { + this.queueSizeFactor = queueSizeFactor; + return this; + } public TestProperties setApplicationId(ApplicationId applicationId) { this.applicationId = applicationId; return this; diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/MapEvaluationTypeContext.java b/config-model/src/main/java/com/yahoo/searchdefinition/MapEvaluationTypeContext.java index 6de7c985326..4011ce43841 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/MapEvaluationTypeContext.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/MapEvaluationTypeContext.java @@ -41,6 +41,9 @@ public class MapEvaluationTypeContext extends FunctionReferenceContext implement private final Map<Reference, TensorType> resolvedTypes = new HashMap<>(); + /** To avoid re-resolving diamond-shaped dependencies */ + private final Map<Reference, TensorType> globallyResolvedTypes; + /** For invocation loop detection */ private final Deque<Reference> currentResolutionCallStack; @@ -53,6 +56,7 @@ public class MapEvaluationTypeContext extends FunctionReferenceContext implement this.currentResolutionCallStack = new ArrayDeque<>(); this.queryFeaturesNotDeclared = new TreeSet<>(); tensorsAreUsed = false; + globallyResolvedTypes = new HashMap<>(); } private MapEvaluationTypeContext(Map<String, ExpressionFunction> functions, @@ -60,12 +64,14 @@ public class MapEvaluationTypeContext extends FunctionReferenceContext implement Map<Reference, TensorType> featureTypes, Deque<Reference> currentResolutionCallStack, SortedSet<Reference> queryFeaturesNotDeclared, - boolean tensorsAreUsed) { + boolean tensorsAreUsed, + Map<Reference, TensorType> globallyResolvedTypes) { super(functions, bindings); this.featureTypes.putAll(featureTypes); this.currentResolutionCallStack = currentResolutionCallStack; this.queryFeaturesNotDeclared = queryFeaturesNotDeclared; this.tensorsAreUsed = tensorsAreUsed; + this.globallyResolvedTypes = globallyResolvedTypes; } public void setType(Reference reference, TensorType type) { @@ -82,11 +88,25 @@ public class MapEvaluationTypeContext extends FunctionReferenceContext implement resolvedTypes.clear(); } - @Override + private boolean referenceCanBeResolvedGlobally(Reference reference) { + Optional<ExpressionFunction> function = functionInvocation(reference); + return function.isPresent() && function.get().arguments().size() == 0; + // are there other cases we would like to resolve globally? + } + + @Override public TensorType getType(Reference reference) { // computeIfAbsent without concurrent modification due to resolve adding more resolved entries: + + boolean canBeResolvedGlobally = referenceCanBeResolvedGlobally(reference); + TensorType resolvedType = resolvedTypes.get(reference); - if (resolvedType != null) return resolvedType; + if (resolvedType == null && canBeResolvedGlobally) { + resolvedType = globallyResolvedTypes.get(reference); + } + if (resolvedType != null) { + return resolvedType; + } resolvedType = resolveType(reference); if (resolvedType == null) @@ -94,6 +114,11 @@ public class MapEvaluationTypeContext extends FunctionReferenceContext implement resolvedTypes.put(reference, resolvedType); if (resolvedType.rank() > 0) tensorsAreUsed = true; + + if (canBeResolvedGlobally) { + globallyResolvedTypes.put(reference, resolvedType); + } + return resolvedType; } @@ -254,7 +279,8 @@ public class MapEvaluationTypeContext extends FunctionReferenceContext implement featureTypes, currentResolutionCallStack, queryFeaturesNotDeclared, - tensorsAreUsed); + tensorsAreUsed, + globallyResolvedTypes); } } diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/RankProfile.java b/config-model/src/main/java/com/yahoo/searchdefinition/RankProfile.java index 23eb814de81..ea126123a25 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/RankProfile.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/RankProfile.java @@ -680,11 +680,12 @@ public class RankProfile implements Cloneable { Map<String, RankingExpressionFunction> inlineFunctions = compileFunctions(this::getInlineFunctions, queryProfiles, featureTypes, importedModels, Collections.emptyMap(), expressionTransforms); + firstPhaseRanking = compile(this.getFirstPhaseRanking(), queryProfiles, featureTypes, importedModels, getConstants(), inlineFunctions, expressionTransforms); + secondPhaseRanking = compile(this.getSecondPhaseRanking(), queryProfiles, featureTypes, importedModels, getConstants(), inlineFunctions, expressionTransforms); + // Function compiling second pass: compile all functions and insert previously compiled inline functions functions = compileFunctions(this::getFunctions, queryProfiles, featureTypes, importedModels, inlineFunctions, expressionTransforms); - firstPhaseRanking = compile(this.getFirstPhaseRanking(), queryProfiles, featureTypes, importedModels, getConstants(), inlineFunctions, expressionTransforms); - secondPhaseRanking = compile(this.getSecondPhaseRanking(), queryProfiles, featureTypes, importedModels, getConstants(), inlineFunctions, expressionTransforms); } private void checkNameCollisions(Map<String, RankingExpressionFunction> functions, Map<String, Value> constants) { diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/derived/RawRankProfile.java b/config-model/src/main/java/com/yahoo/searchdefinition/derived/RawRankProfile.java index 1a22b98fd9f..c3c10139684 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/derived/RawRankProfile.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/derived/RawRankProfile.java @@ -200,9 +200,7 @@ public class RawRankProfile implements RankProfilesConfig.Producer { if (functions.isEmpty()) return; List<ExpressionFunction> functionExpressions = functions.values().stream().map(f -> f.function()).collect(Collectors.toList()); - Map<String, String> functionProperties = new LinkedHashMap<>(); - functionProperties.putAll(deriveFunctionProperties(functions, functionExpressions)); if (firstPhaseRanking != null) { functionProperties.putAll(firstPhaseRanking.getRankProperties(functionExpressions)); @@ -211,20 +209,30 @@ public class RawRankProfile implements RankProfilesConfig.Producer { functionProperties.putAll(secondPhaseRanking.getRankProperties(functionExpressions)); } + SerializationContext context = new SerializationContext(functionExpressions, null, functionProperties); + replaceFunctionSummaryFeatures(context); + + // First phase, second phase and summary features should add all required functions to the context. + // However, we need to add any functions not referenced in those anyway for model-evaluation. + deriveFunctionProperties(functions, functionExpressions, functionProperties); + for (Map.Entry<String, String> e : functionProperties.entrySet()) { rankProperties.add(new RankProfile.RankProperty(e.getKey(), e.getValue())); } - SerializationContext context = new SerializationContext(functionExpressions, null, functionProperties); - replaceFunctionSummaryFeatures(context); } - private Map<String, String> deriveFunctionProperties(Map<String, RankProfile.RankingExpressionFunction> functions, - List<ExpressionFunction> functionExpressions) { - SerializationContext context = new SerializationContext(functionExpressions); + private void deriveFunctionProperties(Map<String, RankProfile.RankingExpressionFunction> functions, + List<ExpressionFunction> functionExpressions, + Map<String, String> functionProperties) { + SerializationContext context = new SerializationContext(functionExpressions, null, functionProperties); for (Map.Entry<String, RankProfile.RankingExpressionFunction> e : functions.entrySet()) { + String propertyName = RankingExpression.propertyName(e.getKey()); + if (context.serializedFunctions().containsKey(propertyName)) { + continue; + } String expressionString = e.getValue().function().getBody().getRoot().toString(new StringBuilder(), context, null, null).toString(); - context.addFunctionSerialization(RankingExpression.propertyName(e.getKey()), expressionString); + context.addFunctionSerialization(RankingExpression.propertyName(e.getKey()), expressionString); for (Map.Entry<String, TensorType> argumentType : e.getValue().function().argumentTypes().entrySet()) context.addArgumentTypeSerialization(e.getKey(), argumentType.getKey(), argumentType.getValue()); if (e.getValue().function().returnType().isPresent()) @@ -232,7 +240,7 @@ public class RawRankProfile implements RankProfilesConfig.Producer { // else if (e.getValue().function().arguments().isEmpty()) TODO: Enable this check when we resolve all types // throw new IllegalStateException("Type of function '" + e.getKey() + "' is not resolved"); } - return context.serializedFunctions(); + functionProperties.putAll(context.serializedFunctions()); } private void replaceFunctionSummaryFeatures(SerializationContext context) { @@ -241,9 +249,11 @@ public class RawRankProfile implements RankProfilesConfig.Producer { for (Iterator<ReferenceNode> i = summaryFeatures.iterator(); i.hasNext(); ) { ReferenceNode referenceNode = i.next(); // Is the feature a function? - if (context.getFunction(referenceNode.getName()) != null) { - context.addFunctionSerialization(RankingExpression.propertyName(referenceNode.getName()), - referenceNode.toString(new StringBuilder(), context, null, null).toString()); + ExpressionFunction function = context.getFunction(referenceNode.getName()); + if (function != null) { + String propertyName = RankingExpression.propertyName(referenceNode.getName()); + String expressionString = function.getBody().getRoot().toString(new StringBuilder(), context, null, null).toString(); + context.addFunctionSerialization(propertyName, expressionString); ReferenceNode newReferenceNode = new ReferenceNode("rankingExpression(" + referenceNode.getName() + ")", referenceNode.getArguments().expressions(), referenceNode.getOutput()); functionSummaryFeatures.put(referenceNode.getName(), newReferenceNode); i.remove(); // Will add the expanded one in next block diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/ApplicationContainer.java b/config-model/src/main/java/com/yahoo/vespa/model/container/ApplicationContainer.java index e59baf88422..29bb578a67b 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/ApplicationContainer.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/ApplicationContainer.java @@ -5,6 +5,7 @@ import com.yahoo.config.model.api.container.ContainerServiceType; import com.yahoo.config.model.producer.AbstractConfigProducer; import com.yahoo.config.provision.Flavor; import com.yahoo.container.bundle.BundleInstantiationSpecification; +import com.yahoo.container.handler.ThreadpoolConfig; import com.yahoo.osgi.provider.model.ComponentModel; import com.yahoo.prelude.fastsearch.FS4ResourcePool; import com.yahoo.search.config.QrStartConfig; @@ -15,7 +16,10 @@ import com.yahoo.vespa.model.container.component.Component; * * @author gjoranv */ -public final class ApplicationContainer extends Container implements QrStartConfig.Producer { +public final class ApplicationContainer extends Container implements + QrStartConfig.Producer, + ThreadpoolConfig.Producer +{ private static final String defaultHostedJVMArgs = "-XX:+UseOSErrorReporting -XX:+SuppressFatalErrorMessage"; @@ -73,4 +77,16 @@ public final class ApplicationContainer extends Container implements QrStartConf return (parent instanceof ContainerCluster) && (((ContainerCluster)parent).getDocproc() != null); } + @Override + public void getConfig(ThreadpoolConfig.Builder builder) { + if (! (parent instanceof ContainerCluster)) return; + if ((getHostResource() == null) || getHostResource().getFlavor().isEmpty()) return; + ContainerCluster containerCluster = (ContainerCluster) parent; + if (containerCluster.getThreadPoolSizeFactor() <= 0.0) return; + + NodeFlavorTuning flavorTuning = new NodeFlavorTuning(getHostResource().getFlavor().get()) + .setThreadPoolSizeFactor(containerCluster.getThreadPoolSizeFactor()) + .setQueueSizeFactor(containerCluster.getQueueSizeFactor()); + flavorTuning.getConfig(builder); + } } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerCluster.java index 833f6688fbc..b35b7562704 100755 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerCluster.java @@ -160,12 +160,18 @@ public abstract class ContainerCluster<CONTAINER extends Container> private String jvmGCOptions = null; private String environmentVars = null; + private final double threadPoolSizeFactor; + private final double queueSizeFactor; + public ContainerCluster(AbstractConfigProducer<?> parent, String subId, String name, DeployState deployState) { super(parent, subId); this.name = name; this.isHostedVespa = stateIsHosted(deployState); this.zone = (deployState != null) ? deployState.zone() : Zone.defaultZone(); + this.threadPoolSizeFactor = deployState.getProperties().threadPoolSizeFactor(); + this.queueSizeFactor = deployState.getProperties().queueSizeFactor(); + componentGroup = new ComponentGroup<>(this, "component"); addComponent(new StatisticsComponent()); @@ -186,6 +192,14 @@ public abstract class ContainerCluster<CONTAINER extends Container> addJaxProviders(); } + public double getThreadPoolSizeFactor() { + return threadPoolSizeFactor; + } + + public double getQueueSizeFactor() { + return queueSizeFactor; + } + public void setZone(Zone zone) { this.zone = zone; } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/NodeFlavorTuning.java b/config-model/src/main/java/com/yahoo/vespa/model/container/NodeFlavorTuning.java index 67938b36fd9..f9b50d0e641 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/NodeFlavorTuning.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/NodeFlavorTuning.java @@ -2,6 +2,7 @@ package com.yahoo.vespa.model.container; import com.yahoo.config.provision.Flavor; +import com.yahoo.container.handler.ThreadpoolConfig; import com.yahoo.search.config.QrStartConfig; /** @@ -9,10 +10,26 @@ import com.yahoo.search.config.QrStartConfig; * * @author balder */ -public class NodeFlavorTuning implements QrStartConfig.Producer { +public class NodeFlavorTuning implements + QrStartConfig.Producer, + ThreadpoolConfig.Producer +{ private final Flavor flavor; + public NodeFlavorTuning setThreadPoolSizeFactor(double threadPoolSizeFactor) { + this.threadPoolSizeFactor = threadPoolSizeFactor; + return this; + } + + public NodeFlavorTuning setQueueSizeFactor(double queueSizeFactor) { + this.queueSizeFactor = queueSizeFactor; + return this; + } + + private double threadPoolSizeFactor = 8.0; + private double queueSizeFactor = 8.0; + NodeFlavorTuning(Flavor flavor) { this.flavor = flavor; } @@ -22,4 +39,15 @@ public class NodeFlavorTuning implements QrStartConfig.Producer { builder.jvm.availableProcessors(Math.max(2, (int)Math.ceil(flavor.getMinCpuCores()))); } + @Override + public void getConfig(ThreadpoolConfig.Builder builder) { + // Controls max number of concurrent requests per container + int workerThreads = Math.max(2, (int)Math.ceil(flavor.getMinCpuCores() * threadPoolSizeFactor)); + builder.maxthreads(workerThreads); + + // This controls your burst handling capability. + // 0 => No extra burst handling beyond you max concurrent requests (maxthreads). + // N => N times max concurrent requests as a buffer for handling bursts + builder.queueSize((int)(workerThreads * queueSizeFactor)); + } } diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/RankingExpressionShadowingTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/RankingExpressionShadowingTestCase.java index 5c1134f928c..e4ca83640e9 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/RankingExpressionShadowingTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/RankingExpressionShadowingTestCase.java @@ -50,10 +50,10 @@ public class RankingExpressionShadowingTestCase extends SchemaTestCase { new QueryProfileRegistry(), new ImportedMlModels(), new AttributeFields(s)).configProperties(); - assertEquals("(rankingExpression(sin).rankingScript,x * x)", - testRankProperties.get(0).toString()); assertEquals("(rankingExpression(sin@).rankingScript,2 * 2)", - censorBindingHash(testRankProperties.get(1).toString())); + censorBindingHash(testRankProperties.get(0).toString())); + assertEquals("(rankingExpression(sin).rankingScript,x * x)", + testRankProperties.get(1).toString()); assertEquals("(vespa.rank.firstphase,rankingExpression(sin@))", censorBindingHash(testRankProperties.get(2).toString())); } @@ -94,27 +94,26 @@ public class RankingExpressionShadowingTestCase extends SchemaTestCase { new QueryProfileRegistry(), new ImportedMlModels(), new AttributeFields(s)).configProperties(); + assertEquals("(rankingExpression(tan@).rankingScript,2 * 2)", + censorBindingHash(testRankProperties.get(0).toString())); + assertEquals("(rankingExpression(cos@).rankingScript,rankingExpression(tan@))", + censorBindingHash(testRankProperties.get(1).toString())); + assertEquals("(rankingExpression(sin@).rankingScript,rankingExpression(cos@))", + censorBindingHash(testRankProperties.get(2).toString())); assertEquals("(rankingExpression(tan).rankingScript,x * x)", - testRankProperties.get(0).toString()); + testRankProperties.get(3).toString()); assertEquals("(rankingExpression(tan@).rankingScript,x * x)", - censorBindingHash(testRankProperties.get(1).toString())); - assertEquals("(rankingExpression(cos).rankingScript,rankingExpression(tan@))", - censorBindingHash(testRankProperties.get(2).toString())); - assertEquals("(rankingExpression(cos@).rankingScript,rankingExpression(tan@))", - censorBindingHash(testRankProperties.get(3).toString())); - assertEquals("(rankingExpression(sin).rankingScript,rankingExpression(cos@))", censorBindingHash(testRankProperties.get(4).toString())); - assertEquals("(rankingExpression(tan@).rankingScript,2 * 2)", + assertEquals("(rankingExpression(cos).rankingScript,rankingExpression(tan@))", censorBindingHash(testRankProperties.get(5).toString())); assertEquals("(rankingExpression(cos@).rankingScript,rankingExpression(tan@))", - censorBindingHash(testRankProperties.get(6).toString())); - assertEquals("(rankingExpression(sin@).rankingScript,rankingExpression(cos@))", + censorBindingHash(testRankProperties.get(6).toString())); + assertEquals("(rankingExpression(sin).rankingScript,rankingExpression(cos@))", censorBindingHash(testRankProperties.get(7).toString())); assertEquals("(vespa.rank.firstphase,rankingExpression(sin@))", censorBindingHash(testRankProperties.get(8).toString())); } - @Test public void testFunctionShadowingArguments() throws ParseException { RankProfileRegistry rankProfileRegistry = new RankProfileRegistry(); @@ -144,12 +143,12 @@ public class RankingExpressionShadowingTestCase extends SchemaTestCase { new QueryProfileRegistry(), new ImportedMlModels(), new AttributeFields(s)).configProperties(); - assertEquals("(rankingExpression(sin).rankingScript,x * x)", - testRankProperties.get(0).toString()); assertEquals("(rankingExpression(sin@).rankingScript,4.0 * 4.0)", - censorBindingHash(testRankProperties.get(1).toString())); + censorBindingHash(testRankProperties.get(0).toString())); assertEquals("(rankingExpression(sin@).rankingScript,cos(5.0) * cos(5.0))", - censorBindingHash(testRankProperties.get(2).toString())); + censorBindingHash(testRankProperties.get(1).toString())); + assertEquals("(rankingExpression(sin).rankingScript,x * x)", + testRankProperties.get(2).toString()); assertEquals("(vespa.rank.firstphase,rankingExpression(firstphase))", censorBindingHash(testRankProperties.get(3).toString())); assertEquals("(rankingExpression(firstphase).rankingScript,cos(rankingExpression(sin@)) + rankingExpression(sin@))", @@ -208,17 +207,17 @@ public class RankingExpressionShadowingTestCase extends SchemaTestCase { queryProfiles, new ImportedMlModels(), new AttributeFields(s)).configProperties(); - assertEquals("(rankingExpression(relu).rankingScript,max(1.0,x))", - testRankProperties.get(0).toString()); assertEquals("(rankingExpression(relu@).rankingScript,max(1.0,reduce(query(q) * constant(W_hidden), sum, input) + constant(b_input)))", - censorBindingHash(testRankProperties.get(1).toString())); + censorBindingHash(testRankProperties.get(0).toString())); assertEquals("(rankingExpression(hidden_layer).rankingScript,rankingExpression(relu@))", - censorBindingHash(testRankProperties.get(2).toString())); + censorBindingHash(testRankProperties.get(1).toString())); assertEquals("(rankingExpression(hidden_layer).type,tensor(x[]))", - censorBindingHash(testRankProperties.get(3).toString())); + censorBindingHash(testRankProperties.get(2).toString())); assertEquals("(rankingExpression(final_layer).rankingScript,sigmoid(reduce(rankingExpression(hidden_layer) * constant(W_final), sum, hidden) + constant(b_final)))", - testRankProperties.get(4).toString()); + testRankProperties.get(3).toString()); assertEquals("(rankingExpression(final_layer).type,tensor(x[]))", + testRankProperties.get(4).toString()); + assertEquals("(rankingExpression(relu).rankingScript,max(1.0,x))", testRankProperties.get(5).toString()); assertEquals("(vespa.rank.secondphase,rankingExpression(secondphase))", testRankProperties.get(6).toString()); diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankProfileSearchFixture.java b/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankProfileSearchFixture.java index 0cd6674751e..e6616ce0dd1 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankProfileSearchFixture.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankProfileSearchFixture.java @@ -41,6 +41,14 @@ class RankProfileSearchFixture { private Search search; private Map<String, RankProfile> compiledRankProfiles = new HashMap<>(); + public RankProfileRegistry getRankProfileRegistry() { + return rankProfileRegistry; + } + + public QueryProfileRegistry getQueryProfileRegistry() { + return queryProfileRegistry; + } + RankProfileSearchFixture(String rankProfiles) throws ParseException { this(MockApplicationPackage.createEmpty(), new QueryProfileRegistry(), rankProfiles); } diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionWithTensorFlowTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionWithTensorFlowTestCase.java index a64a964727c..680f2dd9659 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionWithTensorFlowTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionWithTensorFlowTestCase.java @@ -319,7 +319,7 @@ public class RankingExpressionWithTensorFlowTestCase { @Test public void testFunctionGeneration() { final String name = "mnist_saved"; - final String expression = "join(join(reduce(join(join(join(reduce(constant(" + name + "_dnn_hidden2_Const), sum, d2), imported_ml_function_" + name + "_dnn_hidden2_add, f(a,b)(a * b)), imported_ml_function_" + name + "_dnn_hidden2_add, f(a,b)(max(a,b))), constant(" + name + "_dnn_outputs_weights_read), f(a,b)(a * b)), sum, d2), constant(" + name + "_dnn_outputs_bias_read), f(a,b)(a + b)), tensor(d0[1])(1.0), f(a,b)(a * b))"; + final String expression = "join(reduce(join(join(join(reduce(constant(" + name + "_dnn_hidden2_Const), sum, d2), imported_ml_function_" + name + "_dnn_hidden2_add, f(a,b)(a * b)), imported_ml_function_" + name + "_dnn_hidden2_add, f(a,b)(max(a,b))), constant(" + name + "_dnn_outputs_weights_read), f(a,b)(a * b)), sum, d2), constant(" + name + "_dnn_outputs_bias_read), f(a,b)(a + b))"; final String functionExpression1 = "join(reduce(join(reduce(rename(input, (d0, d1), (d0, d4)), sum, d0), constant(" + name + "_dnn_hidden1_weights_read), f(a,b)(a * b)), sum, d4), constant(" + name + "_dnn_hidden1_bias_read), f(a,b)(a + b))"; final String functionExpression2 = "join(reduce(join(join(join(0.009999999776482582, imported_ml_function_" + name + "_dnn_hidden1_add, f(a,b)(a * b)), imported_ml_function_" + name + "_dnn_hidden1_add, f(a,b)(max(a,b))), constant(" + name + "_dnn_hidden2_weights_read), f(a,b)(a * b)), sum, d3), constant(" + name + "_dnn_hidden2_bias_read), f(a,b)(a + b))"; @@ -349,7 +349,7 @@ public class RankingExpressionWithTensorFlowTestCase { " rank-profile my_profile_child inherits my_profile {\n" + " }"; - final String expression = "join(join(reduce(join(join(join(reduce(constant(" + name + "_dnn_hidden2_Const), sum, d2), imported_ml_function_" + name + "_dnn_hidden2_add, f(a,b)(a * b)), imported_ml_function_" + name + "_dnn_hidden2_add, f(a,b)(max(a,b))), constant(" + name + "_dnn_outputs_weights_read), f(a,b)(a * b)), sum, d2), constant(" + name + "_dnn_outputs_bias_read), f(a,b)(a + b)), tensor(d0[1])(1.0), f(a,b)(a * b))"; + final String expression = "join(reduce(join(join(join(reduce(constant(" + name + "_dnn_hidden2_Const), sum, d2), imported_ml_function_" + name + "_dnn_hidden2_add, f(a,b)(a * b)), imported_ml_function_" + name + "_dnn_hidden2_add, f(a,b)(max(a,b))), constant(" + name + "_dnn_outputs_weights_read), f(a,b)(a * b)), sum, d2), constant(" + name + "_dnn_outputs_bias_read), f(a,b)(a + b))"; final String functionExpression1 = "join(reduce(join(reduce(rename(input, (d0, d1), (d0, d4)), sum, d0), constant(" + name + "_dnn_hidden1_weights_read), f(a,b)(a * b)), sum, d4), constant(" + name + "_dnn_hidden1_bias_read), f(a,b)(a + b))"; final String functionExpression2 = "join(reduce(join(join(join(0.009999999776482582, imported_ml_function_" + name + "_dnn_hidden1_add, f(a,b)(a * b)), imported_ml_function_" + name + "_dnn_hidden1_add, f(a,b)(max(a,b))), constant(" + name + "_dnn_hidden2_weights_read), f(a,b)(a * b)), sum, d3), constant(" + name + "_dnn_hidden2_bias_read), f(a,b)(a + b))"; diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionsTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionsTestCase.java index b3eda9b7e13..1567a4c3b5e 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionsTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionsTestCase.java @@ -44,20 +44,20 @@ public class RankingExpressionsTestCase extends SchemaTestCase { new AttributeFields(search)).configProperties(); assertEquals(6, rankProperties.size()); - assertEquals("rankingExpression(titlematch$).rankingScript", rankProperties.get(0).getFirst()); - assertEquals("var1 * var2 + 890", rankProperties.get(0).getSecond()); + assertEquals("rankingExpression(titlematch$).rankingScript", rankProperties.get(2).getFirst()); + assertEquals("var1 * var2 + 890", rankProperties.get(2).getSecond()); - assertEquals("rankingExpression(artistmatch).rankingScript", rankProperties.get(1).getFirst()); - assertEquals("78 + closeness(distance)", rankProperties.get(1).getSecond()); + assertEquals("rankingExpression(artistmatch).rankingScript", rankProperties.get(3).getFirst()); + assertEquals("78 + closeness(distance)", rankProperties.get(3).getSecond()); assertEquals("rankingExpression(firstphase).rankingScript", rankProperties.get(5).getFirst()); assertEquals("0.8 + 0.2 * rankingExpression(titlematch$@126063073eb2deb.ab95cd69909927c) + 0.8 * rankingExpression(titlematch$@c7e4c2d0e6d9f2a1.1d4ed08e56cce2e6) * closeness(distance)", rankProperties.get(5).getSecond()); - assertEquals("rankingExpression(titlematch$@c7e4c2d0e6d9f2a1.1d4ed08e56cce2e6).rankingScript", rankProperties.get(3).getFirst()); - assertEquals("7 * 8 + 890", rankProperties.get(3).getSecond()); + assertEquals("rankingExpression(titlematch$@c7e4c2d0e6d9f2a1.1d4ed08e56cce2e6).rankingScript", rankProperties.get(1).getFirst()); + assertEquals("7 * 8 + 890", rankProperties.get(1).getSecond()); - assertEquals("rankingExpression(titlematch$@126063073eb2deb.ab95cd69909927c).rankingScript", rankProperties.get(2).getFirst()); - assertEquals("4 * 5 + 890", rankProperties.get(2).getSecond()); + assertEquals("rankingExpression(titlematch$@126063073eb2deb.ab95cd69909927c).rankingScript", rankProperties.get(0).getFirst()); + assertEquals("4 * 5 + 890", rankProperties.get(0).getSecond()); } @Test(expected = IllegalArgumentException.class) diff --git a/config-model/src/test/java/com/yahoo/vespa/model/container/ContainerClusterTest.java b/config-model/src/test/java/com/yahoo/vespa/model/container/ContainerClusterTest.java index 33cf0635349..0f94df80421 100755 --- a/config-model/src/test/java/com/yahoo/vespa/model/container/ContainerClusterTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/container/ContainerClusterTest.java @@ -9,9 +9,11 @@ import com.yahoo.config.model.deploy.DeployState; import com.yahoo.config.model.deploy.TestProperties; import com.yahoo.config.model.test.MockRoot; import com.yahoo.config.provision.Environment; +import com.yahoo.config.provision.Flavor; import com.yahoo.config.provision.RegionName; import com.yahoo.config.provision.SystemName; import com.yahoo.config.provision.Zone; +import com.yahoo.config.provisioning.FlavorsConfig; import com.yahoo.container.handler.ThreadpoolConfig; import com.yahoo.search.config.QrStartConfig; import com.yahoo.vespa.model.Host; @@ -164,6 +166,8 @@ public class ContainerClusterTest { cluster.getConfig(tpBuilder); ThreadpoolConfig threadpoolConfig = new ThreadpoolConfig(tpBuilder); assertEquals(10, threadpoolConfig.maxthreads()); + assertEquals(0, threadpoolConfig.queueSize()); + assertEquals(0, threadpoolConfig.softStartSeconds(), 0); } @Test @@ -213,10 +217,44 @@ public class ContainerClusterTest { cluster.getConfig(tpBuilder); ThreadpoolConfig threadpoolConfig = new ThreadpoolConfig(tpBuilder); assertEquals(500, threadpoolConfig.maxthreads()); + assertEquals(0, threadpoolConfig.queueSize()); assertEquals(300.0, threadpoolConfig.softStartSeconds(), 0.0); } @Test + public void requireThatPoolAndQueueCanNotBeControlledByPropertiesWhenNoFlavor() { + DeployState state = new DeployState.Builder().properties(new TestProperties() + .setThreadPoolSizeFactor(8.5) + .setQueueSizeFactor(13.3)) + .build(); + MockRoot root = new MockRoot("foo", state); + ApplicationContainerCluster cluster = createContainerCluster(root, false); + addContainer(root.deployLogger(), cluster, "c1", "host-c1"); + + ThreadpoolConfig.Builder tpBuilder = new ThreadpoolConfig.Builder(); + cluster.getConfig(tpBuilder); + ThreadpoolConfig threadpoolConfig = new ThreadpoolConfig(tpBuilder); + assertEquals(500, threadpoolConfig.maxthreads()); + assertEquals(0, threadpoolConfig.queueSize()); + assertEquals(0.0, threadpoolConfig.softStartSeconds(), 0.0); + } + + @Test + public void requireThatPoolAndQueueCanBeControlledByPropertiesAndFlavor() { + FlavorsConfig.Flavor.Builder flavorBuilder = new FlavorsConfig.Flavor.Builder().name("my_flavor").minCpuCores(3); + NodeFlavorTuning nodeFlavorTuning = new NodeFlavorTuning(new Flavor(new FlavorsConfig.Flavor(flavorBuilder))) + .setThreadPoolSizeFactor(13.3) + .setQueueSizeFactor(17.5); + + ThreadpoolConfig.Builder tpBuilder = new ThreadpoolConfig.Builder(); + nodeFlavorTuning.getConfig(tpBuilder); + ThreadpoolConfig threadpoolConfig = new ThreadpoolConfig(tpBuilder); + assertEquals(40, threadpoolConfig.maxthreads()); + assertEquals(700, threadpoolConfig.queueSize()); + assertEquals(0.0, threadpoolConfig.softStartSeconds(), 0.0); + } + + @Test public void requireThatDefaultThreadPoolConfigIsSane() { MockRoot root = new MockRoot("foo"); ApplicationContainerCluster cluster = createContainerCluster(root, false); @@ -226,6 +264,7 @@ public class ContainerClusterTest { cluster.getConfig(tpBuilder); ThreadpoolConfig threadpoolConfig = new ThreadpoolConfig(tpBuilder); assertEquals(500, threadpoolConfig.maxthreads()); + assertEquals(0, threadpoolConfig.queueSize()); assertEquals(0.0, threadpoolConfig.softStartSeconds(), 0.0); } diff --git a/config-model/src/test/java/com/yahoo/vespa/model/ml/MlModelsTest.java b/config-model/src/test/java/com/yahoo/vespa/model/ml/MlModelsTest.java index ca84eb5eed7..57dbf132883 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/ml/MlModelsTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/ml/MlModelsTest.java @@ -64,10 +64,10 @@ public class MlModelsTest { private final String testProfile = "rankingExpression(input).rankingScript: attribute(argument)\n" + "rankingExpression(input).type: tensor<float>(d0[],d1[784])\n" + - "rankingExpression(Placeholder).rankingScript: attribute(argument)\n" + - "rankingExpression(Placeholder).type: tensor<float>(d0[],d1[784])\n" + "rankingExpression(imported_ml_function_mnist_saved_dnn_hidden1_add).rankingScript: join(reduce(join(rename(rankingExpression(input), (d0, d1), (d0, d4)), constant(mnist_saved_dnn_hidden1_weights_read), f(a,b)(a * b)), sum, d4), constant(mnist_saved_dnn_hidden1_bias_read), f(a,b)(a + b))\n" + "rankingExpression(mnist_tensorflow).rankingScript: join(reduce(join(map(join(reduce(join(join(join(0.009999999776482582, rankingExpression(imported_ml_function_mnist_saved_dnn_hidden1_add), f(a,b)(a * b)), rankingExpression(imported_ml_function_mnist_saved_dnn_hidden1_add), f(a,b)(max(a,b))), constant(mnist_saved_dnn_hidden2_weights_read), f(a,b)(a * b)), sum, d3), constant(mnist_saved_dnn_hidden2_bias_read), f(a,b)(a + b)), f(a)(1.0507009873554805 * if (a >= 0, a, 1.6732632423543772 * (exp(a) - 1)))), constant(mnist_saved_dnn_outputs_weights_read), f(a,b)(a * b)), sum, d2), constant(mnist_saved_dnn_outputs_bias_read), f(a,b)(a + b))\n" + + "rankingExpression(Placeholder).rankingScript: attribute(argument)\n" + + "rankingExpression(Placeholder).type: tensor<float>(d0[],d1[784])\n" + "rankingExpression(mnist_softmax_tensorflow).rankingScript: join(reduce(join(rename(rankingExpression(Placeholder), (d0, d1), (d0, d2)), constant(mnist_softmax_saved_layer_Variable_read), f(a,b)(a * b)), sum, d2), constant(mnist_softmax_saved_layer_Variable_1_read), f(a,b)(a + b))\n" + "rankingExpression(mnist_softmax_onnx).rankingScript: join(reduce(join(rename(rankingExpression(Placeholder), (d0, d1), (d0, d2)), constant(mnist_softmax_Variable), f(a,b)(a * b)), sum, d2), constant(mnist_softmax_Variable_1), f(a,b)(a + b))\n" + "rankingExpression(my_xgboost).rankingScript: if (f29 < -0.1234567, if (!(f56 >= -0.242398), 1.71218, -1.70044), if (f109 < 0.8723473, -1.94071, 1.85965)) + if (!(f60 >= -0.482947), if (f29 < -4.2387498, 0.784718, -0.96853), -6.23624)\n" + diff --git a/config-provisioning/src/main/java/com/yahoo/config/provision/Capacity.java b/config-provisioning/src/main/java/com/yahoo/config/provision/Capacity.java index 3c7a14b9496..f723575c342 100644 --- a/config-provisioning/src/main/java/com/yahoo/config/provision/Capacity.java +++ b/config-provisioning/src/main/java/com/yahoo/config/provision/Capacity.java @@ -72,7 +72,12 @@ public final class Capacity { /** Create a non-required, failable capacity request */ public static Capacity from(ClusterResources resources) { - return from(resources, false, true); + return from(resources, resources); + } + + /** Create a non-required, failable capacity request */ + public static Capacity from(ClusterResources min, ClusterResources max) { + return from(min, max, false, true); } public static Capacity from(ClusterResources resources, boolean required, boolean canFail) { diff --git a/config-provisioning/src/main/java/com/yahoo/config/provision/ClusterSpec.java b/config-provisioning/src/main/java/com/yahoo/config/provision/ClusterSpec.java index 97549e851ad..3a230c89732 100644 --- a/config-provisioning/src/main/java/com/yahoo/config/provision/ClusterSpec.java +++ b/config-provisioning/src/main/java/com/yahoo/config/provision/ClusterSpec.java @@ -29,7 +29,7 @@ public final class ClusterSpec { this.type = type; this.id = id; this.groupId = groupId; - this.vespaVersion = vespaVersion; + this.vespaVersion = Objects.requireNonNull(vespaVersion); this.exclusive = exclusive; // TODO(mpolden): Require combinedId to always be present for type combined after April 2020 if (type != Type.combined && combinedId.isPresent()) { diff --git a/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/ConfigSourceClient.java b/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/ConfigSourceClient.java index 9fb78e7e812..e9110a0bc78 100644 --- a/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/ConfigSourceClient.java +++ b/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/ConfigSourceClient.java @@ -24,8 +24,6 @@ interface ConfigSourceClient { List<String> getSourceConnections(); - void updateSubscribers(RawConfig config); - DelayedResponses delayedResponses(); } diff --git a/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/MemoryCacheConfigClient.java b/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/MemoryCacheConfigClient.java index df0b274f32b..2e55b9a6f86 100644 --- a/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/MemoryCacheConfigClient.java +++ b/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/MemoryCacheConfigClient.java @@ -60,9 +60,6 @@ class MemoryCacheConfigClient implements ConfigSourceClient { } @Override - public void updateSubscribers(RawConfig config) {} - - @Override public DelayedResponses delayedResponses() { return delayedResponses; } diff --git a/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/RpcConfigSourceClient.java b/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/RpcConfigSourceClient.java index 12fd95b8503..ba58c369afd 100644 --- a/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/RpcConfigSourceClient.java +++ b/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/RpcConfigSourceClient.java @@ -10,6 +10,10 @@ import com.yahoo.jrt.Spec; import com.yahoo.jrt.Supervisor; import com.yahoo.jrt.Target; import com.yahoo.jrt.Transport; + +import java.util.Map; +import java.util.Optional; +import java.util.concurrent.ConcurrentHashMap; import java.util.logging.Level; import com.yahoo.vespa.config.ConfigCacheKey; import com.yahoo.vespa.config.RawConfig; @@ -17,15 +21,14 @@ import com.yahoo.vespa.config.TimingValues; import com.yahoo.vespa.config.protocol.JRTServerConfigRequest; import java.util.ArrayList; -import java.util.HashMap; import java.util.List; import java.util.concurrent.DelayQueue; -import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.ScheduledFuture; import java.util.logging.Logger; +import static java.util.concurrent.TimeUnit.MILLISECONDS; import static java.util.concurrent.TimeUnit.SECONDS; /** @@ -33,7 +36,7 @@ import static java.util.concurrent.TimeUnit.SECONDS; * * @author hmusum */ -class RpcConfigSourceClient implements ConfigSourceClient { +class RpcConfigSourceClient implements ConfigSourceClient, Runnable { private final static Logger log = Logger.getLogger(RpcConfigSourceClient.class.getName()); private static final double timingValuesRatio = 0.8; @@ -42,16 +45,18 @@ class RpcConfigSourceClient implements ConfigSourceClient { private final RpcServer rpcServer; private final ConfigSourceSet configSourceSet; - private final HashMap<ConfigCacheKey, Subscriber> activeSubscribers = new HashMap<>(); - private final Object activeSubscribersLock = new Object(); + private final Map<ConfigCacheKey, Subscriber> activeSubscribers = new ConcurrentHashMap<>(); private final MemoryCache memoryCache; private final DelayedResponses delayedResponses; private final static TimingValues timingValues; - private final ExecutorService exec; + private final ScheduledExecutorService nextConfigScheduler = + Executors.newScheduledThreadPool(1, new DaemonThreadFactory("next config")); + private ScheduledFuture<?> nextConfigFuture; private final JRTConfigRequester requester; // Scheduled executor that periodically checks for requests that have timed out and response should be returned to clients - private final ScheduledExecutorService scheduler = Executors.newScheduledThreadPool(1, new DaemonThreadFactory()); - private ScheduledFuture<?> delayedResponseScheduler; + private final ScheduledExecutorService delayedResponsesScheduler = + Executors.newScheduledThreadPool(1, new DaemonThreadFactory("delayed responses")); + private ScheduledFuture<?> delayedResponsesFuture; static { // Proxy should time out before clients upon subscription. @@ -69,14 +74,10 @@ class RpcConfigSourceClient implements ConfigSourceClient { this.memoryCache = memoryCache; this.delayedResponses = new DelayedResponses(); checkConfigSources(); - exec = Executors.newCachedThreadPool(new DaemonThreadFactory("subscriber-")); + nextConfigFuture = nextConfigScheduler.scheduleAtFixedRate(this, 0, 10, MILLISECONDS); requester = JRTConfigRequester.create(configSourceSet, timingValues); - // Wait for 5 seconds initially, then run every second - delayedResponseScheduler = scheduler.scheduleAtFixedRate( - new DelayedResponseHandler(delayedResponses, memoryCache, rpcServer), - 5, - 1, - SECONDS); + DelayedResponseHandler command = new DelayedResponseHandler(delayedResponses, memoryCache, rpcServer); + delayedResponsesFuture = delayedResponsesScheduler.scheduleAtFixedRate(command, 5, 1, SECONDS); } /** @@ -153,30 +154,36 @@ class RpcConfigSourceClient implements ConfigSourceClient { } private void subscribeToConfig(RawConfig input, ConfigCacheKey configCacheKey) { - synchronized (activeSubscribersLock) { - if (activeSubscribers.containsKey(configCacheKey)) { - log.log(Level.FINE, () -> "Already a subscriber running for: " + configCacheKey); - } else { - log.log(Level.FINE, () -> "Could not find good config in cache, creating subscriber for: " + configCacheKey); - UpstreamConfigSubscriber subscriber = - new UpstreamConfigSubscriber(input, this, configSourceSet, timingValues, requester, memoryCache); - try { - subscriber.subscribe(); - activeSubscribers.put(configCacheKey, subscriber); - exec.execute(subscriber); - } catch (ConfigurationRuntimeException e) { - log.log(Level.INFO, "Subscribe for '" + configCacheKey + "' failed, closing subscriber"); - subscriber.cancel(); - } - } + if (activeSubscribers.containsKey(configCacheKey)) return; + + log.log(Level.FINE, () -> "Could not find good config in cache, creating subscriber for: " + configCacheKey); + var subscriber = new Subscriber(input, configSourceSet, timingValues, requester); + try { + subscriber.subscribe(); + activeSubscribers.put(configCacheKey, subscriber); + } catch (ConfigurationRuntimeException e) { + log.log(Level.INFO, "Subscribe for '" + configCacheKey + "' failed, closing subscriber"); + subscriber.cancel(); } } @Override + public void run() { + activeSubscribers.values().forEach(subscriber -> { + if (!subscriber.isClosed()) { + Optional<RawConfig> config = subscriber.nextGeneration(); + config.ifPresent(this::updateWithNewConfig); + } + }); + } + + @Override public void cancel() { shutdownSourceConnections(); - delayedResponseScheduler.cancel(true); - scheduler.shutdown(); + delayedResponsesFuture.cancel(true); + delayedResponsesScheduler.shutdown(); + nextConfigFuture.cancel(true); + nextConfigScheduler.shutdown(); } /** @@ -184,13 +191,9 @@ class RpcConfigSourceClient implements ConfigSourceClient { */ @Override public void shutdownSourceConnections() { - synchronized (activeSubscribersLock) { - for (Subscriber subscriber : activeSubscribers.values()) { - subscriber.cancel(); - } - activeSubscribers.clear(); - } - exec.shutdown(); + activeSubscribers.values().forEach(Subscriber::cancel); + activeSubscribers.clear(); + nextConfigScheduler.shutdown(); requester.close(); } @@ -250,4 +253,12 @@ class RpcConfigSourceClient implements ConfigSourceClient { return delayedResponses; } + private void updateWithNewConfig(RawConfig newConfig) { + log.log(Level.FINE, () -> "config to be returned for '" + newConfig.getKey() + + "', generation=" + newConfig.getGeneration() + + ", payload=" + newConfig.getPayload()); + memoryCache.update(newConfig); + updateSubscribers(newConfig); + } + } diff --git a/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/Subscriber.java b/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/Subscriber.java index 74c99b7670b..f96d6470679 100644 --- a/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/Subscriber.java +++ b/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/Subscriber.java @@ -1,12 +1,68 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.config.proxy; +import com.yahoo.config.subscription.ConfigSourceSet; +import com.yahoo.config.subscription.impl.GenericConfigHandle; +import com.yahoo.config.subscription.impl.GenericConfigSubscriber; +import com.yahoo.config.subscription.impl.JRTConfigRequester; + +import java.util.Optional; +import java.util.logging.Level; +import com.yahoo.vespa.config.ConfigKey; +import com.yahoo.vespa.config.RawConfig; +import com.yahoo.vespa.config.TimingValues; +import com.yahoo.yolean.Exceptions; + +import java.util.Map; +import java.util.logging.Logger; + /** - * Interface for subscribing to config from upstream config sources. - * * @author hmusum */ -public interface Subscriber extends Runnable { +public class Subscriber { + + private final static Logger log = Logger.getLogger(Subscriber.class.getName()); + + private final RawConfig config; + private final ConfigSourceSet configSourceSet; + private final TimingValues timingValues; + private final GenericConfigSubscriber subscriber; + private GenericConfigHandle handle; + + Subscriber(RawConfig config, ConfigSourceSet configSourceSet, TimingValues timingValues, JRTConfigRequester requester) { + this.config = config; + this.configSourceSet = configSourceSet; + this.timingValues = timingValues; + this.subscriber = new GenericConfigSubscriber(Map.of(configSourceSet, requester)); + } + + void subscribe() { + ConfigKey<?> key = config.getKey(); + handle = subscriber.subscribe(new ConfigKey<>(key.getName(), key.getConfigId(), key.getNamespace()), + config.getDefContent(), configSourceSet, timingValues); + } + + public Optional<RawConfig> nextGeneration() { + if (subscriber.nextGeneration(0)) { + try { + return Optional.of(handle.getRawConfig()); + } catch (Exception e) { // To avoid thread throwing exception and loop never running this again + log.log(Level.WARNING, "Got exception: " + Exceptions.toMessageString(e)); + } catch (Throwable e) { + com.yahoo.protect.Process.logAndDie("Got error, exiting: " + Exceptions.toMessageString(e)); + } + } + return Optional.empty(); + } + + public void cancel() { + if (subscriber != null) { + subscriber.close(); + } + } + + boolean isClosed() { + return subscriber.isClosed(); + } - void cancel(); } diff --git a/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/UpstreamConfigSubscriber.java b/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/UpstreamConfigSubscriber.java deleted file mode 100644 index a24407588be..00000000000 --- a/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/UpstreamConfigSubscriber.java +++ /dev/null @@ -1,82 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.config.proxy; - -import com.yahoo.config.subscription.ConfigSourceSet; -import com.yahoo.config.subscription.impl.GenericConfigHandle; -import com.yahoo.config.subscription.impl.GenericConfigSubscriber; -import com.yahoo.config.subscription.impl.JRTConfigRequester; -import java.util.logging.Level; -import com.yahoo.vespa.config.ConfigKey; -import com.yahoo.vespa.config.RawConfig; -import com.yahoo.vespa.config.TimingValues; -import com.yahoo.yolean.Exceptions; - -import java.util.Map; -import java.util.logging.Logger; - -/** - * @author hmusum - */ -public class UpstreamConfigSubscriber implements Subscriber { - - private final static Logger log = Logger.getLogger(UpstreamConfigSubscriber.class.getName()); - - private final RawConfig config; - private final ConfigSourceClient configSourceClient; - private final ConfigSourceSet configSourceSet; - private final TimingValues timingValues; - private final JRTConfigRequester requester; - private final MemoryCache memoryCache; - private GenericConfigSubscriber subscriber; - private GenericConfigHandle handle; - - UpstreamConfigSubscriber(RawConfig config, ConfigSourceClient configSourceClient, ConfigSourceSet configSourceSet, - TimingValues timingValues, JRTConfigRequester requester, - MemoryCache memoryCache) { - this.config = config; - this.configSourceClient = configSourceClient; - this.configSourceSet = configSourceSet; - this.timingValues = timingValues; - this.requester = requester; - this.memoryCache = memoryCache; - } - - void subscribe() { - subscriber = new GenericConfigSubscriber(Map.of(configSourceSet, requester)); - ConfigKey<?> key = config.getKey(); - handle = subscriber.subscribe(new ConfigKey<>(key.getName(), key.getConfigId(), key.getNamespace()), - config.getDefContent(), configSourceSet, timingValues); - } - - @Override - public void run() { - do { - if (! subscriber.nextGeneration()) continue; - - try { - updateWithNewConfig(handle); - } catch (Exception e) { // To avoid thread throwing exception and loop never running this again - log.log(Level.WARNING, "Got exception: " + Exceptions.toMessageString(e)); - } catch (Throwable e) { - com.yahoo.protect.Process.logAndDie("Got error, exiting: " + Exceptions.toMessageString(e)); - } - } while (!subscriber.isClosed()); - } - - private void updateWithNewConfig(GenericConfigHandle handle) { - RawConfig newConfig = handle.getRawConfig(); - log.log(Level.FINE, () -> "config to be returned for '" + newConfig.getKey() + - "', generation=" + newConfig.getGeneration() + - ", payload=" + newConfig.getPayload()); - memoryCache.update(newConfig); - configSourceClient.updateSubscribers(newConfig); - } - - @Override - public void cancel() { - if (subscriber != null) { - subscriber.close(); - } - } - -} diff --git a/config-proxy/src/test/java/com/yahoo/vespa/config/proxy/MockConfigSourceClient.java b/config-proxy/src/test/java/com/yahoo/vespa/config/proxy/MockConfigSourceClient.java index 06e55eef4fa..2126d673da0 100644 --- a/config-proxy/src/test/java/com/yahoo/vespa/config/proxy/MockConfigSourceClient.java +++ b/config-proxy/src/test/java/com/yahoo/vespa/config/proxy/MockConfigSourceClient.java @@ -54,9 +54,6 @@ public class MockConfigSourceClient implements ConfigSourceClient{ } @Override - public void updateSubscribers(RawConfig config) { } - - @Override public DelayedResponses delayedResponses() { return delayedResponses; } } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java index 2773659559b..0f141dee5b4 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java @@ -150,6 +150,8 @@ public class ModelContextImpl implements ModelContext { private final Optional<EndpointCertificateSecrets> endpointCertificateSecrets; private final double defaultTermwiseLimit; private final double defaultSoftStartSeconds; + private final double threadPoolSizeFactor; + private final double queueSizefactor; private final Optional<AthenzDomain> athenzDomain; public Properties(ApplicationId applicationId, @@ -186,6 +188,10 @@ public class ModelContextImpl implements ModelContext { .with(FetchVector.Dimension.APPLICATION_ID, applicationId.serializedForm()).value(); defaultTopKprobability = Flags.DEFAULT_TOP_K_PROBABILITY.bindTo(flagSource) .with(FetchVector.Dimension.APPLICATION_ID, applicationId.serializedForm()).value(); + threadPoolSizeFactor = Flags.DEFAULT_THREADPOOL_SIZE_FACTOR.bindTo(flagSource) + .with(FetchVector.Dimension.APPLICATION_ID, applicationId.serializedForm()).value(); + queueSizefactor = Flags.DEFAULT_QUEUE_SIZE_FACTOR.bindTo(flagSource) + .with(FetchVector.Dimension.APPLICATION_ID, applicationId.serializedForm()).value(); this.athenzDomain = athenzDomain; } @@ -238,8 +244,18 @@ public class ModelContextImpl implements ModelContext { @Override public double defaultTermwiseLimit() { return defaultTermwiseLimit; } + @Override + public double threadPoolSizeFactor() { + return threadPoolSizeFactor; + } + + @Override + public double queueSizeFactor() { + return queueSizefactor; + } + public double defaultSoftStartSeconds() { - return 0; + return defaultSoftStartSeconds; } @Override diff --git a/container-core/abi-spec.json b/container-core/abi-spec.json index 6d683c53984..82d34d88b99 100644 --- a/container-core/abi-spec.json +++ b/container-core/abi-spec.json @@ -171,6 +171,7 @@ "public void <init>()", "public void <init>(com.yahoo.container.handler.ThreadpoolConfig)", "public com.yahoo.container.handler.ThreadpoolConfig$Builder maxthreads(int)", + "public com.yahoo.container.handler.ThreadpoolConfig$Builder queueSize(int)", "public com.yahoo.container.handler.ThreadpoolConfig$Builder maxThreadExecutionTimeSeconds(int)", "public com.yahoo.container.handler.ThreadpoolConfig$Builder softStartSeconds(double)", "public final boolean dispatchGetConfig(com.yahoo.config.ConfigInstance$Producer)", @@ -210,6 +211,7 @@ "public static java.lang.String getDefVersion()", "public void <init>(com.yahoo.container.handler.ThreadpoolConfig$Builder)", "public int maxthreads()", + "public int queueSize()", "public int maxThreadExecutionTimeSeconds()", "public double softStartSeconds()" ], diff --git a/container-core/src/main/java/com/yahoo/container/handler/ThreadPoolProvider.java b/container-core/src/main/java/com/yahoo/container/handler/ThreadPoolProvider.java index b427a58c9b7..0e786cfbc8f 100644 --- a/container-core/src/main/java/com/yahoo/container/handler/ThreadPoolProvider.java +++ b/container-core/src/main/java/com/yahoo/container/handler/ThreadPoolProvider.java @@ -10,6 +10,7 @@ import com.yahoo.container.protect.ProcessTerminator; import com.yahoo.jdisc.Metric; import java.util.Map; +import java.util.concurrent.ArrayBlockingQueue; import java.util.concurrent.BlockingQueue; import java.util.concurrent.Executor; import java.util.concurrent.ExecutorService; @@ -33,17 +34,30 @@ public class ThreadPoolProvider extends AbstractComponent implements Provider<Ex private final ExecutorServiceWrapper threadpool; + private static BlockingQueue<Runnable> createQ(int queueSize, int maxThreads) { + return (queueSize == 0) + ? new SynchronousQueue<>(false) + : (queueSize < 0) + ? new ArrayBlockingQueue<>(maxThreads*4) + : new ArrayBlockingQueue<>(queueSize); + } + + private static int computeThreadPoolSize(int maxNumThreads) { + return (maxNumThreads <= 0) + ? Runtime.getRuntime().availableProcessors() * 4 + : maxNumThreads; + } @Inject public ThreadPoolProvider(ThreadpoolConfig threadpoolConfig, Metric metric) { this(threadpoolConfig, metric, new ProcessTerminator()); } public ThreadPoolProvider(ThreadpoolConfig threadpoolConfig, Metric metric, ProcessTerminator processTerminator) { + int maxNumThreads = computeThreadPoolSize(threadpoolConfig.maxthreads()); WorkerCompletionTimingThreadPoolExecutor executor = - new WorkerCompletionTimingThreadPoolExecutor(threadpoolConfig.maxthreads(), - threadpoolConfig.maxthreads(), + new WorkerCompletionTimingThreadPoolExecutor(maxNumThreads, maxNumThreads, 0L, TimeUnit.SECONDS, - new SynchronousQueue<>(false), + createQ(threadpoolConfig.queueSize(), maxNumThreads), ThreadFactoryFactory.getThreadFactory("threadpool"), metric); // Prestart needed, if not all threads will be created by the fist N tasks and hence they might also @@ -87,8 +101,9 @@ public class ThreadPoolProvider extends AbstractComponent implements Provider<Ex /** * A service executor wrapper which emits metrics and * shuts down the vm when no workers are available for too long to avoid containers lingering in a blocked state. + * Package private for testing */ - private final static class ExecutorServiceWrapper extends ForwardingExecutorService { + final static class ExecutorServiceWrapper extends ForwardingExecutorService { private final WorkerCompletionTimingThreadPoolExecutor wrapped; private final Metric metric; @@ -160,8 +175,11 @@ public class ThreadPoolProvider extends AbstractComponent implements Provider<Ex } - /** A thread pool executor which maintains the last time a worker completed */ - private final static class WorkerCompletionTimingThreadPoolExecutor extends ThreadPoolExecutor { + /** + * A thread pool executor which maintains the last time a worker completed + * package private for testing + **/ + final static class WorkerCompletionTimingThreadPoolExecutor extends ThreadPoolExecutor { private static final String UNHANDLED_EXCEPTIONS_METRIC = "jdisc.thread_pool.unhandled_exceptions"; diff --git a/container-core/src/main/resources/configdefinitions/threadpool.def b/container-core/src/main/resources/configdefinitions/threadpool.def index 9bb9badd9b5..abc60f9f06d 100644 --- a/container-core/src/main/resources/configdefinitions/threadpool.def +++ b/container-core/src/main/resources/configdefinitions/threadpool.def @@ -2,8 +2,16 @@ namespace=container.handler +## Num ber of thread in the thread pool +## Setting it to 0 or negative number will cause it to be set to #cores * 4 maxthreads int default=500 +## max queue size +## There can be queueSize + maxthreads requests inflight concurrently +## The container will start replying 503 +## Negative value will cause it to set to maxthreads*4 +queueSize int default=0 + # The max time the container tolerates having no threads available before it shuts down to # get out of a bad state. This should be set a bit higher than the expected max execution # time of each request when in a state of overload, i.e about "worst case execution time*2" diff --git a/container-core/src/test/java/com/yahoo/container/handler/ThreadPoolProviderTestCase.java b/container-core/src/test/java/com/yahoo/container/handler/ThreadPoolProviderTestCase.java index 918863f6dda..761ed40763c 100644 --- a/container-core/src/test/java/com/yahoo/container/handler/ThreadPoolProviderTestCase.java +++ b/container-core/src/test/java/com/yahoo/container/handler/ThreadPoolProviderTestCase.java @@ -5,6 +5,7 @@ import static org.junit.Assert.fail; import java.util.concurrent.Executor; import java.util.concurrent.RejectedExecutionException; +import java.util.concurrent.ThreadPoolExecutor; import com.yahoo.container.protect.ProcessTerminator; import org.junit.Ignore; @@ -58,6 +59,39 @@ public class ThreadPoolProviderTestCase { fail("Pool did not reject tasks after shutdown."); } + private ThreadPoolExecutor createPool(int maxThreads, int queueSize) { + ThreadpoolConfig config = new ThreadpoolConfig(new ThreadpoolConfig.Builder().maxthreads(maxThreads).queueSize(queueSize)); + ThreadPoolProvider provider = new ThreadPoolProvider(config, Mockito.mock(Metric.class)); + ThreadPoolProvider.ExecutorServiceWrapper wrapper = (ThreadPoolProvider.ExecutorServiceWrapper) provider.get(); + ThreadPoolProvider.WorkerCompletionTimingThreadPoolExecutor executor = (ThreadPoolProvider.WorkerCompletionTimingThreadPoolExecutor)wrapper.delegate(); + return executor; + } + + @Test + public void testThatThreadPoolSizeFollowsConfig() { + ThreadPoolExecutor executor = createPool(3, 9); + assertEquals(3, executor.getMaximumPoolSize()); + assertEquals(9, executor.getQueue().remainingCapacity()); + } + @Test + public void testThatThreadPoolSizeAutoDetected() { + ThreadPoolExecutor executor = createPool(0, 0); + assertEquals(Runtime.getRuntime().availableProcessors()*4, executor.getMaximumPoolSize()); + assertEquals(0, executor.getQueue().remainingCapacity()); + } + @Test + public void testThatQueueSizeAutoDetected() { + ThreadPoolExecutor executor = createPool(3, -1); + assertEquals(3, executor.getMaximumPoolSize()); + assertEquals(executor.getMaximumPoolSize()*4, executor.getQueue().remainingCapacity()); + } + @Test + public void testThatThreadPoolSizeAndQueueSizeAutoDetected() { + ThreadPoolExecutor executor = createPool(0, -1); + assertEquals(Runtime.getRuntime().availableProcessors()*4, executor.getMaximumPoolSize()); + assertEquals(executor.getMaximumPoolSize()*4, executor.getQueue().remainingCapacity()); + } + private class FlipIt implements Runnable { public final Receiver<Boolean> didItRun = new Receiver<>(); diff --git a/default_build_settings.cmake b/default_build_settings.cmake index 66fcf1b2e16..67d2be5917d 100644 --- a/default_build_settings.cmake +++ b/default_build_settings.cmake @@ -8,16 +8,17 @@ function(setup_vespa_default_build_settings_rhel_6_10) set(DEFAULT_CMAKE_SHARED_LINKER_FLAGS "-lrt" PARENT_SCOPE) endfunction() -function(setup_vespa_default_build_settings_rhel_7_7) - message("-- Setting up default build settings for rhel 7.7") +function(setup_vespa_default_build_settings_rhel_7) + message("-- Setting up default build settings for rhel 7") set(DEFAULT_EXTRA_LINK_DIRECTORY "${VESPA_DEPS}/lib64" "/usr/lib64/llvm7.0/lib" PARENT_SCOPE) set(DEFAULT_EXTRA_INCLUDE_DIRECTORY "${VESPA_DEPS}/include" "/usr/include/llvm7.0" "/usr/include/openblas" PARENT_SCOPE) set(DEFAULT_VESPA_LLVM_VERSION "7" PARENT_SCOPE) endfunction() -function(setup_vespa_default_build_settings_rhel_8_1) - message("-- Setting up default build settings for rhel 8.1") - set(DEFAULT_VESPA_LLVM_VERSION "7" PARENT_SCOPE) +function(setup_vespa_default_build_settings_rhel_8) + message("-- Setting up default build settings for rhel 8") + set(DEFAULT_EXTRA_INCLUDE_DIRECTORY "${VESPA_DEPS}/include" "/usr/include/openblas" PARENT_SCOPE) + set(DEFAULT_VESPA_LLVM_VERSION "8" PARENT_SCOPE) endfunction() function(setup_vespa_default_build_settings_centos_7) @@ -159,9 +160,9 @@ function(vespa_use_default_build_settings) if(VESPA_OS_DISTRO_COMBINED STREQUAL "rhel 6.10") setup_vespa_default_build_settings_rhel_6_10() elseif(VESPA_OS_DISTRO_COMBINED STREQUAL "rhel 7.7") - setup_vespa_default_build_settings_rhel_7_7() + setup_vespa_default_build_settings_rhel_7() elseif(VESPA_OS_DISTRO_COMBINED STREQUAL "rhel 8.1") - setup_vespa_default_build_settings_rhel_8_1() + setup_vespa_default_build_settings_rhel_8() elseif(VESPA_OS_DISTRO_COMBINED STREQUAL "centos 7") setup_vespa_default_build_settings_centos_7() elseif(VESPA_OS_DISTRO_COMBINED STREQUAL "centos 8") diff --git a/dist/vespa.spec b/dist/vespa.spec index 26310c19750..aa0232fa421 100644 --- a/dist/vespa.spec +++ b/dist/vespa.spec @@ -38,9 +38,10 @@ BuildRequires: rh-maven35 %define _rhmaven35_enable /opt/rh/rh-maven35/enable %endif %if 0%{?el8} -BuildRequires: gcc-c++ -BuildRequires: libatomic +BuildRequires: gcc-toolset-9-gcc-c++ +BuildRequires: gcc-toolset-9-binutils BuildRequires: maven +%define _devtoolset_enable /opt/rh/gcc-toolset-9/enable %endif %if 0%{?fedora} BuildRequires: gcc-c++ diff --git a/flags/src/main/java/com/yahoo/vespa/flags/Flags.java b/flags/src/main/java/com/yahoo/vespa/flags/Flags.java index 76206996207..85d841e4bb0 100644 --- a/flags/src/main/java/com/yahoo/vespa/flags/Flags.java +++ b/flags/src/main/java/com/yahoo/vespa/flags/Flags.java @@ -150,7 +150,16 @@ public class Flags { "Default number of seconds that a soft start shall use", "Takes effect at redeployment", ZONE_ID, APPLICATION_ID); - + public static final UnboundDoubleFlag DEFAULT_THREADPOOL_SIZE_FACTOR = defineDoubleFlag( + "default-threadpool-size-factor", 0.0, + "Default multiplication factor when computing maxthreads for main container threadpool based on available cores", + "Takes effect at redeployment", + ZONE_ID, APPLICATION_ID); + public static final UnboundDoubleFlag DEFAULT_QUEUE_SIZE_FACTOR = defineDoubleFlag( + "default-queue-size-factor", 0.0, + "Default multiplication factor when computing queuesize for burst handling", + "Takes effect at redeployment", + ZONE_ID, APPLICATION_ID); public static final UnboundDoubleFlag DEFAULT_TOP_K_PROBABILITY = defineDoubleFlag( "default-top-k-probability", 1.0, "Default probability that you will get the globally top K documents when merging many partitions.", diff --git a/model-integration/src/main/java/ai/vespa/rankingexpression/importer/operations/MatMul.java b/model-integration/src/main/java/ai/vespa/rankingexpression/importer/operations/MatMul.java index 2b0af93fd8e..d326291a5a5 100644 --- a/model-integration/src/main/java/ai/vespa/rankingexpression/importer/operations/MatMul.java +++ b/model-integration/src/main/java/ai/vespa/rankingexpression/importer/operations/MatMul.java @@ -126,7 +126,7 @@ public class MatMul extends IntermediateOperation { // a1 < a2 < a3 < a4 for (int j = i+1; j < typeA.rank(); ++j) { String jDim = typeA.dimensionNames().get(j); - renamer.addConstraint(iDim, jDim, DimensionRenamer.Constraint.lessThan(true), this); + renamer.addConstraint(iDim, jDim, DimensionRenamer.Constraint.lessThan(false), this); } // not equal to last 2 dimensions in B for (int j = typeB.rank()-2; j < typeB.rank(); ++j) { @@ -148,7 +148,7 @@ public class MatMul extends IntermediateOperation { // b1 < b2 < b3 < b4 for (int j = i+1; j < typeB.rank(); ++j) { String jDim = typeB.dimensionNames().get(j); - renamer.addConstraint(iDim, jDim, DimensionRenamer.Constraint.lessThan(true), this); + renamer.addConstraint(iDim, jDim, DimensionRenamer.Constraint.lessThan(false), this); } // not equal to last 2 dimensions in A for (int j = typeA.rank()-2; j < typeA.rank(); ++j) { diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/RealNodeRepositoryTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/RealNodeRepositoryTest.java index 4e0fd95384c..2e40596bcf0 100644 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/RealNodeRepositoryTest.java +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/RealNodeRepositoryTest.java @@ -9,6 +9,7 @@ import com.yahoo.config.provision.NodeType; import com.yahoo.config.provision.host.FlavorOverrides; import com.yahoo.vespa.hosted.node.admin.configserver.ConfigServerApi; import com.yahoo.vespa.hosted.node.admin.configserver.ConfigServerApiImpl; +import com.yahoo.vespa.hosted.provision.restapi.NodesV2ApiHandler; import com.yahoo.vespa.hosted.provision.testutils.ContainerConfig; import org.junit.After; import org.junit.Before; @@ -52,7 +53,7 @@ public class RealNodeRepositoryTest { * Starts NodeRepository with * {@link com.yahoo.vespa.hosted.provision.testutils.MockNodeFlavors} * {@link com.yahoo.vespa.hosted.provision.testutils.MockNodeRepository} - * {@link com.yahoo.vespa.hosted.provision.restapi.v2.NodesApiHandler} + * {@link NodesV2ApiHandler} * These classes define some test data that is used in these tests. */ @Before diff --git a/node-repository/src/main/config/node-repository.xml b/node-repository/src/main/config/node-repository.xml index 186f052a274..90e98d11ac0 100644 --- a/node-repository/src/main/config/node-repository.xml +++ b/node-repository/src/main/config/node-repository.xml @@ -9,11 +9,11 @@ <component id="com.yahoo.config.provision.NodeFlavors" bundle="config-provisioning" /> <component id="com.yahoo.vespa.hosted.provision.provisioning.ProvisionServiceProvider" class="com.yahoo.vespa.hosted.provision.provisioning.EmptyProvisionServiceProvider" bundle="node-repository"/> -<handler id="com.yahoo.vespa.hosted.provision.restapi.v2.NodesApiHandler" bundle="node-repository"> +<handler id="com.yahoo.vespa.hosted.provision.restapi.NodesV2ApiHandler" bundle="node-repository"> <binding>http://*/nodes/v2/*</binding> </handler> -<handler id="com.yahoo.vespa.hosted.provision.restapi.v2.LoadBalancersApiHandler" bundle="node-repository"> +<handler id="com.yahoo.vespa.hosted.provision.restapi.LoadBalancersV1ApiHandler" bundle="node-repository"> <binding>http://*/loadbalancers/v1/*</binding> </handler> diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/NodeRepository.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/NodeRepository.java index 8b019d54d49..a4e5255ad09 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/NodeRepository.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/NodeRepository.java @@ -38,7 +38,7 @@ import com.yahoo.vespa.hosted.provision.persistence.DnsNameResolver; import com.yahoo.vespa.hosted.provision.persistence.NameResolver; import com.yahoo.vespa.hosted.provision.provisioning.DockerImages; import com.yahoo.vespa.hosted.provision.provisioning.FirmwareChecks; -import com.yahoo.vespa.hosted.provision.restapi.v2.NotFoundException; +import com.yahoo.vespa.hosted.provision.restapi.NotFoundException; import java.time.Clock; import java.time.Duration; diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeRepositoryProvisioner.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeRepositoryProvisioner.java index a3340e4cedb..4000354243f 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeRepositoryProvisioner.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeRepositoryProvisioner.java @@ -137,32 +137,60 @@ public class NodeRepositoryProvisioner implements Provisioner { application = application.withClusterLimits(clusterId, requested.minResources(), requested.maxResources()); nodeRepository.applications().put(application, lock); return application.clusters().get(clusterId).targetResources() - .orElseGet(() -> currentResources(applicationId, clusterId, requested) - .orElse(requested.minResources())); + .orElseGet(() -> currentResources(applicationId, clusterId, requested)); } } - /** Returns the current resources of this cluster, if it's already deployed and inside the requested limits */ - private Optional<ClusterResources> currentResources(ApplicationId applicationId, - ClusterSpec.Id clusterId, - Capacity requested) { + /** Returns the current resources of this cluster, or the closes */ + private ClusterResources currentResources(ApplicationId applicationId, + ClusterSpec.Id clusterId, + Capacity requested) { List<Node> nodes = NodeList.copyOf(nodeRepository.getNodes(applicationId, Node.State.active)) .cluster(clusterId) .not().retired() .not().removable() .asList(); - if (nodes.isEmpty()) return Optional.empty(); + if (nodes.isEmpty()) return requested.minResources(); // New deployment: Start at min + long groups = nodes.stream().map(node -> node.allocation().get().membership().cluster().group()).distinct().count(); + var currentResources = new ClusterResources(nodes.size(), (int)groups, nodes.get(0).flavor().resources()); + return ensureWithin(requested.minResources(), requested.maxResources(), currentResources); + } + + /** Make the minimal adjustments needed to the current resources to stay within the limits */ + private ClusterResources ensureWithin(ClusterResources min, ClusterResources max, ClusterResources current) { + int nodes = between(min.nodes(), max.nodes(), current.nodes()); + int groups = between(min.groups(), max.groups(), current.groups()); + if (nodes % groups != 0) { + // That didn't work - try to preserve current group size instead. + // Rounding here is needed because a node may be missing due to node failing. + int currentGroupsSize = Math.round((float)current.nodes() / current.groups()); + nodes = currentGroupsSize * groups; + if (nodes != between(min.nodes(), max.nodes(), nodes)) { + // Give up: Use max + nodes = max.nodes(); + groups = max.groups(); + } + } + if (min.nodeResources() != NodeResources.unspecified && max.nodeResources() != NodeResources.unspecified) { + double vcpu = between(min.nodeResources().vcpu(), max.nodeResources().vcpu(), current.nodeResources().vcpu()); + double memoryGb = between(min.nodeResources().memoryGb(), max.nodeResources().memoryGb(), current.nodeResources().memoryGb()); + double diskGb = between(min.nodeResources().diskGb(), max.nodeResources().diskGb(), current.nodeResources().diskGb()); + // Combine computed scaled resources with requested non-scaled resources (for which min=max) + NodeResources nodeResources = min.nodeResources().withVcpu(vcpu).withMemoryGb(memoryGb).withDiskGb(diskGb); + return new ClusterResources(nodes, groups, nodeResources); + } + else { + return new ClusterResources(nodes, groups, current.nodeResources()); + } + } - // To allow non-numeric settings to be updated without resetting to the min target, we need to use - // the non-numeric settings of the current min limit with the current numeric settings - NodeResources nodeResources = nodes.get(0).allocation().get().requestedResources() - .with(requested.minResources().nodeResources().diskSpeed()) - .with(requested.minResources().nodeResources().storageType()); - var currentResources = new ClusterResources(nodes.size(), (int)groups, nodeResources); - if ( ! currentResources.isWithin(requested.minResources(), requested.maxResources())) return Optional.empty(); + private int between(int min, int max, int n) { + return Math.min(max, Math.max(min, n)); + } - return Optional.of(currentResources); + private double between(double min, double max, double n) { + return Math.min(max, Math.max(min, n)); } private void logIfDownscaled(int targetNodes, int actualNodes, ClusterSpec cluster, ProvisionLogger logger) { diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/ApplicationSerializer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/ApplicationSerializer.java index 54f60651795..90081ecb604 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/ApplicationSerializer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/ApplicationSerializer.java @@ -1,8 +1,7 @@ // Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.provision.restapi.v2; +package com.yahoo.vespa.hosted.provision.restapi; import com.yahoo.config.provision.ClusterResources; -import com.yahoo.config.provision.NodeResources; import com.yahoo.slime.Cursor; import com.yahoo.slime.Slime; import com.yahoo.vespa.hosted.provision.applications.Application; diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/HostCapacityResponse.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/HostCapacityResponse.java index 488b11e627b..12a29707303 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/HostCapacityResponse.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/HostCapacityResponse.java @@ -1,5 +1,5 @@ -// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.provision.restapi.v2; +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.provision.restapi; import com.yahoo.container.jdisc.HttpRequest; import com.yahoo.container.jdisc.HttpResponse; diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/JobsResponse.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/JobsResponse.java index 4dfdef742d6..3b6de41f142 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/JobsResponse.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/JobsResponse.java @@ -1,5 +1,5 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.provision.restapi.v2; +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.provision.restapi; import com.yahoo.container.jdisc.HttpResponse; import com.yahoo.slime.Cursor; diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/LoadBalancersResponse.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/LoadBalancersResponse.java index 3147d4caded..092426c75d4 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/LoadBalancersResponse.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/LoadBalancersResponse.java @@ -1,5 +1,5 @@ -// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.provision.restapi.v2; +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.provision.restapi; import com.yahoo.config.provision.ApplicationId; import com.yahoo.container.jdisc.HttpRequest; diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/LoadBalancersApiHandler.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/LoadBalancersV1ApiHandler.java index 24aa1fe0b25..f81e3240397 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/LoadBalancersApiHandler.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/LoadBalancersV1ApiHandler.java @@ -1,5 +1,5 @@ -// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.provision.restapi.v2; +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.provision.restapi; import com.yahoo.container.jdisc.HttpRequest; import com.yahoo.container.jdisc.HttpResponse; @@ -15,12 +15,12 @@ import java.util.logging.Level; /** * @author mpolden */ -public class LoadBalancersApiHandler extends LoggingRequestHandler { +public class LoadBalancersV1ApiHandler extends LoggingRequestHandler { private final NodeRepository nodeRepository; @Inject - public LoadBalancersApiHandler(LoggingRequestHandler.Context parentCtx, NodeRepository nodeRepository) { + public LoadBalancersV1ApiHandler(LoggingRequestHandler.Context parentCtx, NodeRepository nodeRepository) { super(parentCtx); this.nodeRepository = nodeRepository; } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/NodeAclResponse.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/NodeAclResponse.java index 9947d2d5108..a7577392fe2 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/NodeAclResponse.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/NodeAclResponse.java @@ -1,5 +1,5 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.provision.restapi.v2; +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.provision.restapi; import com.yahoo.container.jdisc.HttpRequest; import com.yahoo.container.jdisc.HttpResponse; diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/NodePatcher.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/NodePatcher.java index 12b245ede7f..8b5639cc514 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/NodePatcher.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/NodePatcher.java @@ -1,5 +1,5 @@ -// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.provision.restapi.v2; +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.provision.restapi; import com.yahoo.component.Version; import com.yahoo.config.provision.DockerImage; diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/NodeResourcesSerializer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/NodeResourcesSerializer.java index 0d0c9305a92..625ee89434e 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/NodeResourcesSerializer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/NodeResourcesSerializer.java @@ -1,5 +1,5 @@ // Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.provision.restapi.v2; +package com.yahoo.vespa.hosted.provision.restapi; import com.yahoo.config.provision.NodeResources; import com.yahoo.slime.Cursor; diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/NodeSerializer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/NodeSerializer.java index 67f3b293de8..bd65894101c 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/NodeSerializer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/NodeSerializer.java @@ -1,5 +1,5 @@ -// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.provision.restapi.v2; +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.provision.restapi; import com.yahoo.config.provision.NodeResources; import com.yahoo.config.provision.NodeType; diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/NodesResponse.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/NodesResponse.java index c6b92d0a7f5..9d02d2907cc 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/NodesResponse.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/NodesResponse.java @@ -1,5 +1,5 @@ -// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.provision.restapi.v2; +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.provision.restapi; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.ClusterMembership; @@ -53,7 +53,7 @@ class NodesResponse extends HttpResponse { super(200); this.parentUrl = toParentUrl(request); this.nodeParentUrl = toNodeParentUrl(request); - filter = NodesApiHandler.toNodeFilter(request); + filter = NodesV2ApiHandler.toNodeFilter(request); this.recursive = request.getBooleanProperty("recursive"); this.orchestrator = orchestrator.getHostResolver(); this.nodeRepository = nodeRepository; diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/NodesApiHandler.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/NodesV2ApiHandler.java index 6799b68daae..1e8bf622f72 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/NodesApiHandler.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/NodesV2ApiHandler.java @@ -1,5 +1,5 @@ -// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.provision.restapi.v2; +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.provision.restapi; import com.yahoo.component.Version; import com.yahoo.config.provision.ApplicationId; @@ -38,7 +38,7 @@ import com.yahoo.vespa.hosted.provision.node.filter.NodeOsVersionFilter; import com.yahoo.vespa.hosted.provision.node.filter.NodeTypeFilter; import com.yahoo.vespa.hosted.provision.node.filter.ParentHostFilter; import com.yahoo.vespa.hosted.provision.node.filter.StateFilter; -import com.yahoo.vespa.hosted.provision.restapi.v2.NodesResponse.ResponseType; +import com.yahoo.vespa.hosted.provision.restapi.NodesResponse.ResponseType; import com.yahoo.vespa.orchestrator.Orchestrator; import com.yahoo.yolean.Exceptions; @@ -65,15 +65,15 @@ import static com.yahoo.slime.SlimeUtils.optionalString; * * @author bratseth */ -public class NodesApiHandler extends LoggingRequestHandler { +public class NodesV2ApiHandler extends LoggingRequestHandler { private final Orchestrator orchestrator; private final NodeRepository nodeRepository; private final NodeFlavors nodeFlavors; @Inject - public NodesApiHandler(LoggingRequestHandler.Context parentCtx, Orchestrator orchestrator, - NodeRepository nodeRepository, NodeFlavors flavors) { + public NodesV2ApiHandler(LoggingRequestHandler.Context parentCtx, Orchestrator orchestrator, + NodeRepository nodeRepository, NodeFlavors flavors) { super(parentCtx); this.orchestrator = orchestrator; this.nodeRepository = nodeRepository; diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/NotFoundException.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/NotFoundException.java index a2bfba95b8a..664aad939f2 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/NotFoundException.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/NotFoundException.java @@ -1,5 +1,5 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.provision.restapi.v2; +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.provision.restapi; /** * Thrown when a resource is not found diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/UpgradeResponse.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/UpgradeResponse.java index 381a1bc27aa..16858ec6963 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/UpgradeResponse.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/UpgradeResponse.java @@ -1,5 +1,5 @@ -// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.provision.restapi.v2; +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.provision.restapi; import com.yahoo.container.jdisc.HttpResponse; import com.yahoo.slime.Cursor; diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/ContainerConfig.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/ContainerConfig.java index d26accd7a84..d66bb7d6efb 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/ContainerConfig.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/ContainerConfig.java @@ -32,10 +32,10 @@ public class ContainerConfig { " <component id='com.yahoo.vespa.hosted.provision.maintenance.NodeRepositoryMaintenance'/>\n" + " <component id='com.yahoo.vespa.flags.InMemoryFlagSource'/>\n" + " <component id='com.yahoo.config.provision.Zone'/>\n" + - " <handler id='com.yahoo.vespa.hosted.provision.restapi.v2.NodesApiHandler'>\n" + + " <handler id='com.yahoo.vespa.hosted.provision.restapi.NodesV2ApiHandler'>\n" + " <binding>http://*/nodes/v2/*</binding>\n" + " </handler>\n" + - " <handler id='com.yahoo.vespa.hosted.provision.restapi.v2.LoadBalancersApiHandler'>\n" + + " <handler id='com.yahoo.vespa.hosted.provision.restapi.LoadBalancersV1ApiHandler'>\n" + " <binding>http://*/loadbalancers/v1/*</binding>\n" + " </handler>\n" + " <http>\n" + diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/LoadBalancerExpirerTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/LoadBalancerExpirerTest.java index e278cdc992a..a5e96369591 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/LoadBalancerExpirerTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/LoadBalancerExpirerTest.java @@ -155,7 +155,7 @@ public class LoadBalancerExpirerTest { } private void deployApplication(ApplicationId application, boolean activate, ClusterSpec.Id... clusters) { - tester.makeReadyNodes(10, "d-1-4-10"); + tester.makeReadyNodes(10, new NodeResources(1, 4, 10, 0.3)); List<HostSpec> hosts = new ArrayList<>(); for (var cluster : clusters) { hosts.addAll(tester.prepare(application, ClusterSpec.request(ClusterSpec.Type.container, cluster).vespaVersion(Vtag.currentVersion).build(), diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/MultigroupProvisioningTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/MultigroupProvisioningTest.java index 1265961e351..050f3b7e865 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/MultigroupProvisioningTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/MultigroupProvisioningTest.java @@ -109,7 +109,7 @@ public class MultigroupProvisioningTest { } @Test - public void test_one_node_and_group_to_two_with_flavor_migration() { + public void test_one_node_and_group_to_two_with_resource_change() { ProvisioningTester tester = new ProvisioningTester.Builder().zone(new Zone(Environment.perf, RegionName.from("us-east"))).build(); ApplicationId application1 = tester.makeApplicationId(); diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTest.java index 8fc73d68785..463a3ef3fb9 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTest.java @@ -59,7 +59,7 @@ public class ProvisioningTest { ApplicationId application1 = tester.makeApplicationId(); ApplicationId application2 = tester.makeApplicationId(); - tester.makeReadyNodes(21, defaultResources); + tester.makeReadyHosts(21, defaultResources).deployZoneApp(); // deploy SystemState state1 = prepare(application1, 2, 2, 3, 3, defaultResources, tester); @@ -137,7 +137,7 @@ public class ProvisioningTest { public void nodeVersionIsReturnedIfSet() { ProvisioningTester tester = new ProvisioningTester.Builder().zone(new Zone(Environment.dev, RegionName.from("us-east"))).build(); - tester.makeReadyNodes(4, defaultResources, NodeType.host, 1); + tester.makeReadyHosts(4, defaultResources); tester.prepareAndActivateInfraApplication(tester.makeApplicationId(), NodeType.host); // deploy @@ -162,7 +162,7 @@ public class ProvisioningTest { public void dockerImageRepoIsReturnedIfSet() { ProvisioningTester tester = new ProvisioningTester.Builder().zone(new Zone(Environment.dev, RegionName.from("us-east"))).build(); - tester.makeReadyNodes(4, defaultResources, NodeType.host, 1); + tester.makeReadyHosts(4, defaultResources); tester.prepareAndActivateInfraApplication(tester.makeApplicationId(), NodeType.host); // deploy @@ -192,7 +192,8 @@ public class ProvisioningTest { ApplicationId application1 = tester.makeApplicationId(); - tester.makeReadyNodes(24, defaultResources); + tester.makeReadyHosts(24, defaultResources); + tester.deployZoneApp(); // deploy SystemState state1 = prepare(application1, 2, 2, 3, 3, defaultResources, tester); @@ -262,18 +263,18 @@ public class ProvisioningTest { ApplicationId application1 = tester.makeApplicationId(); - tester.makeReadyNodes(12, small); + tester.makeReadyHosts(12, small); + tester.makeReadyHosts(16, large); + tester.deployZoneApp(); // deploy SystemState state1 = prepare(application1, 2, 2, 4, 4, small, tester); tester.activate(application1, state1.allHosts); - // redeploy with reduced size (to cause us to have retired nodes before switching flavor) + // redeploy with reduced size (to cause us to have retired nodes before switching resources) SystemState state2 = prepare(application1, 2, 2, 3, 3, small, tester); tester.activate(application1, state2.allHosts); - tester.makeReadyNodes(16, large); - // redeploy with increased sizes and new flavor SystemState state3 = prepare(application1, 3, 4, 4, 5, large, tester); assertEquals("New nodes are reserved", 16, tester.nodeRepository().getNodes(application1, Node.State.reserved).size()); @@ -292,7 +293,7 @@ public class ProvisioningTest { ApplicationId application1 = tester.makeApplicationId(); - tester.makeReadyNodes(5, defaultResources); + tester.makeReadyHosts(5, defaultResources).deployZoneApp(); // deploy SystemState state1 = prepare(application1, 2, 0, 3, 0, defaultResources, tester); @@ -315,7 +316,7 @@ public class ProvisioningTest { public void dev_deployment_node_size() { ProvisioningTester tester = new ProvisioningTester.Builder().zone(new Zone(Environment.dev, RegionName.from("us-east"))).build(); - tester.makeReadyNodes(4, defaultResources, NodeType.host, 1); + tester.makeReadyHosts(4, defaultResources); tester.prepareAndActivateInfraApplication(tester.makeApplicationId(), NodeType.host); ApplicationId application = tester.makeApplicationId(); @@ -327,8 +328,8 @@ public class ProvisioningTest { @Test public void requested_resources_info_is_retained() { ProvisioningTester tester = new ProvisioningTester.Builder().zone(new Zone(Environment.prod, RegionName.from("us-east"))).build(); + tester.makeReadyHosts(13, defaultResources).deployZoneApp(); - tester.makeReadyNodes(13, defaultResources, NodeType.host, 1); tester.prepareAndActivateInfraApplication(tester.makeApplicationId(), NodeType.host); ApplicationId application = tester.makeApplicationId(); @@ -371,7 +372,7 @@ public class ProvisioningTest { public void deploy_specific_vespa_version() { ProvisioningTester tester = new ProvisioningTester.Builder().zone(new Zone(Environment.dev, RegionName.from("us-east"))).build(); - tester.makeReadyNodes(4, defaultResources, NodeType.host, 1); + tester.makeReadyHosts(4, defaultResources).deployZoneApp(); tester.prepareAndActivateInfraApplication(tester.makeApplicationId(), NodeType.host); ApplicationId application = tester.makeApplicationId(); @@ -384,7 +385,7 @@ public class ProvisioningTest { public void deploy_specific_vespa_version_and_docker_image() { ProvisioningTester tester = new ProvisioningTester.Builder().zone(new Zone(Environment.dev, RegionName.from("us-east"))).build(); - tester.makeReadyNodes(4, defaultResources, NodeType.host, 1); + tester.makeReadyHosts(4, defaultResources).deployZoneApp(); tester.prepareAndActivateInfraApplication(tester.makeApplicationId(), NodeType.host); ApplicationId application = tester.makeApplicationId(); @@ -399,18 +400,69 @@ public class ProvisioningTest { ProvisioningTester tester = new ProvisioningTester.Builder().zone(new Zone(Environment.test, RegionName.from("us-east"))).build(); ApplicationId application = tester.makeApplicationId(); - tester.makeReadyNodes(4, defaultResources); + tester.makeReadyHosts(4, defaultResources).deployZoneApp(); SystemState state = prepare(application, 2, 2, 3, 3, defaultResources, tester); assertEquals(4, state.allHosts.size()); tester.activate(application, state.allHosts); } + @Test + public void test_changing_limits() { + ProvisioningTester tester = new ProvisioningTester.Builder().zone(new Zone(Environment.prod, RegionName.from("us-east"))).build(); + tester.makeReadyHosts(30, new NodeResources(20, 40, 100, 4)).deployZoneApp(); + + ApplicationId app1 = tester.makeApplicationId("app1"); + ClusterSpec cluster1 = ClusterSpec.request(ClusterSpec.Type.content, new ClusterSpec.Id("cluster1")).vespaVersion("7").build(); + + // Initial deployment + tester.activate(app1, cluster1, Capacity.from(resources(4, 2, 2, 10, 20), + resources(8, 4, 4, 20, 40))); + tester.assertNodes("Initial allocation at min", + 4, 2, 2, 10, 20, + app1, cluster1); + + // Move window above current allocation + tester.activate(app1, cluster1, Capacity.from(resources(8, 4, 4, 20, 40), + resources(10, 5, 5, 25, 50))); + tester.assertNodes("New allocation at new min", + 8, 4, 4, 20, 40, + app1, cluster1); + + // Move window below current allocation + tester.activate(app1, cluster1, Capacity.from(resources(4, 2, 2, 10, 20), + resources(6, 3, 3, 15, 25))); + tester.assertNodes("New allocation at new max", + 6, 3, 3, 15, 25, + app1, cluster1); + + // Widening window does not change allocation + tester.activate(app1, cluster1, Capacity.from(resources(2, 1, 1, 5, 15), + resources(8, 4, 4, 20, 30))); + tester.assertNodes("Same allocation", + 6, 3, 3, 15, 25, + app1, cluster1); + + // Changing limits in opposite directions cause a mixture of min and max + tester.activate(app1, cluster1, Capacity.from(resources(2, 1, 10, 30, 5), + resources(4, 2, 14, 40, 10))); + tester.assertNodes("A mix of min and max", + 4, 2, 10, 30, 10, + app1, cluster1); + + // Changing group size + tester.activate(app1, cluster1, Capacity.from(resources(6, 3, 8, 25, 5), + resources(9, 3, 12, 35, 15))); + tester.assertNodes("Groups changed", + 6, 3, 10, 30, 10, + app1, cluster1); + } + @Test(expected = IllegalArgumentException.class) public void prod_deployment_requires_redundancy() { ProvisioningTester tester = new ProvisioningTester.Builder().zone(new Zone(Environment.prod, RegionName.from("us-east"))).build(); ApplicationId application = tester.makeApplicationId(); - tester.makeReadyNodes(10, defaultResources); + tester.makeReadyHosts(10, defaultResources).deployZoneApp(); prepare(application, 1, 2, 3, 3, defaultResources, tester); } @@ -419,7 +471,7 @@ public class ProvisioningTest { ProvisioningTester tester = new ProvisioningTester.Builder().zone(new Zone(Environment.prod, RegionName.from("us-east"))).build(); ApplicationId application = tester.makeApplicationId(); - tester.makeReadyNodes(10, defaultResources); + tester.makeReadyHosts(10, defaultResources).deployZoneApp(); try { prepare(application, 2, 2, 3, 3, new NodeResources(2, 2, 10, 2), tester); @@ -434,7 +486,7 @@ public class ProvisioningTest { public void dev_deployment_flavor() { ProvisioningTester tester = new ProvisioningTester.Builder().zone(new Zone(Environment.dev, RegionName.from("us-east"))).build(); - tester.makeReadyNodes(4, new NodeResources(2, 4, 10, 2), NodeType.host, 1); + tester.makeReadyHosts(4, new NodeResources(2, 4, 10, 2)); tester.prepareAndActivateInfraApplication(tester.makeApplicationId(), NodeType.host); ApplicationId application = tester.makeApplicationId(); @@ -452,7 +504,7 @@ public class ProvisioningTest { ProvisioningTester tester = new ProvisioningTester.Builder().zone(new Zone(Environment.test, RegionName.from("us-east"))).build(); ApplicationId application = tester.makeApplicationId(); - tester.makeReadyNodes(4, large); + tester.makeReadyHosts(4, large).deployZoneApp(); SystemState state = prepare(application, 2, 2, 3, 3, large, tester); assertEquals(4, state.allHosts.size()); tester.activate(application, state.allHosts); @@ -463,7 +515,7 @@ public class ProvisioningTest { ProvisioningTester tester = new ProvisioningTester.Builder().zone(new Zone(Environment.staging, RegionName.from("us-east"))).build(); ApplicationId application = tester.makeApplicationId(); - tester.makeReadyNodes(14, defaultResources); + tester.makeReadyHosts(14, defaultResources).deployZoneApp(); SystemState state = prepare(application, 1, 1, 1, 64, defaultResources, tester); // becomes 1, 1, 1, 1, 6 assertEquals(9, state.allHosts.size()); tester.activate(application, state.allHosts); @@ -473,7 +525,7 @@ public class ProvisioningTest { public void activate_after_reservation_timeout() { ProvisioningTester tester = new ProvisioningTester.Builder().zone(new Zone(Environment.prod, RegionName.from("us-east"))).build(); - tester.makeReadyNodes(10, defaultResources); + tester.makeReadyHosts(10, defaultResources).deployZoneApp(); ApplicationId application = tester.makeApplicationId(); SystemState state = prepare(application, 2, 2, 3, 3, defaultResources, tester); @@ -495,7 +547,7 @@ public class ProvisioningTest { public void out_of_capacity() { ProvisioningTester tester = new ProvisioningTester.Builder().zone(new Zone(Environment.prod, RegionName.from("us-east"))).build(); - tester.makeReadyNodes(9, defaultResources); // need 2+2+3+3=10 + tester.makeReadyHosts(9, defaultResources).deployZoneApp(); // need 2+2+3+3=10 ApplicationId application = tester.makeApplicationId(); try { prepare(application, 2, 2, 3, 3, defaultResources, tester); @@ -512,7 +564,7 @@ public class ProvisioningTest { Environment.prod, RegionName.from("us-east"))).build(); - tester.makeReadyNodes(13, defaultResources); + tester.makeReadyHosts(13, defaultResources).deployZoneApp(); ApplicationId application = tester.makeApplicationId(); try { prepare(application, 2, 2, 6, 3, defaultResources, tester); @@ -530,7 +582,7 @@ public class ProvisioningTest { Environment.prod, RegionName.from("us-east"))).build(); - tester.makeReadyNodes(13, defaultResources); + tester.makeReadyHosts(13, defaultResources).deployZoneApp(); ApplicationId application = tester.makeApplicationId(); prepare(application, 2, 2, 6, 3, defaultResources, tester); } @@ -538,7 +590,7 @@ public class ProvisioningTest { @Test public void out_of_capacity_but_cannot_fail() { ProvisioningTester tester = new ProvisioningTester.Builder().zone(new Zone(Environment.prod, RegionName.from("us-east"))).build(); - tester.makeReadyNodes(4, defaultResources); + tester.makeReadyHosts(4, defaultResources).deployZoneApp(); ApplicationId application = tester.makeApplicationId(); ClusterSpec cluster = ClusterSpec.request(ClusterSpec.Type.content, ClusterSpec.Id.from("music")).vespaVersion("4.5.6").build(); tester.prepare(application, cluster, Capacity.from(new ClusterResources(5, 1, NodeResources.unspecified), false, false)); @@ -572,7 +624,7 @@ public class ProvisioningTest { ApplicationId application = tester.makeApplicationId(); // Create 10 nodes - tester.makeReadyNodes(10, defaultResources); + tester.makeReadyHosts(10, defaultResources).deployZoneApp(); // Allocate 5 nodes ClusterSpec cluster = ClusterSpec.request(ClusterSpec.Type.content, ClusterSpec.Id.from("music")).vespaVersion("4.5.6").build(); tester.activate(application, tester.prepare(application, cluster, capacity)); @@ -603,7 +655,7 @@ public class ProvisioningTest { ApplicationId application1 = tester.makeApplicationId(); - tester.makeReadyNodes(14, defaultResources); + tester.makeReadyHosts(14, defaultResources).deployZoneApp(); // deploy SystemState state1 = prepare(application1, 3, 3, 4, 4, defaultResources, tester); @@ -631,7 +683,7 @@ public class ProvisioningTest { ProvisioningTester tester = new ProvisioningTester.Builder().zone(new Zone(Environment.prod, RegionName.from("us-east"))).build(); ApplicationId application = tester.makeApplicationId(); - tester.makeReadyNodes(10, defaultResources); + tester.makeReadyHosts(10, defaultResources).deployZoneApp(); // Deploy application { @@ -659,7 +711,7 @@ public class ProvisioningTest { ProvisioningTester tester = new ProvisioningTester.Builder().zone(new Zone(Environment.prod, RegionName.from("us-east"))).build(); ApplicationId application = tester.makeApplicationId(); - tester.makeReadyNodes(2, defaultResources); + tester.makeReadyHosts(2, defaultResources).deployZoneApp(); // Deploy fails with out of capacity try { @@ -670,7 +722,7 @@ public class ProvisioningTest { tester.getNodes(application, Node.State.reserved).size()); // Enough nodes become available - tester.makeReadyNodes(2, defaultResources); + tester.makeReadyHosts(2, defaultResources).deployZoneApp(); // Deploy is retried after a few minutes tester.clock().advance(Duration.ofMinutes(2)); @@ -740,7 +792,7 @@ public class ProvisioningTest { var tester = new ProvisioningTester.Builder().zone(new Zone(Environment.prod, RegionName.from("us-east"))).build(); var application = tester.makeApplicationId(); - tester.makeReadyNodes(4, defaultResources); + tester.makeReadyHosts(4, defaultResources).deployZoneApp(); // Application allocates two content nodes initially, with cluster type content ClusterSpec cluster = ClusterSpec.request(ClusterSpec.Type.content, ClusterSpec.Id.from("music")).vespaVersion("1.2.3").build(); @@ -920,4 +972,8 @@ public class ProvisioningTest { .isPresent(); } + private ClusterResources resources(int nodes, int groups, double vcpu, double memory, double disk) { + return new ClusterResources(nodes, groups, new NodeResources(vcpu, memory, disk, 0.1)); + } + } diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTester.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTester.java index a8df47aab1a..cbb15311867 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTester.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTester.java @@ -149,6 +149,10 @@ public class ProvisioningTester { return hosts1; } + public Collection<HostSpec> activate(ApplicationId application, ClusterSpec cluster, Capacity capacity) { + return activate(application, prepare(application, cluster, capacity, true)); + } + public Collection<HostSpec> activate(ApplicationId application, Collection<HostSpec> hosts) { NestedTransaction transaction = new NestedTransaction(); transaction.add(new CuratorTransaction(curator)); @@ -197,6 +201,22 @@ public class ProvisioningTester { } } + /** Assert on the current *non retired* nodes */ + public void assertNodes(String explanation, int nodes, int groups, double vcpu, double memory, double disk, + ApplicationId app, ClusterSpec cluster) { + List<Node> nodeList = nodeRepository.list().owner(app).cluster(cluster.id()).not().retired().asList(); + assertEquals(explanation + ": Node count", + nodes, + nodeList.size()); + assertEquals(explanation + ": Group count", + groups, + nodeList.stream().map(n -> n.allocation().get().membership().cluster().group().get()).distinct().count()); + for (Node node : nodeList) + assertEquals(explanation + ": Resources", + new NodeResources(vcpu, memory, disk, 0.1), + node.flavor().resources()); + } + public void fail(HostSpec host) { int beforeFailCount = nodeRepository.getNode(host.hostname(), Node.State.active).get().status().failCount(); Node failedNode = nodeRepository.fail(host.hostname(), Agent.system, "Failing to unit test"); @@ -241,6 +261,12 @@ public class ProvisioningTester { return makeReadyNodes(n, flavor, NodeType.tenant); } + /** Call deployZoneApp() after this before deploying applications */ + public ProvisioningTester makeReadyHosts(int n, NodeResources resources) { + makeReadyNodes(n, resources, NodeType.host, 5); + return this; + } + public List<Node> makeReadyNodes(int n, NodeResources resources) { return makeReadyNodes(n, resources, NodeType.tenant, 0); } diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/LoadBalancersV1ApiTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/LoadBalancersV1ApiTest.java new file mode 100644 index 00000000000..e9811985b7d --- /dev/null +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/LoadBalancersV1ApiTest.java @@ -0,0 +1,30 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.provision.restapi; + +import com.yahoo.application.container.handler.Request; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; + +public class LoadBalancersV1ApiTest { + + private RestApiTester tester; + + @Before + public void createTester() { + tester = new RestApiTester(); + } + + @After + public void closeTester() { + tester.close(); + } + + @Test + public void test_load_balancers() throws Exception { + tester.assertFile(new Request("http://localhost:8080/loadbalancers/v1/"), "load-balancers.json"); + tester.assertFile(new Request("http://localhost:8080/loadbalancers/v1/?application=tenant4.application4.instance4"), "load-balancers-single.json"); + tester.assertResponse(new Request("http://localhost:8080/loadbalancers/v1/?application=tenant.nonexistent.default"), "{\"loadBalancers\":[]}"); + } + +} diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/NodeSerializerTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/NodeSerializerTest.java index ed2aae64b5f..9bc27c0cc04 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/NodeSerializerTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/NodeSerializerTest.java @@ -1,9 +1,10 @@ -// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.provision.restapi.v2; +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.provision.restapi; import com.yahoo.config.provision.NodeType; import com.yahoo.vespa.hosted.provision.Node; +import com.yahoo.vespa.hosted.provision.restapi.NodeSerializer; import org.junit.Test; import static org.junit.Assert.assertEquals; diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/RestApiTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/NodesV2ApiTest.java index 838a39ba01e..3e23b424675 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/RestApiTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/NodesV2ApiTest.java @@ -1,5 +1,5 @@ -// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.provision.restapi.v2; +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.provision.restapi; import com.yahoo.application.Networking; import com.yahoo.application.container.JDisc; @@ -40,20 +40,18 @@ import static org.junit.Assert.assertFalse; * * @author bratseth */ -public class RestApiTest { +public class NodesV2ApiTest { - private final static String responsesPath = "src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/"; - - private JDisc container; + private RestApiTester tester; @Before - public void startContainer() { - container = JDisc.fromServicesXml(ContainerConfig.servicesXmlV2(0), Networking.disable); + public void createTester() { + tester = new RestApiTester(); } @After - public void stopContainer() { - if (container != null) container.close(); + public void closeTester() { + tester.close(); } /** This test gives examples of the node requests that can be made to nodes/v2 */ @@ -82,8 +80,8 @@ public class RestApiTest { new byte[0], Request.Method.POST)); assertRestart(11, new Request("http://localhost:8080/nodes/v2/command/restart", new byte[0], Request.Method.POST)); - assertResponseContains(new Request("http://localhost:8080/nodes/v2/node/host2.yahoo.com"), - "\"restartGeneration\":3"); + tester.assertResponseContains(new Request("http://localhost:8080/nodes/v2/node/host2.yahoo.com"), + "\"restartGeneration\":3"); // POST reboot command assertReboot(12, new Request("http://localhost:8080/nodes/v2/command/reboot?state=failed%20active", @@ -92,28 +90,28 @@ public class RestApiTest { new byte[0], Request.Method.POST)); assertReboot(19, new Request("http://localhost:8080/nodes/v2/command/reboot", new byte[0], Request.Method.POST)); - assertResponseContains(new Request("http://localhost:8080/nodes/v2/node/host2.yahoo.com"), - "\"rebootGeneration\":4"); + tester.assertResponseContains(new Request("http://localhost:8080/nodes/v2/node/host2.yahoo.com"), + "\"rebootGeneration\":4"); // POST new nodes assertResponse(new Request("http://localhost:8080/nodes/v2/node", ("[" + asNodeJson("host8.yahoo.com", "default", "127.0.8.1") + "," + // test with only 1 ip address - asNodeJson("host9.yahoo.com", "large-variant", "127.0.9.1", "::9:1") + "," + - asHostJson("parent2.yahoo.com", "large-variant", Optional.of(TenantName.from("myTenant")), "127.0.127.1", "::127:1") + "," + - asDockerNodeJson("host11.yahoo.com", "parent.host.yahoo.com", "::11") + "]"). + asNodeJson("host9.yahoo.com", "large-variant", "127.0.9.1", "::9:1") + "," + + asHostJson("parent2.yahoo.com", "large-variant", Optional.of(TenantName.from("myTenant")), "127.0.127.1", "::127:1") + "," + + asDockerNodeJson("host11.yahoo.com", "parent.host.yahoo.com", "::11") + "]"). getBytes(StandardCharsets.UTF_8), Request.Method.POST), - "{\"message\":\"Added 4 nodes to the provisioned state\"}"); + "{\"message\":\"Added 4 nodes to the provisioned state\"}"); assertFile(new Request("http://localhost:8080/nodes/v2/node/host8.yahoo.com"), "node8.json"); assertFile(new Request("http://localhost:8080/nodes/v2/node/host9.yahoo.com"), "node9.json"); assertFile(new Request("http://localhost:8080/nodes/v2/node/host11.yahoo.com"), "node11.json"); assertFile(new Request("http://localhost:8080/nodes/v2/node/parent2.yahoo.com"), "parent2.json"); // POST duplicate node - assertResponse(new Request("http://localhost:8080/nodes/v2/node", - ("[" + asNodeJson("host8.yahoo.com", "default", "127.0.254.8") + "]").getBytes(StandardCharsets.UTF_8), - Request.Method.POST), 400, - "{\"error-code\":\"BAD_REQUEST\",\"message\":\"Cannot add provisioned host host8.yahoo.com: A node with this name already exists\"}"); + tester.assertResponse(new Request("http://localhost:8080/nodes/v2/node", + ("[" + asNodeJson("host8.yahoo.com", "default", "127.0.254.8") + "]").getBytes(StandardCharsets.UTF_8), + Request.Method.POST), 400, + "{\"error-code\":\"BAD_REQUEST\",\"message\":\"Cannot add provisioned host host8.yahoo.com: A node with this name already exists\"}"); // DELETE a provisioned node assertResponse(new Request("http://localhost:8080/nodes/v2/node/host9.yahoo.com", @@ -127,8 +125,8 @@ public class RestApiTest { assertResponse(new Request("http://localhost:8080/nodes/v2/state/ready/host8.yahoo.com", new byte[0], Request.Method.PUT), "{\"message\":\"Moved host8.yahoo.com to ready\"}"); - assertResponseContains(new Request("http://localhost:8080/nodes/v2/node/host8.yahoo.com"), - "\"state\":\"ready\""); + tester.assertResponseContains(new Request("http://localhost:8080/nodes/v2/node/host8.yahoo.com"), + "\"state\":\"ready\""); // calling ready again is a noop: assertResponse(new Request("http://localhost:8080/nodes/v2/state/ready/host8.yahoo.com", new byte[0], Request.Method.PUT), @@ -138,8 +136,8 @@ public class RestApiTest { assertResponse(new Request("http://localhost:8080/nodes/v2/state/failed/host2.yahoo.com", new byte[0], Request.Method.PUT), "{\"message\":\"Moved host2.yahoo.com to failed\"}"); - assertResponseContains(new Request("http://localhost:8080/nodes/v2/node/host2.yahoo.com"), - "\"state\":\"failed\""); + tester.assertResponseContains(new Request("http://localhost:8080/nodes/v2/node/host2.yahoo.com"), + "\"state\":\"failed\""); // ... and put it back in active (after fixing). This is useful to restore data when multiple nodes fail. assertResponse(new Request("http://localhost:8080/nodes/v2/state/active/host2.yahoo.com", new byte[0], Request.Method.PUT), @@ -149,8 +147,8 @@ public class RestApiTest { assertResponse(new Request("http://localhost:8080/nodes/v2/state/parked/host8.yahoo.com", new byte[0], Request.Method.PUT), "{\"message\":\"Moved host8.yahoo.com to parked\"}"); - assertResponseContains(new Request("http://localhost:8080()/nodes/v2/node/host8.yahoo.com"), - "\"state\":\"parked\""); + tester.assertResponseContains(new Request("http://localhost:8080()/nodes/v2/node/host8.yahoo.com"), + "\"state\":\"parked\""); // ... and delete it assertResponse(new Request("http://localhost:8080/nodes/v2/node/host8.yahoo.com", new byte[0], Request.Method.DELETE), @@ -160,8 +158,8 @@ public class RestApiTest { assertResponse(new Request("http://localhost:8080/nodes/v2/state/failed/test-node-pool-102-2", new byte[0], Request.Method.PUT), "{\"message\":\"Moved test-node-pool-102-2 to failed\"}"); - assertResponseContains(new Request("http://localhost:8080/nodes/v2/node/test-node-pool-102-2"), - "\"state\":\"failed\""); + tester.assertResponseContains(new Request("http://localhost:8080/nodes/v2/node/test-node-pool-102-2"), + "\"state\":\"failed\""); // ... and deallocate it such that it moves to dirty and is recycled assertResponse(new Request("http://localhost:8080/nodes/v2/state/dirty/test-node-pool-102-2", new byte[0], Request.Method.PUT), @@ -172,8 +170,8 @@ public class RestApiTest { new byte[0], Request.Method.PUT), "{\"message\":\"Moved test-node-pool-102-2 to ready\"}"); - assertResponse(new Request("http://localhost:8080/nodes/v2/node/test-node-pool-102-2", new byte[0], Request.Method.GET), - 404, "{\"error-code\":\"NOT_FOUND\",\"message\":\"No node with hostname 'test-node-pool-102-2'\"}"); + tester.assertResponse(new Request("http://localhost:8080/nodes/v2/node/test-node-pool-102-2", new byte[0], Request.Method.GET), + 404, "{\"error-code\":\"NOT_FOUND\",\"message\":\"No node with hostname 'test-node-pool-102-2'\"}"); // Put a host in failed and make sure it's children are also failed assertResponse(new Request("http://localhost:8080/nodes/v2/state/failed/dockerhost1.yahoo.com", new byte[0], Request.Method.PUT), @@ -218,14 +216,14 @@ public class RestApiTest { assertResponse(new Request("http://localhost:8080/nodes/v2/node/dockerhost1.yahoo.com", Utf8.toBytes("{\"wantToDeprovision\": true}"), Request.Method.PATCH), "{\"message\":\"Updated dockerhost1.yahoo.com\"}"); - assertResponseContains(new Request("http://localhost:8080/nodes/v2/node/dockerhost1.yahoo.com"), "\"modelName\":\"foo\""); + tester.assertResponseContains(new Request("http://localhost:8080/nodes/v2/node/dockerhost1.yahoo.com"), "\"modelName\":\"foo\""); assertResponse(new Request("http://localhost:8080/nodes/v2/node/dockerhost1.yahoo.com", Utf8.toBytes("{\"modelName\": null}"), Request.Method.PATCH), "{\"message\":\"Updated dockerhost1.yahoo.com\"}"); - assertPartialResponse(new Request("http://localhost:8080/nodes/v2/node/dockerhost1.yahoo.com"), "modelName", false); - container.handleRequest((new Request("http://localhost:8080/nodes/v2/upgrade/tenant", Utf8.toBytes("{\"dockerImage\": \"docker.domain.tld/my/image\"}"), Request.Method.PATCH))); + tester.assertPartialResponse(new Request("http://localhost:8080/nodes/v2/node/dockerhost1.yahoo.com"), "modelName", false); + tester.container().handleRequest((new Request("http://localhost:8080/nodes/v2/upgrade/tenant", Utf8.toBytes("{\"dockerImage\": \"docker.domain.tld/my/image\"}"), Request.Method.PATCH))); - ((OrchestratorMock) container.components().getComponent(OrchestratorMock.class.getName())) + ((OrchestratorMock) tester.container().components().getComponent(OrchestratorMock.class.getName())) .suspend(new HostName("host4.yahoo.com")); assertFile(new Request("http://localhost:8080/nodes/v2/node/host4.yahoo.com"), "node4-after-changes.json"); @@ -260,10 +258,10 @@ public class RestApiTest { "{\"message\":\"Executed job 'PeriodicApplicationMaintainer'\"}"); // POST run of unknown maintenance job - assertResponse(new Request("http://localhost:8080/nodes/v2/maintenance/run/foo", - new byte[0], Request.Method.POST), - 400, - "{\"error-code\":\"BAD_REQUEST\",\"message\":\"No such job 'foo'\"}"); + tester.assertResponse(new Request("http://localhost:8080/nodes/v2/maintenance/run/foo", + new byte[0], Request.Method.POST), + 400, + "{\"error-code\":\"BAD_REQUEST\",\"message\":\"No such job 'foo'\"}"); } @Test @@ -279,7 +277,8 @@ public class RestApiTest { Request req = new Request("http://localhost:8080/nodes/v2/node/host4.yahoo.com", Utf8.toBytes("{\"currentRestartGeneration\": 1}"), Request.Method.POST); req.getHeaders().add("X-HTTP-Method-Override", "GET"); - assertResponse(req, 400, "{\"error-code\":\"BAD_REQUEST\",\"message\":\"Illegal X-HTTP-Method-Override header for POST request. Accepts 'PATCH' but got 'GET'\"}"); + tester.assertResponse(req, 400, + "{\"error-code\":\"BAD_REQUEST\",\"message\":\"Illegal X-HTTP-Method-Override header for POST request. Accepts 'PATCH' but got 'GET'\"}"); } @Test @@ -307,56 +306,56 @@ public class RestApiTest { ("[" + asNodeJson("host-with-ip.yahoo.com", "default", "foo") + "]"). getBytes(StandardCharsets.UTF_8), Request.Method.POST); - assertResponse(req, 400, "{\"error-code\":\"BAD_REQUEST\",\"message\":\"Found one or more invalid addresses in [foo]: 'foo' is not an IP string literal.\"}"); + tester.assertResponse(req, 400, "{\"error-code\":\"BAD_REQUEST\",\"message\":\"Found one or more invalid addresses in [foo]: 'foo' is not an IP string literal.\"}"); // Attempt to POST tenant node with already assigned IP - assertResponse(new Request("http://localhost:8080/nodes/v2/node", - "[" + asNodeJson("tenant-node-foo.yahoo.com", "default", "127.0.1.1") + "]", - Request.Method.POST), 400, - "{\"error-code\":\"BAD_REQUEST\",\"message\":\"Cannot assign [127.0.1.1] to tenant-node-foo.yahoo.com: [127.0.1.1] already assigned to host1.yahoo.com\"}"); + tester.assertResponse(new Request("http://localhost:8080/nodes/v2/node", + "[" + asNodeJson("tenant-node-foo.yahoo.com", "default", "127.0.1.1") + "]", + Request.Method.POST), 400, + "{\"error-code\":\"BAD_REQUEST\",\"message\":\"Cannot assign [127.0.1.1] to tenant-node-foo.yahoo.com: [127.0.1.1] already assigned to host1.yahoo.com\"}"); // Attempt to PATCH existing tenant node with already assigned IP - assertResponse(new Request("http://localhost:8080/nodes/v2/node/test-node-pool-102-2", - "{\"ipAddresses\": [\"127.0.2.1\"]}", - Request.Method.PATCH), 400, - "{\"error-code\":\"BAD_REQUEST\",\"message\":\"Could not set field 'ipAddresses': Cannot assign [127.0.2.1] to test-node-pool-102-2: [127.0.2.1] already assigned to host2.yahoo.com\"}"); + tester.assertResponse(new Request("http://localhost:8080/nodes/v2/node/test-node-pool-102-2", + "{\"ipAddresses\": [\"127.0.2.1\"]}", + Request.Method.PATCH), 400, + "{\"error-code\":\"BAD_REQUEST\",\"message\":\"Could not set field 'ipAddresses': Cannot assign [127.0.2.1] to test-node-pool-102-2: [127.0.2.1] already assigned to host2.yahoo.com\"}"); // Attempt to POST host node with already assigned IP - assertResponse(new Request("http://localhost:8080/nodes/v2/node", - "[" + asHostJson("host200.yahoo.com", "default", Optional.empty(), "127.0.2.1") + "]", - Request.Method.POST), 400, + tester.assertResponse(new Request("http://localhost:8080/nodes/v2/node", + "[" + asHostJson("host200.yahoo.com", "default", Optional.empty(), "127.0.2.1") + "]", + Request.Method.POST), 400, "{\"error-code\":\"BAD_REQUEST\",\"message\":\"Cannot assign [127.0.2.1] to host200.yahoo.com: [127.0.2.1] already assigned to host2.yahoo.com\"}"); // Attempt to PATCH host node with IP in the pool of another node - assertResponse(new Request("http://localhost:8080/nodes/v2/node/dockerhost1.yahoo.com", - "{\"ipAddresses\": [\"::104:3\"]}", - Request.Method.PATCH), 400, + tester.assertResponse(new Request("http://localhost:8080/nodes/v2/node/dockerhost1.yahoo.com", + "{\"ipAddresses\": [\"::104:3\"]}", + Request.Method.PATCH), 400, "{\"error-code\":\"BAD_REQUEST\",\"message\":\"Could not set field 'ipAddresses': Cannot assign [::100:4, ::100:3, ::100:2, ::104:3] to dockerhost1.yahoo.com: [::104:3] already assigned to dockerhost5.yahoo.com\"}"); // Node types running a single container can share their IP address with child node - assertResponse(new Request("http://localhost:8080/nodes/v2/node", - "[" + asNodeJson("cfghost42.yahoo.com", NodeType.confighost, "default", Optional.empty(), "127.0.42.1") + "]", - Request.Method.POST), 200, + tester.assertResponse(new Request("http://localhost:8080/nodes/v2/node", + "[" + asNodeJson("cfghost42.yahoo.com", NodeType.confighost, "default", Optional.empty(), "127.0.42.1") + "]", + Request.Method.POST), 200, "{\"message\":\"Added 1 nodes to the provisioned state\"}"); - assertResponse(new Request("http://localhost:8080/nodes/v2/node", - "[" + asDockerNodeJson("cfg42.yahoo.com", NodeType.config, "cfghost42.yahoo.com", "127.0.42.1") + "]", - Request.Method.POST), 200, + tester.assertResponse(new Request("http://localhost:8080/nodes/v2/node", + "[" + asDockerNodeJson("cfg42.yahoo.com", NodeType.config, "cfghost42.yahoo.com", "127.0.42.1") + "]", + Request.Method.POST), 200, "{\"message\":\"Added 1 nodes to the provisioned state\"}"); // ... but cannot share with child node of wrong type - assertResponse(new Request("http://localhost:8080/nodes/v2/node", - "[" + asDockerNodeJson("proxy42.yahoo.com", NodeType.proxy, "cfghost42.yahoo.com", "127.0.42.1") + "]", - Request.Method.POST), 400, + tester.assertResponse(new Request("http://localhost:8080/nodes/v2/node", + "[" + asDockerNodeJson("proxy42.yahoo.com", NodeType.proxy, "cfghost42.yahoo.com", "127.0.42.1") + "]", + Request.Method.POST), 400, "{\"error-code\":\"BAD_REQUEST\",\"message\":\"Cannot assign [127.0.42.1] to proxy42.yahoo.com: [127.0.42.1] already assigned to cfg42.yahoo.com\"}"); // ... nor with child node on different host - assertResponse(new Request("http://localhost:8080/nodes/v2/node", - "[" + asNodeJson("cfghost43.yahoo.com", NodeType.confighost, "default", Optional.empty(), "127.0.43.1") + "]", - Request.Method.POST), 200, + tester.assertResponse(new Request("http://localhost:8080/nodes/v2/node", + "[" + asNodeJson("cfghost43.yahoo.com", NodeType.confighost, "default", Optional.empty(), "127.0.43.1") + "]", + Request.Method.POST), 200, "{\"message\":\"Added 1 nodes to the provisioned state\"}"); - assertResponse(new Request("http://localhost:8080/nodes/v2/node/cfg42.yahoo.com", - "{\"ipAddresses\": [\"127.0.43.1\"]}", - Request.Method.PATCH), 400, + tester.assertResponse(new Request("http://localhost:8080/nodes/v2/node/cfg42.yahoo.com", + "{\"ipAddresses\": [\"127.0.43.1\"]}", + Request.Method.PATCH), 400, "{\"error-code\":\"BAD_REQUEST\",\"message\":\"Could not set field 'ipAddresses': Cannot assign [127.0.43.1] to cfg42.yahoo.com: [127.0.43.1] already assigned to cfghost43.yahoo.com\"}"); } @@ -385,10 +384,10 @@ public class RestApiTest { Utf8.toBytes("{\"reports\":{\"diskSpace\":{\"createdMillis\":2,\"description\":\"" + msg + "\",\"type\": \"HARD_FAIL\"}}}"), Request.Method.PATCH), "{\"message\":\"Updated host12.yahoo.com\"}"); - assertResponse(new Request("http://localhost:8080/nodes/v2/state/ready/host12.yahoo.com", new byte[0], Request.Method.PUT), - 400, - "{\"error-code\":\"BAD_REQUEST\",\"message\":\"provisioned host host12.yahoo.com cannot be readied because it has " + - "hard failures: [diskSpace reported 1970-01-01T00:00:00.002Z: " + msg + "]\"}"); + tester.assertResponse(new Request("http://localhost:8080/nodes/v2/state/ready/host12.yahoo.com", new byte[0], Request.Method.PUT), + 400, + "{\"error-code\":\"BAD_REQUEST\",\"message\":\"provisioned host host12.yahoo.com cannot be readied because it has " + + "hard failures: [diskSpace reported 1970-01-01T00:00:00.002Z: " + msg + "]\"}"); } @Test @@ -404,13 +403,13 @@ public class RestApiTest { assertResponse(new Request("http://localhost:8080/nodes/v2/state/dirty/foo.yahoo.com", new byte[0], Request.Method.PUT), "{\"message\":\"Moved foo.yahoo.com to dirty\"}"); - assertResponseContains(new Request("http://localhost:8080/nodes/v2/node/foo.yahoo.com"), - "\"rebootGeneration\":1"); + tester.assertResponseContains(new Request("http://localhost:8080/nodes/v2/node/foo.yahoo.com"), + "\"rebootGeneration\":1"); assertResponse(new Request("http://localhost:8080/nodes/v2/node/foo.yahoo.com", Utf8.toBytes("{\"currentRebootGeneration\": 42}"), Request.Method.PATCH), "{\"message\":\"Updated foo.yahoo.com\"}"); - assertResponseContains(new Request("http://localhost:8080/nodes/v2/node/foo.yahoo.com"), - "\"rebootGeneration\":1"); + tester.assertResponseContains(new Request("http://localhost:8080/nodes/v2/node/foo.yahoo.com"), + "\"rebootGeneration\":1"); } @Test @@ -436,21 +435,21 @@ public class RestApiTest { @Test public void test_invalid_requests() throws Exception { - assertResponse(new Request("http://localhost:8080/nodes/v2/node/node-does-not-exist", - new byte[0], Request.Method.GET), + tester.assertResponse(new Request("http://localhost:8080/nodes/v2/node/node-does-not-exist", + new byte[0], Request.Method.GET), 404, "{\"error-code\":\"NOT_FOUND\",\"message\":\"No node with hostname 'node-does-not-exist'\"}"); // Attempt to fail and ready an allocated node without going through dirty - assertResponse(new Request("http://localhost:8080/nodes/v2/state/failed/node-does-not-exist", - new byte[0], Request.Method.PUT), + tester.assertResponse(new Request("http://localhost:8080/nodes/v2/state/failed/node-does-not-exist", + new byte[0], Request.Method.PUT), 404, "{\"error-code\":\"NOT_FOUND\",\"message\":\"Could not move node-does-not-exist to failed: Node not found\"}"); // Attempt to fail and ready an allocated node without going through dirty assertResponse(new Request("http://localhost:8080/nodes/v2/state/failed/host1.yahoo.com", new byte[0], Request.Method.PUT), "{\"message\":\"Moved host1.yahoo.com to failed\"}"); - assertResponse(new Request("http://localhost:8080/nodes/v2/state/ready/host1.yahoo.com", - new byte[0], Request.Method.PUT), + tester.assertResponse(new Request("http://localhost:8080/nodes/v2/state/ready/host1.yahoo.com", + new byte[0], Request.Method.PUT), 400, "{\"error-code\":\"BAD_REQUEST\",\"message\":\"Cannot make failed host host1.yahoo.com allocated to tenant1.application1.instance1 as 'container/id1/0/0' available for new allocation as it is not in state [dirty]\"}"); @@ -466,8 +465,8 @@ public class RestApiTest { assertResponse(new Request("http://localhost:8080/nodes/v2/state/parked/host2.yahoo.com", new byte[0], Request.Method.PUT), "{\"message\":\"Moved host2.yahoo.com to parked\"}"); - assertResponse(new Request("http://localhost:8080/nodes/v2/state/ready/host2.yahoo.com", - new byte[0], Request.Method.PUT), + tester.assertResponse(new Request("http://localhost:8080/nodes/v2/state/ready/host2.yahoo.com", + new byte[0], Request.Method.PUT), 400, "{\"error-code\":\"BAD_REQUEST\",\"message\":\"Cannot make parked host host2.yahoo.com allocated to tenant2.application2.instance2 as 'content/id2/0/0' available for new allocation as it is not in state [dirty]\"}"); // (... while dirty then ready works (the ready move will be initiated by node maintenance)) assertResponse(new Request("http://localhost:8080/nodes/v2/state/dirty/host2.yahoo.com", @@ -478,42 +477,42 @@ public class RestApiTest { "{\"message\":\"Moved host2.yahoo.com to ready\"}"); // Attempt to DELETE a node which has been removed - assertResponse(new Request("http://localhost:8080/nodes/v2/node/host2.yahoo.com", - new byte[0], Request.Method.DELETE), + tester.assertResponse(new Request("http://localhost:8080/nodes/v2/node/host2.yahoo.com", + new byte[0], Request.Method.DELETE), 404, "{\"error-code\":\"NOT_FOUND\",\"message\":\"No node with hostname 'host2.yahoo.com'\"}"); // Attempt to DELETE allocated node - assertResponse(new Request("http://localhost:8080/nodes/v2/node/host4.yahoo.com", - new byte[0], Request.Method.DELETE), + tester.assertResponse(new Request("http://localhost:8080/nodes/v2/node/host4.yahoo.com", + new byte[0], Request.Method.DELETE), 400, "{\"error-code\":\"BAD_REQUEST\",\"message\":\"active child node host4.yahoo.com allocated to tenant3.application3.instance3 as 'content/id3/0/0' is currently allocated and cannot be removed\"}"); // PUT current restart generation with string instead of long - assertResponse(new Request("http://localhost:8080/nodes/v2/node/host4.yahoo.com", - Utf8.toBytes("{\"currentRestartGeneration\": \"1\"}"), Request.Method.PATCH), + tester.assertResponse(new Request("http://localhost:8080/nodes/v2/node/host4.yahoo.com", + Utf8.toBytes("{\"currentRestartGeneration\": \"1\"}"), Request.Method.PATCH), 400, "{\"error-code\":\"BAD_REQUEST\",\"message\":\"Could not set field 'currentRestartGeneration': Expected a LONG value, got a STRING\"}"); // PUT flavor with long instead of string - assertResponse(new Request("http://localhost:8080/nodes/v2/node/host4.yahoo.com", - Utf8.toBytes("{\"flavor\": 1}"), Request.Method.PATCH), + tester.assertResponse(new Request("http://localhost:8080/nodes/v2/node/host4.yahoo.com", + Utf8.toBytes("{\"flavor\": 1}"), Request.Method.PATCH), 400, "{\"error-code\":\"BAD_REQUEST\",\"message\":\"Could not set field 'flavor': Expected a STRING value, got a LONG\"}"); // Attempt to set nonexisting node to active - assertResponse(new Request("http://localhost:8080/nodes/v2/state/active/host2.yahoo.com", - new byte[0], Request.Method.PUT), 404, - "{\"error-code\":\"NOT_FOUND\",\"message\":\"Could not move host2.yahoo.com to active: Node not found\"}"); + tester.assertResponse(new Request("http://localhost:8080/nodes/v2/state/active/host2.yahoo.com", + new byte[0], Request.Method.PUT), 404, + "{\"error-code\":\"NOT_FOUND\",\"message\":\"Could not move host2.yahoo.com to active: Node not found\"}"); // Attempt to POST duplicate nodes - assertResponse(new Request("http://localhost:8080/nodes/v2/node", - ("[" + asNodeJson("host8.yahoo.com", "default", "127.0.254.1", "::254:1") + "," + - asNodeJson("host8.yahoo.com", "large-variant", "127.0.253.1", "::253:1") + "]").getBytes(StandardCharsets.UTF_8), - Request.Method.POST), 400, - "{\"error-code\":\"BAD_REQUEST\",\"message\":\"Cannot add nodes: provisioned host host8.yahoo.com is duplicated in the argument list\"}"); + tester.assertResponse(new Request("http://localhost:8080/nodes/v2/node", + ("[" + asNodeJson("host8.yahoo.com", "default", "127.0.254.1", "::254:1") + "," + + asNodeJson("host8.yahoo.com", "large-variant", "127.0.253.1", "::253:1") + "]").getBytes(StandardCharsets.UTF_8), + Request.Method.POST), 400, + "{\"error-code\":\"BAD_REQUEST\",\"message\":\"Cannot add nodes: provisioned host host8.yahoo.com is duplicated in the argument list\"}"); // Attempt to PATCH field not relevant for child node - assertResponse(new Request("http://localhost:8080/nodes/v2/node/test-node-pool-102-2", - Utf8.toBytes("{\"modelName\": \"foo\"}"), Request.Method.PATCH), - 400, - "{\"error-code\":\"BAD_REQUEST\",\"message\":\"Could not set field 'modelName': A child node cannot have model name set\"}"); + tester.assertResponse(new Request("http://localhost:8080/nodes/v2/node/test-node-pool-102-2", + Utf8.toBytes("{\"modelName\": \"foo\"}"), Request.Method.PATCH), + 400, + "{\"error-code\":\"BAD_REQUEST\",\"message\":\"Could not set field 'modelName': A child node cannot have model name set\"}"); } @Test @@ -531,15 +530,16 @@ public class RestApiTest { Request.Method.PATCH), "{\"message\":\"Updated host4.yahoo.com\"}"); - assertResponse(new Request("http://localhost:8080/nodes/v2/node/doesnotexist.yahoo.com", - Utf8.toBytes("{\"currentRestartGeneration\": 1}"), - Request.Method.PATCH), - 404, "{\"error-code\":\"NOT_FOUND\",\"message\":\"No node found with hostname doesnotexist.yahoo.com\"}"); + tester.assertResponse(new Request("http://localhost:8080/nodes/v2/node/doesnotexist.yahoo.com", + Utf8.toBytes("{\"currentRestartGeneration\": 1}"), + Request.Method.PATCH), + 404, "{\"error-code\":\"NOT_FOUND\",\"message\":\"No node found with hostname doesnotexist.yahoo.com\"}"); - assertResponse(new Request("http://localhost:8080/nodes/v2/node/host5.yahoo.com", - Utf8.toBytes("{\"currentRestartGeneration\": 1}"), - Request.Method.PATCH), - 400, "{\"error-code\":\"BAD_REQUEST\",\"message\":\"Could not set field 'currentRestartGeneration': Node is not allocated\"}"); + tester.assertResponse(new Request("http://localhost:8080/nodes/v2/node/host5.yahoo.com", + Utf8.toBytes("{\"currentRestartGeneration\": 1}"), + Request.Method.PATCH), + 400, + "{\"error-code\":\"BAD_REQUEST\",\"message\":\"Could not set field 'currentRestartGeneration': Node is not allocated\"}"); } @Test @@ -584,25 +584,25 @@ public class RestApiTest { assertFile(new Request("http://localhost:8080/nodes/v2/node/host6.yahoo.com"), "node6-reports.json"); // Patching with an empty reports is no-op - assertResponse(new Request("http://localhost:8080/nodes/v2/node/host6.yahoo.com", - Utf8.toBytes("{\"reports\": {}}"), - Request.Method.PATCH), - 200, - "{\"message\":\"Updated host6.yahoo.com\"}"); + tester.assertResponse(new Request("http://localhost:8080/nodes/v2/node/host6.yahoo.com", + Utf8.toBytes("{\"reports\": {}}"), + Request.Method.PATCH), + 200, + "{\"message\":\"Updated host6.yahoo.com\"}"); assertFile(new Request("http://localhost:8080/nodes/v2/node/host6.yahoo.com"), "node6-reports.json"); // Patching existing report overwrites - assertResponse(new Request("http://localhost:8080/nodes/v2/node/host6.yahoo.com", - Utf8.toBytes("{" + - " \"reports\": {" + - " \"actualCpuCores\": {" + - " \"createdMillis\": 3 " + - " }" + - " }" + - "}"), - Request.Method.PATCH), - 200, - "{\"message\":\"Updated host6.yahoo.com\"}"); + tester.assertResponse(new Request("http://localhost:8080/nodes/v2/node/host6.yahoo.com", + Utf8.toBytes("{" + + " \"reports\": {" + + " \"actualCpuCores\": {" + + " \"createdMillis\": 3 " + + " }" + + " }" + + "}"), + Request.Method.PATCH), + 200, + "{\"message\":\"Updated host6.yahoo.com\"}"); assertFile(new Request("http://localhost:8080/nodes/v2/node/host6.yahoo.com"), "node6-reports-2.json"); // Clearing one report @@ -645,26 +645,26 @@ public class RestApiTest { "{\"versions\":{\"config\":\"6.123.456\",\"confighost\":\"6.123.456\",\"controller\":\"6.123.456\"},\"osVersions\":{},\"dockerImages\":{}}"); // Setting version for unsupported node type fails - assertResponse(new Request("http://localhost:8080/nodes/v2/upgrade/tenant", - Utf8.toBytes("{\"version\": \"6.123.456\"}"), - Request.Method.PATCH), - 400, - "{\"error-code\":\"BAD_REQUEST\",\"message\":\"Target version for type tenant is not allowed\"}"); + tester.assertResponse(new Request("http://localhost:8080/nodes/v2/upgrade/tenant", + Utf8.toBytes("{\"version\": \"6.123.456\"}"), + Request.Method.PATCH), + 400, + "{\"error-code\":\"BAD_REQUEST\",\"message\":\"Target version for type tenant is not allowed\"}"); // Omitting version field fails - assertResponse(new Request("http://localhost:8080/nodes/v2/upgrade/confighost", - Utf8.toBytes("{}"), - Request.Method.PATCH), - 400, - "{\"error-code\":\"BAD_REQUEST\",\"message\":\"At least one of 'version', 'osVersion' or 'dockerImage' must be set\"}"); + tester.assertResponse(new Request("http://localhost:8080/nodes/v2/upgrade/confighost", + Utf8.toBytes("{}"), + Request.Method.PATCH), + 400, + "{\"error-code\":\"BAD_REQUEST\",\"message\":\"At least one of 'version', 'osVersion' or 'dockerImage' must be set\"}"); // Downgrade without force fails - assertResponse(new Request("http://localhost:8080/nodes/v2/upgrade/confighost", - Utf8.toBytes("{\"version\": \"6.123.1\"}"), - Request.Method.PATCH), - 400, - "{\"error-code\":\"BAD_REQUEST\",\"message\":\"Cannot downgrade version without setting 'force'. " + - "Current target version: 6.123.456, attempted to set target version: 6.123.1\"}"); + tester.assertResponse(new Request("http://localhost:8080/nodes/v2/upgrade/confighost", + Utf8.toBytes("{\"version\": \"6.123.1\"}"), + Request.Method.PATCH), + 400, + "{\"error-code\":\"BAD_REQUEST\",\"message\":\"Cannot downgrade version without setting 'force'. " + + "Current target version: 6.123.456, attempted to set target version: 6.123.1\"}"); // Downgrade with force is OK assertResponse(new Request("http://localhost:8080/nodes/v2/upgrade/confighost", @@ -677,11 +677,11 @@ public class RestApiTest { "{\"versions\":{\"config\":\"6.123.456\",\"confighost\":\"6.123.1\",\"controller\":\"6.123.456\"},\"osVersions\":{},\"dockerImages\":{}}"); // Setting empty version without force fails - assertResponse(new Request("http://localhost:8080/nodes/v2/upgrade/confighost", - Utf8.toBytes("{\"version\": null}"), - Request.Method.PATCH), - 400, - "{\"error-code\":\"BAD_REQUEST\",\"message\":\"Cannot downgrade version without setting 'force'. Current target version: 6.123.1, attempted to set target version: 0.0.0\"}"); + tester.assertResponse(new Request("http://localhost:8080/nodes/v2/upgrade/confighost", + Utf8.toBytes("{\"version\": null}"), + Request.Method.PATCH), + 400, + "{\"error-code\":\"BAD_REQUEST\",\"message\":\"Cannot downgrade version without setting 'force'. Current target version: 6.123.1, attempted to set target version: 0.0.0\"}"); assertResponse(new Request("http://localhost:8080/nodes/v2/upgrade/confighost", Utf8.toBytes("{\"version\": null, \"force\": true}"), @@ -713,18 +713,18 @@ public class RestApiTest { "{\"message\":\"Set version to 6.124.42, osVersion to 7.5.2 for nodes of type confighost\"}"); // Attempt to upgrade unsupported node type - assertResponse(new Request("http://localhost:8080/nodes/v2/upgrade/config", - Utf8.toBytes("{\"osVersion\": \"7.5.2\"}"), - Request.Method.PATCH), - 400, - "{\"error-code\":\"BAD_REQUEST\",\"message\":\"Node type 'config' does not support OS upgrades\"}"); + tester.assertResponse(new Request("http://localhost:8080/nodes/v2/upgrade/config", + Utf8.toBytes("{\"osVersion\": \"7.5.2\"}"), + Request.Method.PATCH), + 400, + "{\"error-code\":\"BAD_REQUEST\",\"message\":\"Node type 'config' does not support OS upgrades\"}"); // Attempt to downgrade OS - assertResponse(new Request("http://localhost:8080/nodes/v2/upgrade/confighost", - Utf8.toBytes("{\"osVersion\": \"7.4.2\"}"), - Request.Method.PATCH), - 400, - "{\"error-code\":\"BAD_REQUEST\",\"message\":\"Cannot set target OS version to 7.4.2 without setting 'force', as it's lower than the current version: 7.5.2\"}"); + tester.assertResponse(new Request("http://localhost:8080/nodes/v2/upgrade/confighost", + Utf8.toBytes("{\"osVersion\": \"7.4.2\"}"), + Request.Method.PATCH), + 400, + "{\"error-code\":\"BAD_REQUEST\",\"message\":\"Cannot set target OS version to 7.4.2 without setting 'force', as it's lower than the current version: 7.5.2\"}"); // Downgrading OS with force succeeds assertResponse(new Request("http://localhost:8080/nodes/v2/upgrade/confighost", @@ -733,11 +733,11 @@ public class RestApiTest { "{\"message\":\"Set osVersion to 7.4.2 for nodes of type confighost\"}"); // Current target is considered bad, remove it - assertResponse(new Request("http://localhost:8080/nodes/v2/upgrade/confighost", - Utf8.toBytes("{\"osVersion\": null}"), - Request.Method.PATCH), - 200, - "{\"message\":\"Set osVersion to null for nodes of type confighost\"}"); + tester.assertResponse(new Request("http://localhost:8080/nodes/v2/upgrade/confighost", + Utf8.toBytes("{\"osVersion\": null}"), + Request.Method.PATCH), + 200, + "{\"message\":\"Set osVersion to null for nodes of type confighost\"}"); // Set docker image for config and tenant assertResponse(new Request("http://localhost:8080/nodes/v2/upgrade/tenant", @@ -753,11 +753,11 @@ public class RestApiTest { "{\"versions\":{\"config\":\"6.123.456\",\"confighost\":\"6.124.42\",\"controller\":\"6.123.456\"},\"osVersions\":{\"host\":\"7.5.2\"},\"dockerImages\":{\"tenant\":\"my-repo.my-domain.example:1234/repo/tenant\",\"config\":\"my-repo.my-domain.example:1234/repo/image\"}}"); // Cannot set docker image for non docker node type - assertResponse(new Request("http://localhost:8080/nodes/v2/upgrade/confighost", - Utf8.toBytes("{\"dockerImage\": \"my-repo.my-domain.example:1234/repo/image\"}"), - Request.Method.PATCH), - 400, - "{\"error-code\":\"BAD_REQUEST\",\"message\":\"Setting docker image for confighost nodes is unsupported\"}"); + tester.assertResponse(new Request("http://localhost:8080/nodes/v2/upgrade/confighost", + Utf8.toBytes("{\"dockerImage\": \"my-repo.my-domain.example:1234/repo/image\"}"), + Request.Method.PATCH), + 400, + "{\"error-code\":\"BAD_REQUEST\",\"message\":\"Setting docker image for confighost nodes is unsupported\"}"); } @Test @@ -769,12 +769,12 @@ public class RestApiTest { "{\"message\":\"Set osVersion to 7.5.2 for nodes of type host\"}"); // Activate target - var nodeRepository = (NodeRepository) container.components().getComponent(MockNodeRepository.class.getName()); + var nodeRepository = (NodeRepository)tester.container().components().getComponent(MockNodeRepository.class.getName()); var osUpgradeActivator = new OsUpgradeActivator(nodeRepository, Duration.ofDays(1)); osUpgradeActivator.run(); // Other node type does not return wanted OS version - Response r = container.handleRequest(new Request("http://localhost:8080/nodes/v2/node/host1.yahoo.com")); + Response r = tester.container().handleRequest(new Request("http://localhost:8080/nodes/v2/node/host1.yahoo.com")); assertFalse("Response omits wantedOsVersions field", r.getBodyAsString().contains("wantedOsVersion")); // Node updates its node object after upgrading OS @@ -854,21 +854,13 @@ public class RestApiTest { } @Test - public void test_load_balancers() throws Exception { - assertFile(new Request("http://localhost:8080/loadbalancers/v1/"), "load-balancers.json"); - assertFile(new Request("http://localhost:8080/loadbalancers/v1/?application=tenant4.application4.instance4"), "load-balancers-single.json"); - assertResponse(new Request("http://localhost:8080/loadbalancers/v1/?application=tenant.nonexistent.default"), "{\"loadBalancers\":[]}"); - } - - @Test public void test_flavor_overrides() throws Exception { String host = "parent2.yahoo.com"; // Test adding with overrides - assertResponse(new Request("http://localhost:8080/nodes/v2/node", - ("[{\"hostname\":\"" + host + "\"," + createIpAddresses("::1") + "\"openStackId\":\"osid-123\"," + - "\"flavor\":\"large-variant\",\"resources\":{\"diskGb\":1234,\"memoryGb\":4321}}]"). - getBytes(StandardCharsets.UTF_8), - Request.Method.POST), + tester.assertResponse(new Request("http://localhost:8080/nodes/v2/node", + ("[{\"hostname\":\"" + host + "\"," + createIpAddresses("::1") + "\"openStackId\":\"osid-123\"," + + "\"flavor\":\"large-variant\",\"resources\":{\"diskGb\":1234,\"memoryGb\":4321}}]").getBytes(StandardCharsets.UTF_8), + Request.Method.POST), 400, "{\"error-code\":\"BAD_REQUEST\",\"message\":\"Can only override disk GB for configured flavor\"}"); @@ -878,8 +870,8 @@ public class RestApiTest { getBytes(StandardCharsets.UTF_8), Request.Method.POST), "{\"message\":\"Added 1 nodes to the provisioned state\"}"); - assertResponseContains(new Request("http://localhost:8080/nodes/v2/node/" + host), - "\"resources\":{\"vcpu\":64.0,\"memoryGb\":128.0,\"diskGb\":1234.0,\"bandwidthGbps\":15.0,\"diskSpeed\":\"fast\",\"storageType\":\"remote\"}"); + tester.assertResponseContains(new Request("http://localhost:8080/nodes/v2/node/" + host), + "\"resources\":{\"vcpu\":64.0,\"memoryGb\":128.0,\"diskGb\":1234.0,\"bandwidthGbps\":15.0,\"diskSpeed\":\"fast\",\"storageType\":\"remote\"}"); // Test adding tenant node String tenant = "node-1-3.yahoo.com"; @@ -890,21 +882,21 @@ public class RestApiTest { getBytes(StandardCharsets.UTF_8), Request.Method.POST), "{\"message\":\"Added 1 nodes to the provisioned state\"}"); - assertResponseContains(new Request("http://localhost:8080/nodes/v2/node/" + tenant), resources); + tester.assertResponseContains(new Request("http://localhost:8080/nodes/v2/node/" + tenant), resources); // Test patching with overrides - assertResponse(new Request("http://localhost:8080/nodes/v2/node/" + host, - "{\"minDiskAvailableGb\":5432,\"minMainMemoryAvailableGb\":2345}".getBytes(StandardCharsets.UTF_8), - Request.Method.PATCH), - 400, - "{\"error-code\":\"BAD_REQUEST\",\"message\":\"Could not set field 'minMainMemoryAvailableGb': Can only override disk GB for configured flavor\"}"); + tester.assertResponse(new Request("http://localhost:8080/nodes/v2/node/" + host, + "{\"minDiskAvailableGb\":5432,\"minMainMemoryAvailableGb\":2345}".getBytes(StandardCharsets.UTF_8), + Request.Method.PATCH), + 400, + "{\"error-code\":\"BAD_REQUEST\",\"message\":\"Could not set field 'minMainMemoryAvailableGb': Can only override disk GB for configured flavor\"}"); assertResponse(new Request("http://localhost:8080/nodes/v2/node/" + host, "{\"minDiskAvailableGb\":5432}".getBytes(StandardCharsets.UTF_8), Request.Method.PATCH), "{\"message\":\"Updated " + host + "\"}"); - assertResponseContains(new Request("http://localhost:8080/nodes/v2/node/" + host), - "\"resources\":{\"vcpu\":64.0,\"memoryGb\":128.0,\"diskGb\":5432.0,\"bandwidthGbps\":15.0,\"diskSpeed\":\"fast\",\"storageType\":\"remote\"}"); + tester.assertResponseContains(new Request("http://localhost:8080/nodes/v2/node/" + host), + "\"resources\":{\"vcpu\":64.0,\"memoryGb\":128.0,\"diskGb\":5432.0,\"bandwidthGbps\":15.0,\"diskSpeed\":\"fast\",\"storageType\":\"remote\"}"); } @Test @@ -912,28 +904,27 @@ public class RestApiTest { String hostname = "node123.yahoo.com"; String resources = "\"resources\":{\"vcpu\":5.0,\"memoryGb\":4321.0,\"diskGb\":1234.0,\"bandwidthGbps\":0.3,\"diskSpeed\":\"slow\",\"storageType\":\"local\"}"; // Test adding new node with resources - assertResponse(new Request("http://localhost:8080/nodes/v2/node", - ("[{\"hostname\":\"" + hostname + "\"," + createIpAddresses("::1") + "\"openStackId\":\"osid-123\"," + - resources.replace("\"memoryGb\":4321.0,", "") + "}]"). - getBytes(StandardCharsets.UTF_8), - Request.Method.POST), - 400, - "{\"error-code\":\"BAD_REQUEST\",\"message\":\"Required field 'memoryGb' is missing\"}"); + tester.assertResponse(new Request("http://localhost:8080/nodes/v2/node", + ("[{\"hostname\":\"" + hostname + "\"," + createIpAddresses("::1") + "\"openStackId\":\"osid-123\"," + + resources.replace("\"memoryGb\":4321.0,", "") + "}]").getBytes(StandardCharsets.UTF_8), + Request.Method.POST), + 400, + "{\"error-code\":\"BAD_REQUEST\",\"message\":\"Required field 'memoryGb' is missing\"}"); assertResponse(new Request("http://localhost:8080/nodes/v2/node", ("[{\"hostname\":\"" + hostname + "\"," + createIpAddresses("::1") + "\"openStackId\":\"osid-123\"," + resources + "}]") .getBytes(StandardCharsets.UTF_8), Request.Method.POST), "{\"message\":\"Added 1 nodes to the provisioned state\"}"); - assertResponseContains(new Request("http://localhost:8080/nodes/v2/node/" + hostname), resources); + tester.assertResponseContains(new Request("http://localhost:8080/nodes/v2/node/" + hostname), resources); // Test patching with overrides assertResponse(new Request("http://localhost:8080/nodes/v2/node/" + hostname, "{\"diskGb\":12,\"memoryGb\":34,\"vcpu\":56,\"fastDisk\":true,\"remoteStorage\":true,\"bandwidthGbps\":78.0}".getBytes(StandardCharsets.UTF_8), Request.Method.PATCH), "{\"message\":\"Updated " + hostname + "\"}"); - assertResponseContains(new Request("http://localhost:8080/nodes/v2/node/" + hostname), - "\"resources\":{\"vcpu\":56.0,\"memoryGb\":34.0,\"diskGb\":12.0,\"bandwidthGbps\":78.0,\"diskSpeed\":\"fast\",\"storageType\":\"remote\"}"); + tester.assertResponseContains(new Request("http://localhost:8080/nodes/v2/node/" + hostname), + "\"resources\":{\"vcpu\":56.0,\"memoryGb\":34.0,\"diskGb\":12.0,\"bandwidthGbps\":78.0,\"diskSpeed\":\"fast\",\"storageType\":\"remote\"}"); } private static String asDockerNodeJson(String hostname, String parentHostname, String... ipAddress) { @@ -974,71 +965,20 @@ public class RestApiTest { "],"; } - /** Asserts a particular response and 200 as response status */ - private void assertResponse(Request request, String responseMessage) throws IOException { - assertResponse(request, 200, responseMessage); - } - - private void assertResponse(Request request, int responseStatus, String responseMessage) throws IOException { - Response response = container.handleRequest(request); - // Compare both status and message at once for easier diagnosis - assertEquals("status: " + responseStatus + "\nmessage: " + responseMessage, - "status: " + response.getStatus() + "\nmessage: " + response.getBodyAsString()); - } - - private void assertResponseContains(Request request, String responseSnippet) throws IOException { - assertPartialResponse(request, responseSnippet, true); - } - - private void assertPartialResponse(Request request, String responseSnippet, boolean match) throws IOException { - String response = container.handleRequest(request).getBodyAsString(); - assertEquals(String.format("Expected response to " + (match ? " " : "not ") + "contain: %s\nResponse: %s", - responseSnippet, response), match, response.contains(responseSnippet)); - } - - private void assertFile(Request request, String responseFile) throws IOException { - String expectedResponse = IOUtils.readFile(new File(responsesPath + responseFile)); - expectedResponse = include(expectedResponse); - expectedResponse = expectedResponse.replaceAll("(\"[^\"]*\")|\\s*", "$1"); // Remove whitespace - String responseString = container.handleRequest(request).getBodyAsString(); - if (expectedResponse.contains("(ignore)")) { - // Convert expected response to a literal pattern and replace any ignored field with a pattern that matches - // until the first stop character - String stopCharacters = "[^,:\\\\[\\\\]{}]"; - String expectedResponsePattern = Pattern.quote(expectedResponse) - .replaceAll("\"?\\(ignore\\)\"?", "\\\\E" + - stopCharacters + "*\\\\Q"); - if (!Pattern.matches(expectedResponsePattern, responseString)) { - throw new ComparisonFailure(responseFile + " (with ignored fields)", expectedResponsePattern, - responseString); - } - } else { - assertEquals(responseFile, expectedResponse, responseString); - } - } - private void assertRestart(int restartCount, Request request) throws IOException { - assertResponse(request, 200, "{\"message\":\"Scheduled restart of " + restartCount + " matching nodes\"}"); + tester.assertResponse(request, 200, "{\"message\":\"Scheduled restart of " + restartCount + " matching nodes\"}"); } private void assertReboot(int rebootCount, Request request) throws IOException { - assertResponse(request, 200, "{\"message\":\"Scheduled reboot of " + rebootCount + " matching nodes\"}"); + tester.assertResponse(request, 200, "{\"message\":\"Scheduled reboot of " + rebootCount + " matching nodes\"}"); + } + + private void assertFile(Request request, String file) throws IOException { + tester.assertFile(request, file); } - /** Replaces @include(localFile) with the content of the file */ - private String include(String response) throws IOException { - // Please don't look at this code - int includeIndex = response.indexOf("@include("); - if (includeIndex < 0) return response; - String prefix = response.substring(0, includeIndex); - String rest = response.substring(includeIndex + "@include(".length()); - int filenameEnd = rest.indexOf(")"); - String includeFileName = rest.substring(0, filenameEnd); - String includedContent = IOUtils.readFile(new File(responsesPath + includeFileName)); - includedContent = include(includedContent); - String postFix = rest.substring(filenameEnd + 1); - postFix = include(postFix); - return prefix + includedContent + postFix; + private void assertResponse(Request request, String file) throws IOException { + tester.assertResponse(request, file); } } diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/RestApiTester.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/RestApiTester.java new file mode 100644 index 00000000000..847a4e2500b --- /dev/null +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/RestApiTester.java @@ -0,0 +1,96 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.provision.restapi; + +import com.yahoo.application.Networking; +import com.yahoo.application.container.JDisc; +import com.yahoo.application.container.handler.Request; +import com.yahoo.application.container.handler.Response; +import com.yahoo.io.IOUtils; +import com.yahoo.vespa.hosted.provision.testutils.ContainerConfig; +import org.junit.ComparisonFailure; + +import java.io.File; +import java.io.IOException; +import java.util.regex.Pattern; + +import static org.junit.Assert.assertEquals; + +/** + * @author bratseth + */ +public class RestApiTester { + + private final static String responsesPath = "src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/"; + + private final JDisc container; + + public RestApiTester() { + container = JDisc.fromServicesXml(ContainerConfig.servicesXmlV2(0), Networking.disable); + } + + public void close() { + if (container != null) container.close(); + } + + public JDisc container() { return container; } + + /** Asserts a particular response and 200 as response status */ + public void assertResponse(Request request, String responseMessage) throws IOException { + assertResponse(request, 200, responseMessage); + } + + public void assertResponse(Request request, int responseStatus, String responseMessage) throws IOException { + Response response = container.handleRequest(request); + // Compare both status and message at once for easier diagnosis + assertEquals("status: " + responseStatus + "\nmessage: " + responseMessage, + "status: " + response.getStatus() + "\nmessage: " + response.getBodyAsString()); + } + + public void assertResponseContains(Request request, String responseSnippet) throws IOException { + assertPartialResponse(request, responseSnippet, true); + } + + public void assertPartialResponse(Request request, String responseSnippet, boolean match) throws IOException { + String response = container.handleRequest(request).getBodyAsString(); + assertEquals(String.format("Expected response to " + (match ? " " : "not ") + "contain: %s\nResponse: %s", + responseSnippet, response), match, response.contains(responseSnippet)); + } + + public void assertFile(Request request, String responseFile) throws IOException { + String expectedResponse = IOUtils.readFile(new File(responsesPath + responseFile)); + expectedResponse = include(expectedResponse); + expectedResponse = expectedResponse.replaceAll("(\"[^\"]*\")|\\s*", "$1"); // Remove whitespace + String responseString = container.handleRequest(request).getBodyAsString(); + if (expectedResponse.contains("(ignore)")) { + // Convert expected response to a literal pattern and replace any ignored field with a pattern that matches + // until the first stop character + String stopCharacters = "[^,:\\\\[\\\\]{}]"; + String expectedResponsePattern = Pattern.quote(expectedResponse) + .replaceAll("\"?\\(ignore\\)\"?", "\\\\E" + + stopCharacters + "*\\\\Q"); + if (!Pattern.matches(expectedResponsePattern, responseString)) { + throw new ComparisonFailure(responseFile + " (with ignored fields)", expectedResponsePattern, + responseString); + } + } else { + assertEquals(responseFile, expectedResponse, responseString); + } + } + + /** Replaces @include(localFile) with the content of the file */ + public String include(String response) throws IOException { + // Please don't look at this code + int includeIndex = response.indexOf("@include("); + if (includeIndex < 0) return response; + String prefix = response.substring(0, includeIndex); + String rest = response.substring(includeIndex + "@include(".length()); + int filenameEnd = rest.indexOf(")"); + String includeFileName = rest.substring(0, filenameEnd); + String includedContent = IOUtils.readFile(new File(responsesPath + includeFileName)); + includedContent = include(includedContent); + String postFix = rest.substring(filenameEnd + 1); + postFix = include(postFix); + return prefix + includedContent + postFix; + } + +} diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/acl-config-server.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/acl-config-server.json index 55891309856..55891309856 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/acl-config-server.json +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/acl-config-server.json diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/acl-tenant-node.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/acl-tenant-node.json index 16cf5c36dcc..16cf5c36dcc 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/acl-tenant-node.json +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/acl-tenant-node.json diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/active-nodes.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/active-nodes.json index c61a755d989..c61a755d989 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/active-nodes.json +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/active-nodes.json diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/application1.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/application1.json index ce2309e4d6c..ce2309e4d6c 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/application1.json +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/application1.json diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/application2-nodes.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/application2-nodes.json index 4581ecba73d..4581ecba73d 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/application2-nodes.json +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/application2-nodes.json diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/application2.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/application2.json index 75af5d4a328..75af5d4a328 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/application2.json +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/application2.json diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/applications.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/applications.json index 6d25b3e59f5..6d25b3e59f5 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/applications.json +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/applications.json diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/capacity-hostremoval-impossible.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/capacity-hostremoval-impossible.json index f85e0dcc270..f85e0dcc270 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/capacity-hostremoval-impossible.json +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/capacity-hostremoval-impossible.json diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/capacity-hostremoval-possible.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/capacity-hostremoval-possible.json index 177e1f317f7..177e1f317f7 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/capacity-hostremoval-possible.json +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/capacity-hostremoval-possible.json diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/capacity-zone.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/capacity-zone.json index f6b2f96023a..f6b2f96023a 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/capacity-zone.json +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/capacity-zone.json diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/cfg1.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/cfg1.json index d2d39b06161..d2d39b06161 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/cfg1.json +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/cfg1.json diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/cfg2.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/cfg2.json index 969c9a1b9c2..969c9a1b9c2 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/cfg2.json +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/cfg2.json diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/child-nodes.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/child-nodes.json index dae92aae091..dae92aae091 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/child-nodes.json +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/child-nodes.json diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/content-nodes.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/content-nodes.json index e84f17dc40d..e84f17dc40d 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/content-nodes.json +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/content-nodes.json diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/controller1.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/controller1.json index 035e15ead49..035e15ead49 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/controller1.json +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/controller1.json diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/docker-container1.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/docker-container1.json index c97509b3b3d..c97509b3b3d 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/docker-container1.json +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/docker-container1.json diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/docker-node1-os-upgrade-complete.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/docker-node1-os-upgrade-complete.json index 77622933866..77622933866 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/docker-node1-os-upgrade-complete.json +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/docker-node1-os-upgrade-complete.json diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/docker-node1-os-upgrade.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/docker-node1-os-upgrade.json index a1775dc794e..a1775dc794e 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/docker-node1-os-upgrade.json +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/docker-node1-os-upgrade.json diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/docker-node1.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/docker-node1.json index b6b93c8d108..b6b93c8d108 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/docker-node1.json +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/docker-node1.json diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/docker-node2.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/docker-node2.json index 88e4d132ed5..88e4d132ed5 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/docker-node2.json +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/docker-node2.json diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/docker-node3.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/docker-node3.json index c890f1e72be..c890f1e72be 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/docker-node3.json +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/docker-node3.json diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/docker-node4.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/docker-node4.json index 197a8c248e9..197a8c248e9 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/docker-node4.json +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/docker-node4.json diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/docker-node5.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/docker-node5.json index 2638b6b74f5..2638b6b74f5 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/docker-node5.json +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/docker-node5.json diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/dockerhost1-with-firmware-data.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/dockerhost1-with-firmware-data.json index ea5c3623303..ea5c3623303 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/dockerhost1-with-firmware-data.json +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/dockerhost1-with-firmware-data.json diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/flags1.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/flags1.json index a606777e9fd..a606777e9fd 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/flags1.json +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/flags1.json diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/flags2.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/flags2.json index 4baf75f2169..4baf75f2169 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/flags2.json +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/flags2.json diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/load-balancers-single.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/load-balancers-single.json index a9a728bab15..a9a728bab15 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/load-balancers-single.json +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/load-balancers-single.json diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/load-balancers.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/load-balancers.json index 515081bcb8e..515081bcb8e 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/load-balancers.json +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/load-balancers.json diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/maintenance.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/maintenance.json index e041a7b8b54..e041a7b8b54 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/maintenance.json +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/maintenance.json diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node1.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/node1.json index 7f6a6e17fa1..7f6a6e17fa1 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node1.json +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/node1.json diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node10.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/node10.json index da0e7b28f3e..da0e7b28f3e 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node10.json +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/node10.json diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node11.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/node11.json index 17f8210fa4d..17f8210fa4d 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node11.json +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/node11.json diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node13.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/node13.json index 2e96c4c9bbc..2e96c4c9bbc 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node13.json +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/node13.json diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node14.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/node14.json index bfbecc09c60..bfbecc09c60 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node14.json +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/node14.json diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node2.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/node2.json index b8ce5c86c4d..b8ce5c86c4d 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node2.json +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/node2.json diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node3.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/node3.json index 70ae2a030bd..70ae2a030bd 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node3.json +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/node3.json diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node4-after-changes.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/node4-after-changes.json index af3552945d9..af3552945d9 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node4-after-changes.json +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/node4-after-changes.json diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node4.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/node4.json index c8680e1b420..c8680e1b420 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node4.json +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/node4.json diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node5-after-changes.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/node5-after-changes.json index beb42ec6907..beb42ec6907 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node5-after-changes.json +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/node5-after-changes.json diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node5.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/node5.json index 3fb5ffa4708..3fb5ffa4708 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node5.json +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/node5.json diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node55.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/node55.json index 3107311b792..3107311b792 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node55.json +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/node55.json diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node6-after-changes.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/node6-after-changes.json index 65c7e9db6cd..65c7e9db6cd 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node6-after-changes.json +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/node6-after-changes.json diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node6-reports-2.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/node6-reports-2.json index a3d53798d7c..a3d53798d7c 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node6-reports-2.json +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/node6-reports-2.json diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node6-reports-3.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/node6-reports-3.json index 7f0c3a5f706..7f0c3a5f706 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node6-reports-3.json +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/node6-reports-3.json diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node6-reports.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/node6-reports.json index 67b8d67c7f1..67b8d67c7f1 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node6-reports.json +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/node6-reports.json diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node6.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/node6.json index 5a665a5a223..5a665a5a223 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node6.json +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/node6.json diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node7.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/node7.json index 38757b81156..38757b81156 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node7.json +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/node7.json diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node8.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/node8.json index aa2b2acdb9f..aa2b2acdb9f 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node8.json +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/node8.json diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node9.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/node9.json index 5f388c13f28..5f388c13f28 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node9.json +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/node9.json diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/nodes-recursive.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/nodes-recursive.json index cf3e3c1b457..cf3e3c1b457 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/nodes-recursive.json +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/nodes-recursive.json diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/nodes.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/nodes.json index cdbd2cac9b9..cdbd2cac9b9 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/nodes.json +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/nodes.json diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/parent1.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/parent1.json index 40f4a3b5160..40f4a3b5160 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/parent1.json +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/parent1.json diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/parent2.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/parent2.json index ecc497172c7..ecc497172c7 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/parent2.json +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/parent2.json diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/root.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/root.json index cc91314df84..cc91314df84 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/root.json +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/root.json diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/states-recursive.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/states-recursive.json index f8343756559..f8343756559 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/states-recursive.json +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/states-recursive.json diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/states.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/states.json index 69579148df3..69579148df3 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/states.json +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/states.json diff --git a/searchcore/src/apps/vespa-gen-testdocs/vespa-gen-testdocs.cpp b/searchcore/src/apps/vespa-gen-testdocs/vespa-gen-testdocs.cpp index 2f54cefac29..080d98734bd 100644 --- a/searchcore/src/apps/vespa-gen-testdocs/vespa-gen-testdocs.cpp +++ b/searchcore/src/apps/vespa-gen-testdocs/vespa-gen-testdocs.cpp @@ -1,6 +1,6 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/searchlib/util/rand48.h> +#include <vespa/vespalib/util/rand48.h> #include <vespa/vespalib/stllike/string.h> #include <vespa/vespalib/stllike/hash_set.h> #include <vespa/vespalib/stllike/asciistream.h> @@ -98,10 +98,10 @@ shafile(const string &baseDir, class StringGenerator { - search::Rand48 &_rnd; + vespalib::Rand48 &_rnd; public: - StringGenerator(search::Rand48 &rnd); + StringGenerator(vespalib::Rand48 &rnd); void rand_string(string &res, uint32_t minLen, uint32_t maxLen); @@ -114,7 +114,7 @@ public: }; -StringGenerator::StringGenerator(search::Rand48 &rnd) +StringGenerator::StringGenerator(vespalib::Rand48 &rnd) : _rnd(rnd) { } @@ -283,7 +283,7 @@ PrefixTextFieldGenerator::generateValue(vespalib::asciistream &doc, uint32_t id) class RandTextFieldGenerator : public FieldGenerator { - search::Rand48 &_rnd; + vespalib::Rand48 &_rnd; uint32_t _numWords; StringArray _strings; uint32_t _minFill; @@ -291,7 +291,7 @@ class RandTextFieldGenerator : public FieldGenerator public: RandTextFieldGenerator(const string &name, - search::Rand48 &rnd, + vespalib::Rand48 &rnd, uint32_t numWords, uint32_t minFill, uint32_t maxFill); @@ -302,7 +302,7 @@ public: RandTextFieldGenerator::RandTextFieldGenerator(const string &name, - search::Rand48 &rnd, + vespalib::Rand48 &rnd, uint32_t numWords, uint32_t minFill, uint32_t randFill) @@ -354,7 +354,7 @@ class ModTextFieldGenerator : public FieldGenerator public: ModTextFieldGenerator(const string &name, - search::Rand48 &rnd, + vespalib::Rand48 &rnd, const std::vector<uint32_t> &mods); virtual ~ModTextFieldGenerator(); virtual void generateValue(vespalib::asciistream &doc, uint32_t id) override; @@ -362,7 +362,7 @@ public: ModTextFieldGenerator::ModTextFieldGenerator(const string &name, - [[maybe_unused]] search::Rand48 &rnd, + [[maybe_unused]] vespalib::Rand48 &rnd, const std::vector<uint32_t> &mods) : FieldGenerator(name), _mods(mods) @@ -419,13 +419,13 @@ IdTextFieldGenerator::generateValue(vespalib::asciistream &doc, uint32_t id) class RandIntFieldGenerator : public FieldGenerator { - search::Rand48 &_rnd; + vespalib::Rand48 &_rnd; uint32_t _low; uint32_t _count; public: RandIntFieldGenerator(const string &name, - search::Rand48 &rnd, + vespalib::Rand48 &rnd, uint32_t low, uint32_t count); virtual ~RandIntFieldGenerator(); @@ -436,7 +436,7 @@ public: RandIntFieldGenerator::RandIntFieldGenerator(const string &name, - search::Rand48 &rnd, + vespalib::Rand48 &rnd, uint32_t low, uint32_t count) : FieldGenerator(name), @@ -617,7 +617,7 @@ class GenTestDocsApp : public SubApp int _optIndex; std::vector<FieldGenerator::SP> _fields; std::vector<uint32_t> _mods; - search::Rand48 _rnd; + vespalib::Rand48 _rnd; string _outFile; bool _headers; bool _json; diff --git a/searchlib/abi-spec.json b/searchlib/abi-spec.json index d058104bf1b..6bce791914c 100644 --- a/searchlib/abi-spec.json +++ b/searchlib/abi-spec.json @@ -1416,7 +1416,8 @@ "public com.yahoo.searchlib.rankingexpression.ExpressionFunction getFunction(java.lang.String)", "protected com.google.common.collect.ImmutableMap functions()", "public java.lang.String getBinding(java.lang.String)", - "public com.yahoo.searchlib.rankingexpression.rule.FunctionReferenceContext withBindings(java.util.Map)" + "public com.yahoo.searchlib.rankingexpression.rule.FunctionReferenceContext withBindings(java.util.Map)", + "public com.yahoo.searchlib.rankingexpression.rule.FunctionReferenceContext withoutBindings()" ], "fields": [] }, @@ -1573,7 +1574,9 @@ "public void addArgumentTypeSerialization(java.lang.String, java.lang.String, com.yahoo.tensor.TensorType)", "public void addFunctionTypeSerialization(java.lang.String, com.yahoo.tensor.TensorType)", "public com.yahoo.searchlib.rankingexpression.rule.SerializationContext withBindings(java.util.Map)", + "public com.yahoo.searchlib.rankingexpression.rule.SerializationContext withoutBindings()", "public java.util.Map serializedFunctions()", + "public bridge synthetic com.yahoo.searchlib.rankingexpression.rule.FunctionReferenceContext withoutBindings()", "public bridge synthetic com.yahoo.searchlib.rankingexpression.rule.FunctionReferenceContext withBindings(java.util.Map)" ], "fields": [] diff --git a/searchlib/src/apps/tests/btreestress_test.cpp b/searchlib/src/apps/tests/btreestress_test.cpp index e6ef0740fc3..a39416c2227 100644 --- a/searchlib/src/apps/tests/btreestress_test.cpp +++ b/searchlib/src/apps/tests/btreestress_test.cpp @@ -6,7 +6,7 @@ #include <vespa/vespalib/btree/btreenodeallocator.h> #include <vespa/vespalib/btree/btree.h> #include <vespa/vespalib/btree/btreestore.h> -#include <vespa/searchlib/util/rand48.h> +#include <vespa/vespalib/util/rand48.h> #include <vespa/vespalib/btree/btreenodeallocator.hpp> #include <vespa/vespalib/btree/btreenode.hpp> @@ -37,7 +37,7 @@ struct Fixture MyTreeIterator _writeItr; vespalib::ThreadStackExecutor _writer; // 1 write thread vespalib::ThreadStackExecutor _readers; // multiple reader threads - search::Rand48 _rnd; + vespalib::Rand48 _rnd; uint32_t _keyLimit; std::atomic<long> _readSeed; std::atomic<long> _doneWriteWork; @@ -135,7 +135,7 @@ Fixture::remove(uint32_t key) void Fixture::readWork(uint32_t cnt) { - search::Rand48 rnd; + vespalib::Rand48 rnd; rnd.srand48(++_readSeed); uint32_t i; for (i = 0; i < cnt && _stopRead.load() == 0; ++i) { @@ -159,7 +159,7 @@ Fixture::readWork() void Fixture::writeWork(uint32_t cnt) { - search::Rand48 &rnd(_rnd); + vespalib::Rand48 &rnd(_rnd); for (uint32_t i = 0; i < cnt; ++i) { uint32_t key = rnd.lrand48() % _keyLimit; if ((rnd.lrand48() & 1) == 0) { diff --git a/searchlib/src/apps/tests/memoryindexstress_test.cpp b/searchlib/src/apps/tests/memoryindexstress_test.cpp index 6bbd93bda84..041335b7ed1 100644 --- a/searchlib/src/apps/tests/memoryindexstress_test.cpp +++ b/searchlib/src/apps/tests/memoryindexstress_test.cpp @@ -20,7 +20,7 @@ #include <vespa/searchlib/queryeval/fake_searchable.h> #include <vespa/searchlib/queryeval/searchiterator.h> #include <vespa/searchlib/test/index/mock_field_length_inspector.h> -#include <vespa/searchlib/util/rand48.h> +#include <vespa/vespalib/util/rand48.h> #include <vespa/vespalib/testkit/testapp.h> #include <vespa/vespalib/util/threadstackexecutor.h> #include <vespa/vespalib/util/sequencedtaskexecutor.h> @@ -201,7 +201,7 @@ struct Fixture { uint32_t _readThreads; vespalib::ThreadStackExecutor _writer; // 1 write thread vespalib::ThreadStackExecutor _readers; // multiple reader threads - search::Rand48 _rnd; + vespalib::Rand48 _rnd; uint32_t _keyLimit; std::atomic<long> _readSeed; std::atomic<long> _doneWriteWork; @@ -285,7 +285,7 @@ Fixture::~Fixture() void Fixture::readWork(uint32_t cnt) { - search::Rand48 rnd; + vespalib::Rand48 rnd; rnd.srand48(++_readSeed); uint32_t i; uint32_t emptyCount = 0; @@ -345,7 +345,7 @@ Fixture::readWork() void Fixture::writeWork(uint32_t cnt) { - search::Rand48 &rnd(_rnd); + vespalib::Rand48 &rnd(_rnd); for (uint32_t i = 0; i < cnt; ++i) { uint32_t key = rnd.lrand48() % _keyLimit; if ((rnd.lrand48() & 1) == 0) { diff --git a/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/ExpressionFunction.java b/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/ExpressionFunction.java index 674571ff73e..559f690a32d 100755 --- a/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/ExpressionFunction.java +++ b/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/ExpressionFunction.java @@ -134,7 +134,10 @@ public class ExpressionFunction { for (int i = 0; i < arguments.size() && i < argumentValues.size(); ++i) { argumentBindings.put(arguments.get(i), argumentValues.get(i).toString(new StringBuilder(), context, path, null).toString()); } - return new Instance(toSymbol(argumentBindings), body.getRoot().toString(new StringBuilder(), context.withBindings(argumentBindings), path, null).toString()); + context = argumentBindings.isEmpty() ? context.withoutBindings() : context.withBindings(argumentBindings); + String symbol = toSymbol(argumentBindings); + String expressionString = body.getRoot().toString(new StringBuilder(), context, path, null).toString(); + return new Instance(symbol, expressionString); } /** diff --git a/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/rule/FunctionReferenceContext.java b/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/rule/FunctionReferenceContext.java index 83aabada8f0..6717bec0258 100644 --- a/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/rule/FunctionReferenceContext.java +++ b/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/rule/FunctionReferenceContext.java @@ -68,4 +68,9 @@ public class FunctionReferenceContext { return new FunctionReferenceContext(this.functions, bindings); } + /** Returns a fresh context without bindings */ + public FunctionReferenceContext withoutBindings() { + return new FunctionReferenceContext(this.functions); + } + } diff --git a/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/rule/ReferenceNode.java b/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/rule/ReferenceNode.java index 8fec3603f3e..335d3861d9d 100755 --- a/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/rule/ReferenceNode.java +++ b/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/rule/ReferenceNode.java @@ -5,7 +5,6 @@ import com.yahoo.searchlib.rankingexpression.ExpressionFunction; import com.yahoo.searchlib.rankingexpression.RankingExpression; import com.yahoo.searchlib.rankingexpression.Reference; import com.yahoo.searchlib.rankingexpression.evaluation.Context; -import com.yahoo.searchlib.rankingexpression.evaluation.StringValue; import com.yahoo.searchlib.rankingexpression.evaluation.Value; import com.yahoo.tensor.TensorType; import com.yahoo.tensor.evaluation.TypeContext; @@ -13,7 +12,7 @@ import com.yahoo.tensor.evaluation.TypeContext; import java.util.ArrayDeque; import java.util.Deque; import java.util.List; -import java.util.Optional; +import java.util.Map; /** * A node referring either to a value in the context or to a named ranking expression function. @@ -84,13 +83,25 @@ public final class ReferenceNode extends CompositeNode { if (path.contains(myPath)) throw new IllegalStateException("Cycle in ranking expression function: " + path); path.addLast(myPath); - ExpressionFunction.Instance instance = function.expand(context, getArguments().expressions(), path); + + String functionName = getName(); + String functionPropertyName = RankingExpression.propertyName(functionName); + boolean alreadySerialized = context.serializedFunctions().containsKey(functionPropertyName); + + if ( ! alreadySerialized || getArguments().size() > 0) { + ExpressionFunction.Instance instance = function.expand(context, getArguments().expressions(), path); + functionName = instance.getName(); + + context.addFunctionSerialization(RankingExpression.propertyName(functionName), instance.getExpressionString()); + for (Map.Entry<String, TensorType> argumentType : function.argumentTypes().entrySet()) + context.addArgumentTypeSerialization(functionName, argumentType.getKey(), argumentType.getValue()); + if (function.returnType().isPresent()) + context.addFunctionTypeSerialization(functionName, function.returnType().get()); + } path.removeLast(); - context.addFunctionSerialization(RankingExpression.propertyName(instance.getName()), instance.getExpressionString()); - return string.append("rankingExpression(").append(instance.getName()).append(')'); + return string.append("rankingExpression(").append(functionName).append(')'); } - // Not resolved in this context: output as-is return reference.toString(string, context, path, parent); } diff --git a/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/rule/SerializationContext.java b/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/rule/SerializationContext.java index d7807caa2b6..92889d6607b 100644 --- a/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/rule/SerializationContext.java +++ b/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/rule/SerializationContext.java @@ -95,6 +95,12 @@ public class SerializationContext extends FunctionReferenceContext { return new SerializationContext(functions(), bindings, this.serializedFunctions); } + /** Returns a fresh context without bindings */ + @Override + public SerializationContext withoutBindings() { + return new SerializationContext(functions(), null, this.serializedFunctions); + } + public Map<String, String> serializedFunctions() { return serializedFunctions; } } diff --git a/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/rule/TensorFunctionNode.java b/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/rule/TensorFunctionNode.java index 6e1cdf52158..befe2179dc1 100644 --- a/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/rule/TensorFunctionNode.java +++ b/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/rule/TensorFunctionNode.java @@ -333,10 +333,16 @@ public class TensorFunctionNode extends CompositeNode { /** Returns a new context with the bindings replaced by the given bindings */ @Override public ExpressionToStringContext withBindings(Map<String, String> bindings) { - return new ExpressionToStringContext(new SerializationContext(wrappedSerializationContext.functions().values(), bindings), - wrappedToStringContext, path, parent); + SerializationContext serializationContext = new SerializationContext(functions(), bindings, serializedFunctions()); + return new ExpressionToStringContext(serializationContext, wrappedToStringContext, path, parent); } + /** Returns a fresh context without bindings */ + @Override + public SerializationContext withoutBindings() { + SerializationContext serializationContext = new SerializationContext(functions(), null, serializedFunctions()); + return new ExpressionToStringContext(serializationContext, null, path, parent); + } } /** Turns an EvaluationContext into a Context */ diff --git a/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/transform/ConstantDereferencer.java b/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/transform/ConstantDereferencer.java index a541eac2421..fed4f60e983 100644 --- a/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/transform/ConstantDereferencer.java +++ b/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/transform/ConstantDereferencer.java @@ -1,7 +1,7 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.searchlib.rankingexpression.transform; -import com.yahoo.searchlib.rankingexpression.evaluation.TensorValue; +import com.yahoo.searchlib.rankingexpression.Reference; import com.yahoo.searchlib.rankingexpression.evaluation.Value; import com.yahoo.searchlib.rankingexpression.rule.CompositeNode; import com.yahoo.searchlib.rankingexpression.rule.ConstantNode; @@ -28,8 +28,15 @@ public class ConstantDereferencer extends ExpressionTransformer<TransformContext return node; } + /** Returns true if the given reference is an attribute, constant or query feature */ + private static boolean isSimpleFeature(Reference reference) { + if ( ! reference.isSimple()) return false; + String name = reference.name(); + return name.equals("attribute") || name.equals("constant") || name.equals("query"); + } + private ExpressionNode transformFeature(ReferenceNode node, TransformContext context) { - if (!node.getArguments().isEmpty()) + if ( ! node.getArguments().isEmpty() && ! isSimpleFeature(node.reference())) return transformArguments(node, context); else return transformConstantReference(node, context); @@ -44,7 +51,14 @@ public class ConstantDereferencer extends ExpressionTransformer<TransformContext } private ExpressionNode transformConstantReference(ReferenceNode node, TransformContext context) { - Value value = context.constants().get(node.getName()); + String name = node.getName(); + if (node.reference().name().equals("constant")) { + ExpressionNode arg = node.getArguments().expressions().get(0); + if (arg instanceof ReferenceNode) { + name = ((ReferenceNode)arg).getName(); + } + } + Value value = context.constants().get(name); // works if "constant(...)" is added if (value == null || value.type().rank() > 0) { return node; // not a number constant reference } diff --git a/searchlib/src/test/java/com/yahoo/searchlib/rankingexpression/evaluation/EvaluationTestCase.java b/searchlib/src/test/java/com/yahoo/searchlib/rankingexpression/evaluation/EvaluationTestCase.java index 807eb3aa7ce..6a9108311b1 100644 --- a/searchlib/src/test/java/com/yahoo/searchlib/rankingexpression/evaluation/EvaluationTestCase.java +++ b/searchlib/src/test/java/com/yahoo/searchlib/rankingexpression/evaluation/EvaluationTestCase.java @@ -10,7 +10,6 @@ import com.yahoo.searchlib.rankingexpression.rule.ArithmeticOperator; import com.yahoo.searchlib.rankingexpression.rule.ConstantNode; import com.yahoo.searchlib.rankingexpression.rule.ExpressionNode; import com.yahoo.searchlib.rankingexpression.rule.IfNode; -import com.yahoo.tensor.TensorType; import org.junit.Test; import static org.junit.Assert.assertEquals; @@ -390,6 +389,130 @@ public class EvaluationTestCase { } @Test + public void testTile() { + EvaluationTester tester = new EvaluationTester(); + + tester.assertEvaluates("tensor(d0[2],d1[4]):[1,2,1,2,3,4,3,4]", + "tensor(d0[2],d1[4])(tensor0{input0:(d0 % 2), input1:(d1 % 2) } )", + "tensor(input0[2],input1[2]):[1, 2, 3, 4]", + "tensor(repeats0[2]):[1,2]"); + + tester.assertEvaluates("tensor(d0[6],d1[2]):[1,2,3,4,1,2,3,4,1,2,3,4]", + "tensor(d0[6],d1[2])(tensor0{input0:(d0 % 2), input1:(d1 % 2) } )", + "tensor(input0[2],input1[2]):[1, 2, 3, 4]", + "tensor(repeats0[2]):[3,1]"); + } + + @Test + public void testReshape() { + EvaluationTester tester = new EvaluationTester(); + + tester.assertEvaluates("tensor(d0[4]):[1,2,3,4]", + "tensor(d0[4])(tensor0{a0:(d0 / 2), a1:(d0 % 2)})", + "tensor(a0[2],a1[2]):[1,2,3,4]", + "tensor(d0[1]):[4]"); + + tester.assertEvaluates("tensor(d0[2],d1[2]):[1,2,3,4]", + "tensor(d0[2],d1[2])(tensor0{a0:(d0), a1:(d1)})", + "tensor(a0[2],a1[2]):[1,2,3,4]", + "tensor(d0[2]):[2,2]"); + + tester.assertEvaluates("tensor(d0[2],d1[1],d2[2]):[1,2,3,4]", + "tensor(d0[2],d1[1],d2[2])(tensor0{a0:(d0), a1:(d2)})", + "tensor(a0[2],a1[2]):[1,2,3,4]", + "tensor(d0[3]):[2,1,2]"); + + tester.assertEvaluates("tensor(d0[3],d1[2]):[1,2,3,4,5,6]", + "tensor(d0[3],d1[2])(tensor0{a0:0, a1:((d0 * 2 + d1) / 3), a2:((d0 * 2 + d1) % 3) })", + "tensor(a0[1],a1[2],a2[3]):[1,2,3,4,5,6]", + "tensor(d0[2]):[3,2]"); + + tester.assertEvaluates("tensor(d0[3],d1[2],d2[1],d3[1]):[1,2,3,4,5,6]", + "tensor(d0[3],d1[2],d2[1],d3[1])(tensor0{a0:0, a1:((d0 * 2 + d1) / 3), a2:((d0 * 2 + d1) % 3) })", + "tensor(a0[1],a1[2],a2[3]):[1,2,3,4,5,6]", + "tensor(d0[4]):[3,2,-1,1]"); + + } + + @Test + public void testMatmul() { + EvaluationTester tester = new EvaluationTester(); + + tester.assertEvaluates("tensor():{91}", + "reduce(join(tensor0, tensor1, f(x,y)(x*y)), sum, d0)", + "tensor(d0[6]):[1,2,3,4,5,6]", + "tensor(d0[6]):[1,2,3,4,5,6]"); + + tester.assertEvaluates("tensor(d1[2]):[22, 28]", + "reduce(join(tensor0, tensor1, f(x,y)(x*y)), sum, d0)", + "tensor(d0[3]):[1,2,3]", + "tensor(d0[3],d1[2]):[1,2,3,4,5,6]"); + + tester.assertEvaluates("tensor(d1[2]):[22, 28]", + "reduce(join(tensor0, tensor1, f(x,y)(x*y)), sum, d0)", + "tensor(d0[3],d1[2]):[1,2,3,4,5,6]", + "tensor(d0[3]):[1,2,3]"); + + tester.assertEvaluates("tensor(d0[2],d2[2]):[22,28,49,64]", + "reduce(join(tensor0, tensor1, f(x,y)(x*y)), sum, d1)", + "tensor(d0[2],d1[3]):[1,2,3,4,5,6]", + "tensor(d1[3],d2[2]):[1,2,3,4,5,6]"); + + tester.assertEvaluates("tensor(d0[1],d1[2],d3[2]):[22,28,49,64]", + "reduce(join(tensor0, tensor1, f(x,y)(x*y)), sum, d2)", + "tensor(d0[1],d1[2],d2[3]):[1,2,3,4,5,6]", + "tensor(d2[3],d3[2]):[1,2,3,4,5,6]"); + + tester.assertEvaluates("tensor(d0[1],d1[2],d3[2]):[22,28,49,64]", + "reduce(join(tensor0, tensor1, f(x,y)(x*y)), sum, d2)", + "tensor(d1[2],d2[3]):[1,2,3,4,5,6]", + "tensor(d0[1],d2[3],d3[2]):[1,2,3,4,5,6]"); + + tester.assertEvaluates("tensor(d0[2],d1[2],d3[2]):[22,28,49,64,58,64,139,154]", + "reduce(join(tensor0{d0:0}, tensor1, f(x,y)(x*y)), sum, d2)", // notice peek + "tensor(d0[1],d1[2],d2[3]):[1,2,3,4,5,6]", + "tensor(d0[2],d2[3],d3[2]):[1,2,3,4,5,6,7,8,9,10,11,12]"); + + tester.assertEvaluates("tensor(d0[2],d1[2],d2[2],d4[2]):[22,28,49,64,58,64,139,154,76,100,103,136,220,244,301,334]", + "reduce(join(tensor0{d1:0}, tensor1{d0:0}, f(x,y)(x*y)), sum, d3)", // notice peeks + "tensor(d0[2],d1[1],d2[2],d3[3]):[1,2,3,4,5,6,7,8,9,10,11,12]", + "tensor(d0[1],d1[2],d3[3],d4[2]):[1,2,3,4,5,6,7,8,9,10,11,12]"); + + tester.assertEvaluates("tensor(d0[1],d1[4],d2[2],d4[2]):[22,28,49,64,220,244,301,334,634,676,769,820,1264,1324,1453,1522]", + "reduce(join(tensor0, tensor1, f(x,y)(x*y)), sum, d3)", + "tensor(d0[1],d1[4],d2[2],d3[3]):[1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24]", + "tensor(d0[1],d1[4],d3[3],d4[2]):[1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24]"); + } + + @Test + public void testSplit() { + EvaluationTester tester = new EvaluationTester(); + + tester.assertEvaluates("tensor(d0[3]):[1,2,3]", + "tensor(d0[3])(tensor0{input0:(d0)} )", + "tensor(input0[6]):[1,2,3,4,5,6]"); + tester.assertEvaluates("tensor(d0[3]):[4,5,6]", + "tensor(d0[3])(tensor0{input0:(d0+3)} )", + "tensor(input0[6]):[1,2,3,4,5,6]"); + tester.assertEvaluates("tensor(d0[4]):[3,4,5,6]", + "tensor(d0[4])(tensor0{input0:(d0+2)} )", + "tensor(input0[6]):[1,2,3,4,5,6]"); + tester.assertEvaluates("tensor(d0[2]):[3,4]", + "tensor(d0[2])(tensor0{input0:(d0+2)} )", + "tensor(input0[6]):[1,2,3,4,5,6]"); + tester.assertEvaluates("tensor(d0[2]):[5,6]", + "tensor(d0[2])(tensor0{input0:(d0+4)} )", + "tensor(input0[6]):[1,2,3,4,5,6]"); + + tester.assertEvaluates("tensor(d0[1],d1[3]):[1,2,3]", + "tensor(d0[1],d1[3])(tensor0{input0:(d0), input1:(d1)} )", + "tensor(input0[2],input1[3]):[[1,2,3],[4,5,6]]"); + tester.assertEvaluates("tensor(d0[1],d1[3]):[4,5,6]", + "tensor(d0[1],d1[3])(tensor0{input0:(d0+1), input1:(d1)} )", + "tensor(input0[2],input1[3]):[[1,2,3],[4,5,6]]"); + } + + @Test public void testTake() { EvaluationTester tester = new EvaluationTester(); diff --git a/searchlib/src/tests/attribute/attributefilewriter/attributefilewriter_test.cpp b/searchlib/src/tests/attribute/attributefilewriter/attributefilewriter_test.cpp index 8d13987bb4f..eeabbf23c27 100644 --- a/searchlib/src/tests/attribute/attributefilewriter/attributefilewriter_test.cpp +++ b/searchlib/src/tests/attribute/attributefilewriter/attributefilewriter_test.cpp @@ -6,7 +6,7 @@ #include <vespa/searchlib/attribute/attributefilebufferwriter.h> #include <vespa/searchlib/attribute/attribute_header.h> #include <vespa/searchlib/util/fileutil.h> -#include <vespa/searchlib/util/rand48.h> +#include <vespa/vespalib/util/rand48.h> #include <vespa/searchlib/common/tunefileinfo.h> #include <vespa/searchlib/common/fileheadercontext.h> #include <vespa/searchlib/index/dummyfileheadercontext.h> @@ -77,7 +77,7 @@ TEST_F("Test that buffer writer passes on written data", Fixture) const size_t writerBufferSize = AttributeFileBufferWriter::BUFFER_SIZE; EXPECT_GREATER(mysize * sizeof(int), writerBufferSize); a.reserve(mysize); - search::Rand48 rnd; + vespalib::Rand48 rnd; for (uint32_t i = 0; i < mysize; ++i) { a.emplace_back(rnd.lrand48()); } diff --git a/searchlib/src/tests/attribute/enumeratedsave/enumeratedsave_test.cpp b/searchlib/src/tests/attribute/enumeratedsave/enumeratedsave_test.cpp index 41313fc7c53..eeaf85472c3 100644 --- a/searchlib/src/tests/attribute/enumeratedsave/enumeratedsave_test.cpp +++ b/searchlib/src/tests/attribute/enumeratedsave/enumeratedsave_test.cpp @@ -227,7 +227,7 @@ void EnumeratedSaveTest::populate(IntegerAttribute &v, unsigned seed, BasicType bt) { - search::Rand48 rnd; + vespalib::Rand48 rnd; IntegerAttribute::largeint_t mask(std::numeric_limits <IntegerAttribute::largeint_t>::max()); switch (bt.type()) { @@ -284,7 +284,7 @@ EnumeratedSaveTest::populate(FloatingPointAttribute &v, unsigned seed, BasicType bt) { (void) bt; - search::Rand48 rnd; + vespalib::Rand48 rnd; rnd.srand48(seed); int weight = 1; for(size_t i(0), m(v.getNumDocs()); i < m; i++) { diff --git a/searchlib/src/tests/attribute/multi_value_mapping/multi_value_mapping_test.cpp b/searchlib/src/tests/attribute/multi_value_mapping/multi_value_mapping_test.cpp index 2f74e9d31f6..a936ba124ee 100644 --- a/searchlib/src/tests/attribute/multi_value_mapping/multi_value_mapping_test.cpp +++ b/searchlib/src/tests/attribute/multi_value_mapping/multi_value_mapping_test.cpp @@ -3,7 +3,7 @@ #include <vespa/searchlib/attribute/multi_value_mapping.h> #include <vespa/searchlib/attribute/multi_value_mapping.hpp> #include <vespa/searchlib/attribute/not_implemented_attribute.h> -#include <vespa/searchlib/util/rand48.h> +#include <vespa/vespalib/util/rand48.h> #include <vespa/vespalib/gtest/gtest.h> #include <vespa/vespalib/stllike/hash_set.h> #include <vespa/vespalib/test/insertion_operators.h> @@ -151,7 +151,7 @@ using IntMappingTest = MappingTestBase<int>; class CompactionIntMappingTest : public MappingTestBase<int> { - search::Rand48 _rnd; + vespalib::Rand48 _rnd; std::map<uint32_t, std::vector<int>> _refMapping; public: CompactionIntMappingTest() diff --git a/searchlib/src/tests/attribute/postinglist/postinglist.cpp b/searchlib/src/tests/attribute/postinglist/postinglist.cpp index 78b93e2b78b..37d76e4a9e8 100644 --- a/searchlib/src/tests/attribute/postinglist/postinglist.cpp +++ b/searchlib/src/tests/attribute/postinglist/postinglist.cpp @@ -7,7 +7,7 @@ #include <vespa/vespalib/btree/btreeroot.hpp> #include <vespa/vespalib/btree/btreestore.hpp> #include <vespa/vespalib/datastore/datastore.h> -#include <vespa/searchlib/util/rand48.h> +#include <vespa/vespalib/util/rand48.h> #include <vespa/vespalib/testkit/testapp.h> #include <set> #include <map> @@ -143,7 +143,7 @@ private: PostingList *_intPostings; STLValueTree *_stlTree; - Rand48 _randomGenerator; + vespalib::Rand48 _randomGenerator; uint32_t _generation; void diff --git a/searchlib/src/tests/common/bitvector/bitvector_test.cpp b/searchlib/src/tests/common/bitvector/bitvector_test.cpp index 409cc9f2725..db29ab9881f 100644 --- a/searchlib/src/tests/common/bitvector/bitvector_test.cpp +++ b/searchlib/src/tests/common/bitvector/bitvector_test.cpp @@ -8,7 +8,7 @@ #include <vespa/searchlib/common/bitvectoriterator.h> #include <vespa/searchlib/fef/termfieldmatchdata.h> #include <vespa/searchlib/fef/termfieldmatchdataarray.h> -#include <vespa/searchlib/util/rand48.h> +#include <vespa/vespalib/util/rand48.h> #include <algorithm> using namespace search; @@ -81,7 +81,7 @@ myCountInterval(const BitVector &bv, uint32_t low, uint32_t high) } void -scan(uint32_t count, uint32_t offset, uint32_t size, Rand48 &rnd) +scan(uint32_t count, uint32_t offset, uint32_t size, vespalib::Rand48 &rnd) { std::vector<uint32_t> lids; lids.reserve(count); @@ -110,7 +110,7 @@ scan(uint32_t count, uint32_t offset, uint32_t size, Rand48 &rnd) void scanWithOffset(uint32_t offset) { - Rand48 rnd; + vespalib::Rand48 rnd; rnd.srand48(32); scan(10, offset, 1000000, rnd); diff --git a/searchlib/src/tests/diskindex/fieldwriter/fieldwriter_test.cpp b/searchlib/src/tests/diskindex/fieldwriter/fieldwriter_test.cpp index ef55ea60c2b..15a53497db9 100644 --- a/searchlib/src/tests/diskindex/fieldwriter/fieldwriter_test.cpp +++ b/searchlib/src/tests/diskindex/fieldwriter/fieldwriter_test.cpp @@ -1,7 +1,7 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include <vespa/searchlib/common/bitvector.h> -#include <vespa/searchlib/util/rand48.h> +#include <vespa/vespalib/util/rand48.h> #include <vespa/searchlib/test/fakedata/fakeword.h> #include <vespa/searchlib/test/fakedata/fakewordset.h> #include <vespa/searchlib/index/docidandfeatures.h> @@ -99,7 +99,7 @@ private: FakeWordSet _wordSet; FakeWordSet _wordSet2; public: - search::Rand48 _rnd; + vespalib::Rand48 _rnd; private: void Usage(); diff --git a/searchlib/src/tests/diskindex/pagedict4/pagedict4test.cpp b/searchlib/src/tests/diskindex/pagedict4/pagedict4test.cpp index e914fe7c559..6cc2f318f4b 100644 --- a/searchlib/src/tests/diskindex/pagedict4/pagedict4test.cpp +++ b/searchlib/src/tests/diskindex/pagedict4/pagedict4test.cpp @@ -1,7 +1,7 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include <vespa/searchlib/bitcompression/compression.h> -#include <vespa/searchlib/util/rand48.h> +#include <vespa/vespalib/util/rand48.h> #include <vespa/searchlib/index/schemautil.h> #include <vespa/searchlib/bitcompression/countcompression.h> #include <vespa/searchlib/bitcompression/pagedict4.h> @@ -54,7 +54,7 @@ using RandReader = search::diskindex::test::PageDict4MemRandReader; class PageDict4TestApp : public FastOS_Application { public: - search::Rand48 _rnd; + vespalib::Rand48 _rnd; bool _stress; bool _emptyWord; bool _firstWordForcedCommon; @@ -206,7 +206,7 @@ deDup(std::vector<uint32_t> &v) static WordIndexCounts -makeIndex(search::Rand48 &rnd, bool forceCommon) +makeIndex(vespalib::Rand48 &rnd, bool forceCommon) { uint64_t bitLength = 10; uint32_t numDocs = 1; @@ -219,7 +219,7 @@ makeIndex(search::Rand48 &rnd, bool forceCommon) void -makeIndexes(search::Rand48 &rnd, +makeIndexes(vespalib::Rand48 &rnd, WordIndexCounts &counts, bool forceCommon) { @@ -229,7 +229,7 @@ makeIndexes(search::Rand48 &rnd, static void makeWords(std::vector<WordCounts> &v, - search::Rand48 &rnd, + vespalib::Rand48 &rnd, uint32_t numWordIds, uint32_t tupleCount, bool emptyWord, @@ -355,7 +355,7 @@ checkCounts(const std::string &word, void testWords(const std::string &logname, - search::Rand48 &rnd, + vespalib::Rand48 &rnd, uint64_t numWordIds, uint32_t tupleCount, uint32_t chunkSize, diff --git a/searchlib/src/tests/features/beta/beta_features.cpp b/searchlib/src/tests/features/beta/beta_features.cpp index 617bf2fb073..89a72fab6fc 100644 --- a/searchlib/src/tests/features/beta/beta_features.cpp +++ b/searchlib/src/tests/features/beta/beta_features.cpp @@ -31,7 +31,7 @@ #include <vespa/searchlib/fef/featurenamebuilder.h> #include <vespa/searchlib/fef/indexproperties.h> #include <vespa/searchlib/fef/test/plugin/setup.h> -#include <vespa/searchlib/util/rand48.h> +#include <vespa/vespalib/util/rand48.h> #include <vespa/searchlib/fef/test/ftlib.h> #include <vespa/vespalib/util/stringfmt.h> diff --git a/searchlib/src/tests/features/prod_features.cpp b/searchlib/src/tests/features/prod_features.cpp index 1c07c81bc2f..3f71c9f85ea 100644 --- a/searchlib/src/tests/features/prod_features.cpp +++ b/searchlib/src/tests/features/prod_features.cpp @@ -40,7 +40,7 @@ #include <vespa/searchlib/fef/queryproperties.h> #include <vespa/searchlib/fef/test/plugin/setup.h> #include <vespa/searchlib/fef/test/dummy_dependency_handler.h> -#include <vespa/searchlib/util/rand48.h> +#include <vespa/vespalib/util/rand48.h> #include <vespa/vespalib/util/stringfmt.h> #include <vespa/vespalib/geo/zcurve.h> #include <vespa/vespalib/util/string_hash.h> @@ -1747,7 +1747,7 @@ Test::testRandom() FtFeatureTest ft(_factory, "random"); ft.getIndexEnv().getProperties().add("random.seed", "100"); ASSERT_TRUE(ft.setup()); - search::Rand48 rnd; + vespalib::Rand48 rnd; rnd.srand48(100); for (uint32_t i = 0; i < 5; ++i) { feature_t exp = static_cast<feature_t>(rnd.lrand48()) / static_cast<feature_t>(0x80000000u); @@ -1770,7 +1770,7 @@ Test::testRandom() FtFeatureTest ft(_factory, "random.match"); ft.getQueryEnv().getProperties().add("random.match.seed", "100"); ASSERT_TRUE(ft.setup()); - search::Rand48 rnd; + vespalib::Rand48 rnd; for (uint32_t i = 1; i <= 5; ++i) { rnd.srand48(100 + i); // seed + lid feature_t exp = static_cast<feature_t>(rnd.lrand48()) / static_cast<feature_t>(0x80000000u); diff --git a/searchlib/src/tests/postinglistbm/posting_list_test.cpp b/searchlib/src/tests/postinglistbm/posting_list_test.cpp index a7d810cb1a1..845201772f8 100644 --- a/searchlib/src/tests/postinglistbm/posting_list_test.cpp +++ b/searchlib/src/tests/postinglistbm/posting_list_test.cpp @@ -5,7 +5,7 @@ #include <vespa/searchlib/test/fakedata/fakeword.h> #include <vespa/searchlib/test/fakedata/fakewordset.h> #include <vespa/searchlib/test/fakedata/fpfactory.h> -#include <vespa/searchlib/util/rand48.h> +#include <vespa/vespalib/util/rand48.h> #include <vespa/vespalib/gtest/gtest.h> #include <cinttypes> @@ -66,7 +66,7 @@ struct PostingListTest : public ::testing::Test { FakeWordUP word3; FakeWordUP word4; FakeWordUP word5; - search::Rand48 rnd; + vespalib::Rand48 rnd; PostingListTest() : num_docs(36000), diff --git a/searchlib/src/tests/postinglistbm/postinglistbm.cpp b/searchlib/src/tests/postinglistbm/postinglistbm.cpp index 16b8e9cd7f5..0ac3f15e015 100644 --- a/searchlib/src/tests/postinglistbm/postinglistbm.cpp +++ b/searchlib/src/tests/postinglistbm/postinglistbm.cpp @@ -10,7 +10,7 @@ #include <vespa/searchlib/test/fakedata/fakeword.h> #include <vespa/searchlib/test/fakedata/fakewordset.h> #include <vespa/searchlib/test/fakedata/fpfactory.h> -#include <vespa/searchlib/util/rand48.h> +#include <vespa/vespalib/util/rand48.h> #include <vespa/log/log.h> @@ -42,7 +42,7 @@ private: bool _unpack; public: - search::Rand48 _rnd; + vespalib::Rand48 _rnd; public: PostingListBM(); diff --git a/searchlib/src/tests/postinglistbm/stress_runner.cpp b/searchlib/src/tests/postinglistbm/stress_runner.cpp index 9c78515a03f..fd12e9b5aa9 100644 --- a/searchlib/src/tests/postinglistbm/stress_runner.cpp +++ b/searchlib/src/tests/postinglistbm/stress_runner.cpp @@ -33,7 +33,7 @@ private: StressMaster &operator=(const StressMaster &); - search::Rand48 &_rnd; + vespalib::Rand48 &_rnd; uint32_t _numDocs; std::vector<std::string> _postingTypes; StressRunner::OperatorType _operatorType; @@ -62,7 +62,7 @@ private: std::vector<Task> _tasks; public: - StressMaster(search::Rand48 &rnd, + StressMaster(vespalib::Rand48 &rnd, FakeWordSet &wordSet, const std::vector<std::string> &postingType, StressRunner::OperatorType operatorType, @@ -129,7 +129,7 @@ public: }; -StressMaster::StressMaster(search::Rand48 &rnd, +StressMaster::StressMaster(vespalib::Rand48 &rnd, FakeWordSet &wordSet, const std::vector<std::string> &postingTypes, StressRunner::OperatorType operatorType, @@ -420,7 +420,7 @@ OrStressWorker::run_task(const FakePosting& f1, const FakePosting& f2, uint32_t } void -StressRunner::run(search::Rand48 &rnd, +StressRunner::run(vespalib::Rand48 &rnd, FakeWordSet &wordSet, const std::vector<std::string> &postingTypes, OperatorType operatorType, diff --git a/searchlib/src/tests/postinglistbm/stress_runner.h b/searchlib/src/tests/postinglistbm/stress_runner.h index d4974bc969e..249645f70aa 100644 --- a/searchlib/src/tests/postinglistbm/stress_runner.h +++ b/searchlib/src/tests/postinglistbm/stress_runner.h @@ -4,12 +4,9 @@ #include <string> #include <vector> -namespace search { -class Rand48; +namespace vespalib { class Rand48; } -namespace fakedata { class FakeWordSet; } - -} +namespace search::fakedata { class FakeWordSet; } namespace postinglistbm { @@ -21,7 +18,7 @@ public: Or }; - static void run(search::Rand48 &rnd, + static void run(vespalib::Rand48 &rnd, search::fakedata::FakeWordSet &wordSet, const std::vector<std::string> &postingTypes, OperatorType operatorType, diff --git a/searchlib/src/tests/queryeval/sparse_vector_benchmark/sparse_vector_benchmark_test.cpp b/searchlib/src/tests/queryeval/sparse_vector_benchmark/sparse_vector_benchmark_test.cpp index a8aab279a98..1cf39183206 100644 --- a/searchlib/src/tests/queryeval/sparse_vector_benchmark/sparse_vector_benchmark_test.cpp +++ b/searchlib/src/tests/queryeval/sparse_vector_benchmark/sparse_vector_benchmark_test.cpp @@ -9,7 +9,7 @@ #include <vespa/searchlib/queryeval/andnotsearch.h> #include <vespa/searchlib/queryeval/andsearch.h> #include <vespa/searchlib/queryeval/dot_product_search.h> -#include <vespa/searchlib/util/rand48.h> +#include <vespa/vespalib/util/rand48.h> #include <vespa/searchlib/queryeval/orsearch.h> #include <vespa/searchlib/queryeval/simpleresult.h> #include <vespa/searchlib/queryeval/wand/weak_and_search.h> diff --git a/searchlib/src/tests/transactionlogstress/translogstress.cpp b/searchlib/src/tests/transactionlogstress/translogstress.cpp index 74e48081e17..81a3006dbff 100644 --- a/searchlib/src/tests/transactionlogstress/translogstress.cpp +++ b/searchlib/src/tests/transactionlogstress/translogstress.cpp @@ -3,7 +3,7 @@ #include <vespa/vespalib/util/stringfmt.h> #include <vespa/searchlib/transactionlog/translogserver.h> #include <vespa/searchlib/transactionlog/translogclient.h> -#include <vespa/searchlib/util/rand48.h> +#include <vespa/vespalib/util/rand48.h> #include <vespa/searchlib/util/runnable.h> #include <vespa/searchlib/index/dummyfileheadercontext.h> #include <vespa/fastos/app.h> @@ -37,7 +37,7 @@ using Visitor = TransLogClient::Visitor; class BufferGenerator { private: - Rand48 _rnd; + vespalib::Rand48 _rnd; uint32_t _minStrLen; uint32_t _maxStrLen; @@ -71,7 +71,7 @@ BufferGenerator::getRandomBuffer() class EntryGenerator { private: - Rand48 _rnd; + vespalib::Rand48 _rnd; long _baseSeed; BufferGenerator _bufferGenerator; const std::vector<nbostream> * _buffers; @@ -93,7 +93,7 @@ public: }; SerialNum getRandomSerialNum(SerialNum begin, SerialNum end); Packet::Entry getRandomEntry(SerialNum num); - Rand48 & getRnd() { return _rnd; } + vespalib::Rand48 & getRnd() { return _rnd; } void setBuffers(const std::vector<nbostream> & buffers) { _buffers = &buffers; } diff --git a/searchlib/src/tests/util/bufferwriter/bufferwriter_test.cpp b/searchlib/src/tests/util/bufferwriter/bufferwriter_test.cpp index 33afa67e660..8787712663d 100644 --- a/searchlib/src/tests/util/bufferwriter/bufferwriter_test.cpp +++ b/searchlib/src/tests/util/bufferwriter/bufferwriter_test.cpp @@ -4,7 +4,7 @@ #include <vespa/vespalib/stllike/string.h> #include <vespa/vespalib/util/bufferwriter.h> #include <vespa/searchlib/util/drainingbufferwriter.h> -#include <vespa/searchlib/util/rand48.h> +#include <vespa/vespalib/util/rand48.h> namespace search { @@ -130,7 +130,7 @@ TEST("Test that bufferwriter passes on written data") const size_t drainerBufferSize = DrainingBufferWriter::BUFFER_SIZE; EXPECT_GREATER(mysize * sizeof(int), drainerBufferSize); a.reserve(mysize); - search::Rand48 rnd; + vespalib::Rand48 rnd; for (uint32_t i = 0; i < mysize; ++i) { a.emplace_back(rnd.lrand48()); } diff --git a/searchlib/src/vespa/searchlib/features/randomfeature.h b/searchlib/src/vespa/searchlib/features/randomfeature.h index 8c94b71071f..3a93e77d4d0 100644 --- a/searchlib/src/vespa/searchlib/features/randomfeature.h +++ b/searchlib/src/vespa/searchlib/features/randomfeature.h @@ -3,7 +3,7 @@ #pragma once #include <vespa/searchlib/fef/blueprint.h> -#include <vespa/searchlib/util/rand48.h> +#include <vespa/vespalib/util/rand48.h> namespace search::features { @@ -12,8 +12,8 @@ namespace search::features { **/ class RandomExecutor : public search::fef::FeatureExecutor { private: - Rand48 _rnd; // seeded once per query - Rand48 _matchRnd; // seeded once per match + vespalib::Rand48 _rnd; // seeded once per query + vespalib::Rand48 _matchRnd; // seeded once per match uint64_t _matchSeed; public: diff --git a/searchlib/src/vespa/searchlib/test/fakedata/fakeword.cpp b/searchlib/src/vespa/searchlib/test/fakedata/fakeword.cpp index 19080acaad3..4681861dd73 100644 --- a/searchlib/src/vespa/searchlib/test/fakedata/fakeword.cpp +++ b/searchlib/src/vespa/searchlib/test/fakedata/fakeword.cpp @@ -31,7 +31,7 @@ namespace fakedata static void fillbitset(search::BitVector *bitvector, unsigned int size, - search::Rand48 &rnd) + vespalib::Rand48 &rnd) { unsigned int range; unsigned int idx; @@ -70,7 +70,7 @@ static void fillcorrelatedbitset(search::BitVector &bitvector, unsigned int size, const FakeWord &otherword, - search::Rand48 &rnd) + vespalib::Rand48 &rnd) { const FakeWord::DocWordFeatureList &opostings = otherword._postings; @@ -145,7 +145,7 @@ FakeWord::FakeWord(uint32_t docIdLimit, for (uint32_t docId : docIds) { bitmap->setBit(docId); } - search::Rand48 rnd; + vespalib::Rand48 rnd; fakeup(*bitmap, rnd, _postings, _wordPosFeatures); } @@ -153,7 +153,7 @@ FakeWord::FakeWord(uint32_t docIdLimit, uint32_t wordDocs, uint32_t tempWordDocs, const std::string &name, - search::Rand48 &rnd, + vespalib::Rand48 &rnd, const PosOccFieldsParams &fieldsParams, uint32_t packedIndex) : _postings(), @@ -181,7 +181,7 @@ FakeWord::FakeWord(uint32_t docIdLimit, const std::string &name, const FakeWord &otherWord, size_t overlapDocs, - search::Rand48 &rnd, + vespalib::Rand48 &rnd, const PosOccFieldsParams &fieldsParams, uint32_t packedIndex) : _postings(), @@ -211,7 +211,7 @@ FakeWord::~FakeWord() void FakeWord::fakeup(search::BitVector &bitmap, - search::Rand48 &rnd, + vespalib::Rand48 &rnd, DocWordFeatureList &postings, DocWordPosFeatureList &wordPosFeatures) { @@ -301,7 +301,7 @@ FakeWord::fakeup(search::BitVector &bitmap, void -FakeWord::fakeupTemps(search::Rand48 &rnd, +FakeWord::fakeupTemps(vespalib::Rand48 &rnd, uint32_t docIdLimit, uint32_t tempWordDocs) { @@ -315,7 +315,7 @@ FakeWord::fakeupTemps(search::Rand48 &rnd, } void -FakeWord::setupRandomizer(search::Rand48 &rnd) +FakeWord::setupRandomizer(vespalib::Rand48 &rnd) { typedef DocWordFeatureList DWFL; Randomizer randomAdd; diff --git a/searchlib/src/vespa/searchlib/test/fakedata/fakeword.h b/searchlib/src/vespa/searchlib/test/fakedata/fakeword.h index 30f960c2d4c..ffe9e2e54d0 100644 --- a/searchlib/src/vespa/searchlib/test/fakedata/fakeword.h +++ b/searchlib/src/vespa/searchlib/test/fakedata/fakeword.h @@ -1,7 +1,7 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #pragma once -#include <vespa/searchlib/util/rand48.h> +#include <vespa/vespalib/util/rand48.h> #include <vector> #include <vespa/searchlib/queryeval/searchiterator.h> #include <vespa/searchlib/bitcompression/compression.h> @@ -156,16 +156,16 @@ public: void fakeup(search::BitVector &bitmap, - search::Rand48 &rnd, + vespalib::Rand48 &rnd, DocWordFeatureList &postings, DocWordPosFeatureList &wordPosFeatures); void - fakeupTemps(search::Rand48 &rnd, + fakeupTemps(vespalib::Rand48 &rnd, uint32_t docIdLimit, uint32_t tempWordDocs); - void setupRandomizer(search::Rand48 &rnd); + void setupRandomizer(vespalib::Rand48 &rnd); const DocWordFeature & getDocWordFeature(const Randomizer &r) const @@ -220,7 +220,7 @@ public: uint32_t wordDocs, uint32_t tempWordDocs, const std::string &name, - search::Rand48 &rnd, + vespalib::Rand48 &rnd, const PosOccFieldsParams &fieldsParams, uint32_t packedIndex); @@ -230,7 +230,7 @@ public: const std::string &name, const FakeWord &otherWord, size_t overlapDocs, - search::Rand48 &rnd, + vespalib::Rand48 &rnd, const PosOccFieldsParams &fieldsParams, uint32_t packedIndex); diff --git a/searchlib/src/vespa/searchlib/test/fakedata/fakewordset.cpp b/searchlib/src/vespa/searchlib/test/fakedata/fakewordset.cpp index ba691e507fa..6bb39598fab 100644 --- a/searchlib/src/vespa/searchlib/test/fakedata/fakewordset.cpp +++ b/searchlib/src/vespa/searchlib/test/fakedata/fakewordset.cpp @@ -77,7 +77,7 @@ FakeWordSet::setupParams(bool hasElements, } void -FakeWordSet::setupWords(search::Rand48 &rnd, +FakeWordSet::setupWords(vespalib::Rand48 &rnd, uint32_t numDocs, uint32_t commonDocFreq, uint32_t numWordsPerWordClass) @@ -86,7 +86,7 @@ FakeWordSet::setupWords(search::Rand48 &rnd, } void -FakeWordSet::setupWords(search::Rand48 &rnd, +FakeWordSet::setupWords(vespalib::Rand48 &rnd, uint32_t numDocs, uint32_t commonDocFreq, uint32_t mediumDocFreq, diff --git a/searchlib/src/vespa/searchlib/test/fakedata/fakewordset.h b/searchlib/src/vespa/searchlib/test/fakedata/fakewordset.h index 3dcf47cf8c8..de7efef1517 100644 --- a/searchlib/src/vespa/searchlib/test/fakedata/fakewordset.h +++ b/searchlib/src/vespa/searchlib/test/fakedata/fakewordset.h @@ -6,7 +6,7 @@ #include <vespa/searchlib/bitcompression/posocccompression.h> #include <vespa/searchlib/bitcompression/posocc_fields_params.h> -namespace search { class Rand48; } +namespace vespalib { class Rand48; } namespace search::fakedata { @@ -46,12 +46,12 @@ public: void setupParams(bool hasElements, bool hasElementWeights); - void setupWords(search::Rand48 &rnd, + void setupWords(vespalib::Rand48 &rnd, uint32_t numDocs, uint32_t commonDocFreq, uint32_t numWordsPerWordClass); - void setupWords(search::Rand48 &rnd, + void setupWords(vespalib::Rand48 &rnd, uint32_t numDocs, uint32_t commonDocFreq, uint32_t mediumDocFreq, diff --git a/searchlib/src/vespa/searchlib/util/rand48.h b/searchlib/src/vespa/searchlib/util/rand48.h deleted file mode 100644 index 8ef529493c1..00000000000 --- a/searchlib/src/vespa/searchlib/util/rand48.h +++ /dev/null @@ -1,43 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#pragma once - -#include <cstdlib> -#include <cstdint> - -namespace search { - -/* - * Simple random generator based on lrand48() spec. - */ -class Rand48 -{ -private: - uint64_t _state; -public: - void - srand48(long seed) - { - _state = ((static_cast<uint64_t>(seed & 0xffffffffu)) << 16) + 0x330e; - } - - Rand48() - : _state(0) - { - srand48(0x1234abcd); - }; - void iterate() { - _state = (UINT64_C(0x5DEECE66D) * _state + 0xb) & - UINT64_C(0xFFFFFFFFFFFF); - } - /* - * Return value from 0 to 2^31 - 1 - */ - long lrand48() { - iterate(); - return static_cast<long>(_state >> 17); - } -}; - -} // namespace search - diff --git a/searchlib/src/vespa/searchlib/util/random_normal.h b/searchlib/src/vespa/searchlib/util/random_normal.h index 74596066312..c74935de02f 100644 --- a/searchlib/src/vespa/searchlib/util/random_normal.h +++ b/searchlib/src/vespa/searchlib/util/random_normal.h @@ -2,7 +2,7 @@ #pragma once -#include <vespa/searchlib/util/rand48.h> +#include <vespa/vespalib/util/rand48.h> #include <cmath> namespace search { @@ -14,7 +14,7 @@ namespace search { class RandomNormal { private: - Rand48 _rnd; + vespalib::Rand48 _rnd; double _mean; double _stddev; diff --git a/searchlib/src/vespa/searchlib/util/randomgenerator.h b/searchlib/src/vespa/searchlib/util/randomgenerator.h index 40305a1d02d..ac23b387740 100644 --- a/searchlib/src/vespa/searchlib/util/randomgenerator.h +++ b/searchlib/src/vespa/searchlib/util/randomgenerator.h @@ -2,7 +2,7 @@ #pragma once -#include <vespa/searchlib/util/rand48.h> +#include <vespa/vespalib/util/rand48.h> #include <vespa/vespalib/stllike/string.h> #include <vector> #include <cassert> @@ -11,7 +11,7 @@ namespace search { class RandomGenerator { private: - Rand48 _rnd; + vespalib::Rand48 _rnd; public: RandomGenerator() : _rnd() {} diff --git a/vespalib/src/tests/btree/btree_test.cpp b/vespalib/src/tests/btree/btree_test.cpp index 4090283c10f..042a1f69bb2 100644 --- a/vespalib/src/tests/btree/btree_test.cpp +++ b/vespalib/src/tests/btree/btree_test.cpp @@ -673,7 +673,7 @@ void generateData(std::vector<LeafPair> & data, size_t numEntries) { data.reserve(numEntries); - Rand48 rnd; + vespalib::Rand48 rnd; rnd.srand48(10); for (size_t i = 0; i < numEntries; ++i) { int num = rnd.lrand48() % 10000000; diff --git a/vespalib/src/tests/btree/btreeaggregation_test.cpp b/vespalib/src/tests/btree/btreeaggregation_test.cpp index 7636ab40b59..2b4b37af171 100644 --- a/vespalib/src/tests/btree/btreeaggregation_test.cpp +++ b/vespalib/src/tests/btree/btreeaggregation_test.cpp @@ -710,7 +710,7 @@ void generateData(std::vector<LeafPair> & data, size_t numEntries) { data.reserve(numEntries); - Rand48 rnd; + vespalib::Rand48 rnd; rnd.srand48(10); for (size_t i = 0; i < numEntries; ++i) { int num = rnd.lrand48() % 10000000; diff --git a/vespalib/src/tests/btree/frozenbtree_test.cpp b/vespalib/src/tests/btree/frozenbtree_test.cpp index 1826be19a0a..9bf6f3773eb 100644 --- a/vespalib/src/tests/btree/frozenbtree_test.cpp +++ b/vespalib/src/tests/btree/frozenbtree_test.cpp @@ -47,7 +47,7 @@ private: NodeAllocator *_allocator; Tree *_tree; - Rand48 _randomGenerator; + vespalib::Rand48 _randomGenerator; void allocTree(); void freeTree(bool verbose); diff --git a/vespalib/src/vespa/vespalib/util/rand48.h b/vespalib/src/vespa/vespalib/util/rand48.h index 441f8e6e10f..3ebfe00ba35 100644 --- a/vespalib/src/vespa/vespalib/util/rand48.h +++ b/vespalib/src/vespa/vespalib/util/rand48.h @@ -2,8 +2,10 @@ #pragma once +#include <cstdlib> +#include <cstdint> -namespace search { +namespace vespalib { /* * Simple random generator based on lrand48() spec. @@ -19,19 +21,19 @@ public: _state = ((static_cast<uint64_t>(seed & 0xffffffffu)) << 16) + 0x330e; } - Rand48(void) + Rand48() : _state(0) { srand48(0x1234abcd); }; - void iterate(void) { + void iterate() { _state = (UINT64_C(0x5DEECE66D) * _state + 0xb) & UINT64_C(0xFFFFFFFFFFFF); } /* * Return value from 0 to 2^31 - 1 */ - long lrand48(void) { + long lrand48() { iterate(); return static_cast<long>(_state >> 17); } diff --git a/vespamalloc/src/tests/allocfree/CMakeLists.txt b/vespamalloc/src/tests/allocfree/CMakeLists.txt index 52c0c03d079..34d402d0c60 100644 --- a/vespamalloc/src/tests/allocfree/CMakeLists.txt +++ b/vespamalloc/src/tests/allocfree/CMakeLists.txt @@ -21,7 +21,7 @@ vespa_add_executable(vespamalloc_linklist_test_app $<TARGET_OBJECTS:vespamalloc_util> DEPENDS dl - atomic + ${VESPA_ATOMIC_LIB} ) vespa_add_test(NAME vespamalloc_allocfree_shared_test_app NO_VALGRIND COMMAND ${CMAKE_CURRENT_SOURCE_DIR}/allocfree_test.sh BENCHMARK DEPENDS vespamalloc_realloc_test_app vespamalloc_allocfree_shared_test_app vespamalloc_linklist_test_app diff --git a/vespamalloc/src/tests/test1/CMakeLists.txt b/vespamalloc/src/tests/test1/CMakeLists.txt index cbfb9543cda..cade2e092b4 100644 --- a/vespamalloc/src/tests/test1/CMakeLists.txt +++ b/vespamalloc/src/tests/test1/CMakeLists.txt @@ -3,6 +3,6 @@ vespa_add_executable(vespamalloc_testatomic_app TEST SOURCES testatomic.cpp DEPENDS - atomic + ${VESPA_ATOMIC_LIB} ) vespa_add_test(NAME vespamalloc_testatomic_app NO_VALGRIND COMMAND vespamalloc_testatomic_app) diff --git a/vespamalloc/src/vespamalloc/CMakeLists.txt b/vespamalloc/src/vespamalloc/CMakeLists.txt index ac7fd300828..1fe550ad95a 100644 --- a/vespamalloc/src/vespamalloc/CMakeLists.txt +++ b/vespamalloc/src/vespamalloc/CMakeLists.txt @@ -5,7 +5,7 @@ vespa_add_library(vespamalloc $<TARGET_OBJECTS:vespamalloc_util> INSTALL lib64/vespa/malloc DEPENDS - atomic + ${VESPA_ATOMIC_LIB} dl ) vespa_add_library(vespamallocd @@ -14,7 +14,7 @@ vespa_add_library(vespamallocd $<TARGET_OBJECTS:vespamalloc_util> INSTALL lib64/vespa/malloc DEPENDS - atomic + ${VESPA_ATOMIC_LIB} dl ) vespa_add_library(vespamallocdst16 @@ -23,7 +23,7 @@ vespa_add_library(vespamallocdst16 $<TARGET_OBJECTS:vespamalloc_util> INSTALL lib64/vespa/malloc DEPENDS - atomic + ${VESPA_ATOMIC_LIB} dl ) vespa_add_library(vespamallocdst16_nl @@ -32,7 +32,7 @@ vespa_add_library(vespamallocdst16_nl $<TARGET_OBJECTS:vespamalloc_util> INSTALL lib64/vespa/malloc DEPENDS - atomic + ${VESPA_ATOMIC_LIB} dl ) vespa_add_library(vespammap |