diff options
15 files changed, 99 insertions, 68 deletions
diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionWithTensorFlowTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionWithTensorFlowTestCase.java index 83cc3ae418a..3e11eb72a30 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionWithTensorFlowTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionWithTensorFlowTestCase.java @@ -250,46 +250,6 @@ public class RankingExpressionWithTensorFlowTestCase { } } - @Test - public void testImportingFromStoredExpressionsWithSmallConstants() throws IOException { - final String expression = "join(rename(reduce(join(map(join(rename(reduce(join(join(join(constant(\"dnn_hidden1_mul_x\"), join(rename(reduce(join(input, rename(constant(\"dnn_hidden1_weights\"), (d0, d1), (d1, d3)), f(a,b)(a * b)), sum, d1), d3, d1), rename(constant(\"dnn_hidden1_bias\"), d0, d1), f(a,b)(a + b)), f(a,b)(a * b)), join(rename(reduce(join(input, rename(constant(\"dnn_hidden1_weights\"), (d0, d1), (d1, d3)), f(a,b)(a * b)), sum, d1), d3, d1), rename(constant(\"dnn_hidden1_bias\"), d0, d1), f(a,b)(a + b)), f(a,b)(max(a,b))), rename(constant(\"dnn_hidden2_weights\"), (d0, d1), (d1, d3)), f(a,b)(a * b)), sum, d1), d3, d1), rename(constant(\"dnn_hidden2_bias\"), d0, d1), f(a,b)(a + b)), f(a)(1.050701 * if (a >= 0, a, 1.673263 * (exp(a) - 1)))), rename(constant(\"dnn_outputs_weights\"), (d0, d1), (d1, d3)), f(a,b)(a * b)), sum, d1), d3, d1), rename(constant(\"dnn_outputs_bias\"), d0, d1), f(a,b)(a + b))"; - StoringApplicationPackage application = new StoringApplicationPackage(applicationDir); - RankProfileSearchFixture search = fixtureWith("tensor(d0[2],d1[784])(0.0)", - "tensorflow('mnist/saved')", - null, - null, - "input", - application); - search.assertFirstPhaseExpression(expression, "my_profile"); - assertSmallConstant("dnn_hidden1_mul_x", TensorType.empty, search); - - // At this point the expression is stored - copy application to another location which do not have a models dir - Path storedApplicationDirectory = applicationDir.getParentPath().append("copy"); - try { - storedApplicationDirectory.toFile().mkdirs(); - IOUtils.copyDirectory(applicationDir.append(ApplicationPackage.MODELS_GENERATED_DIR).toFile(), - storedApplicationDirectory.append(ApplicationPackage.MODELS_GENERATED_DIR).toFile()); - StoringApplicationPackage storedApplication = new StoringApplicationPackage(storedApplicationDirectory); - RankProfileSearchFixture searchFromStored = fixtureWith("tensor(d0[2],d1[784])(0.0)", - "tensorflow('mnist/saved')", - null, - null, - "input", - storedApplication); - searchFromStored.assertFirstPhaseExpression(expression, "my_profile"); - assertSmallConstant("dnn_hidden1_mul_x", TensorType.empty, search); - } - finally { - IOUtils.recursiveDeleteDir(storedApplicationDirectory.toFile()); - } - } - - private void assertSmallConstant(String name, TensorType type, RankProfileSearchFixture search) { - Value value = search.rankProfile("my_profile").getConstants().get(name); - assertNotNull(value); - assertEquals(type, value.type()); - } - /** * Verifies that the constant with the given name exists, and - only if an expected size is given - * that the content of the constant is available and has the expected size. diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/TensorTransformTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/processing/TensorTransformTestCase.java index c18cfcfe1aa..b001db69768 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/processing/TensorTransformTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/processing/TensorTransformTestCase.java @@ -64,7 +64,7 @@ public class TensorTransformTestCase extends SearchDefinitionTestCase { assertContainsExpression("min(attribute(tensor_field_1) * attribute(tensor_field_2),x)", "reduce(attribute(tensor_field_1)*attribute(tensor_field_2),min,x)"); assertContainsExpression("min(join(attribute(tensor_field_1),attribute(tensor_field_2),f(x,y)(x*y)),x)", "reduce(join(attribute(tensor_field_1),attribute(tensor_field_2),f(x,y)(x*y)),min,x)"); assertContainsExpression("min(join(tensor_field_1,tensor_field_2,f(x,y)(x*y)),x)", "min(join(tensor_field_1,tensor_field_2,f(x,y)(x*y)),x)"); // because tensor fields are not in attribute(...) - assertContainsExpression("min(join(attribute(tensor_field_1),backend_rank_feature,f(x,y)(x*y)),x)", "min(join(attribute(tensor_field_1),backend_rank_feature,f(x,y)(x*y)),x)"); + assertContainsExpression("min(join(attribute(tensor_field_1),backend_rank_feature,f(x,y)(x*y)),x)", "reduce(join(attribute(tensor_field_1),backend_rank_feature,f(x,y)(x*y)),min,x)"); } @Test diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java index 4bf876df1a0..090f25171b1 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java @@ -208,6 +208,7 @@ public class ApplicationController { // RoutingEndpoints that lacks hostname is gone if (hostname != null) { + // Book-keeping if (endpoint.isGlobal()) { hostToGlobalEndpoint.put(hostname, endpoint); } else { @@ -518,12 +519,19 @@ public class ApplicationController { } } - /** Returns the endpoints of the deployment, or throws an exception if this fails. */ - public List<URI> getDeploymentEndpoints(DeploymentId deploymentId) { - return ImmutableList.copyOf(routingGenerator.endpoints(deploymentId).stream() - .map(RoutingEndpoint::getEndpoint) - .map(URI::create) - .iterator()); + /** Returns the endpoints of the deployment, or an empty list if the request fails */ + public Optional<List<URI>> getDeploymentEndpoints(DeploymentId deploymentId) { + try { + return Optional.of(ImmutableList.copyOf(routingGenerator.endpoints(deploymentId).stream() + .map(RoutingEndpoint::getEndpoint) + .map(URI::create) + .iterator())); + } + catch (RuntimeException e) { + log.log(Level.WARNING, "Failed to get endpoint information for " + deploymentId + ": " + + Exceptions.toMessageString(e)); + return Optional.empty(); + } } /** diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java index 58c37e493b1..2105ea1d3d9 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java @@ -441,7 +441,7 @@ public class ApplicationApiHandler extends LoggingRequestHandler { Cursor serviceUrlArray = response.setArray("serviceUrls"); controller.applications().getDeploymentEndpoints(deploymentId) - .forEach(endpoint -> serviceUrlArray.addString(endpoint.toString())); + .ifPresent(endpoints -> endpoints.forEach(endpoint -> serviceUrlArray.addString(endpoint.toString()))); response.setString("nodes", withPath("/zone/v2/" + deploymentId.zoneId().environment() + "/" + deploymentId.zoneId().region() + "/nodes/v2/node/?&recursive=true&application=" + deploymentId.applicationId().tenant() + "." + deploymentId.applicationId().application() + "." + deploymentId.applicationId().instance(), request.getUri()).toString()); diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/routing/MockRoutingGenerator.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/routing/MockRoutingGenerator.java index 89e3219b17f..bc2da7287e1 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/routing/MockRoutingGenerator.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/routing/MockRoutingGenerator.java @@ -16,6 +16,7 @@ public class MockRoutingGenerator implements RoutingGenerator { @Override public List<RoutingEndpoint> endpoints(DeploymentId deployment) { List<RoutingEndpoint> endpoints = new ArrayList<>(); + // TODO: TLS: Update to HTTPS when ready. endpoints.add(new RoutingEndpoint("http://old-endpoint.vespa.yahooapis.com:4080", false)); endpoints.add(new RoutingEndpoint("http://qrs-endpoint.vespa.yahooapis.com:4080", "host1", false)); endpoints.add(new RoutingEndpoint("http://feeding-endpoint.vespa.yahooapis.com:4080", "host2", false)); diff --git a/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/FileReceiver.java b/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/FileReceiver.java index 3d6c7bf0a5c..a580759b03b 100644 --- a/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/FileReceiver.java +++ b/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/FileReceiver.java @@ -129,6 +129,14 @@ public class FileReceiver { } catch (IOException e) { log.log(LogLevel.ERROR, "Failed writing file: " + e.getMessage(), e); throw new RuntimeException("Failed writing file: ", e); + } finally { + try { + if (inprogressFile.exists()) { + Files.delete(inprogressFile.toPath()); + } + } catch (IOException e) { + log.log(LogLevel.ERROR, "Failed deleting " + inprogressFile.getAbsolutePath() + ": " + e.getMessage(), e); + } } return file; } diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/component/TaskContext.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/component/TaskContext.java index 2d94db3d96e..d0a5570b8dc 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/component/TaskContext.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/component/TaskContext.java @@ -1,6 +1,7 @@ // Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.node.admin.component; +import java.util.function.Supplier; import java.util.logging.Logger; public interface TaskContext { @@ -16,10 +17,39 @@ public interface TaskContext { * performed (sometimes this is not possible). * * @param logger Used to log the modification to help locate the source of the modification. - * @param description Description of the modification, e.g. "Changing owner of /foo from alice - * to bob". + * @param message Description of the modification, e.g. "Changing owner of /foo from alice + * to bob". */ - void recordSystemModification(Logger logger, String description); + void recordSystemModification(Logger logger, String message); + default void recordSystemModification(Logger logger, String messageFormat, String... args) { + recordSystemModification(logger, String.format(messageFormat, (Object[]) args)); + } + + /** + * Log message at LogLevel.INFO, scoped to denote the current task. The message may + * also be directed to status pages or similar. + * + * Please do not call this too many times as that spams the log. Typically a task may call + * this zero times, or up to a few times. + * + * Do not log a message that is also recorded with recordSystemModification. + */ + default void log(Logger logger, String message) {} + default void log(Logger logger, String messageFormat, String... args) { + log(logger, String.format(messageFormat, (Object[]) args)); + } + + /** + * Register a message supplier to be called if the task failed, to help with debugging + * of the task (and too verbose to log at every run). The message is also logged at + * LogLevel.DEBUG if enabled. + * + * Do not call logOnFailure for a message passed to either recordSystemModification or log. + * + * @param messageSupplier Supplier to be called possibly immediately (if DEBUG is enabled), + * or later if the task failed. Either way, it will only be called once. + */ + default void logOnFailure(Logger logger, Supplier<String> messageSupplier) {} /** * Execute a task as a child of this task, and with its own sub-TaskContext. Please avoid diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/component/TestTaskContext.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/component/TestTaskContext.java index 6806e5096c5..108d42114f7 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/component/TestTaskContext.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/component/TestTaskContext.java @@ -4,22 +4,29 @@ package com.yahoo.vespa.hosted.node.admin.component; import java.util.ArrayList; import java.util.List; +import java.util.function.Supplier; import java.util.logging.Logger; public class TestTaskContext implements TaskContext { - private final List<String> logs = new ArrayList<>(); + private final List<String> systemModifications = new ArrayList<>(); @Override public void recordSystemModification(Logger logger, String description) { - logs.add(description); + systemModifications.add(description); } + @Override + public void log(Logger logger, String message) { } + + @Override + public void logOnFailure(Logger logger, Supplier<String> messageSupplier) { } + public List<String> getSystemModificationLog() { - return logs; + return systemModifications; } public void clearSystemModificationLog() { - logs.clear(); + systemModifications.clear(); } @Override diff --git a/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/evaluation/Value.java b/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/evaluation/Value.java index 59d2d95b879..e5a9e6a5ef1 100644 --- a/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/evaluation/Value.java +++ b/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/evaluation/Value.java @@ -33,7 +33,7 @@ public abstract class Value { /** Returns this as a tensor value */ public abstract Tensor asTensor(); - /** A utility method for wrapping a sdouble in a rank 0 tensor */ + /** A utility method for wrapping a double in a rank 0 tensor */ protected Tensor doubleAsTensor(double value) { return Tensor.Builder.of(TensorType.empty).cell(TensorAddress.of(), value).build(); } diff --git a/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/integration/tensorflow/OperationMapper.java b/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/integration/tensorflow/OperationMapper.java index 55782c36d18..ef82045e771 100644 --- a/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/integration/tensorflow/OperationMapper.java +++ b/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/integration/tensorflow/OperationMapper.java @@ -156,6 +156,12 @@ class OperationMapper { private static Optional<TypedTensorFunction> constant(TensorFlowImporter.Parameters params) { Tensor value = AttrValueConverter.toVespaTensor(params.node(), "value"); + if (value.type().rank() == 0) { + TypedTensorFunction output = new TypedTensorFunction(value.type(), + new TensorFunctionNode.TensorFunctionExpressionNode( + new ConstantNode(new DoubleValue(value.asDouble())))); + return Optional.of(output); + } return createConstant(params, value); } diff --git a/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/rule/TensorFunctionNode.java b/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/rule/TensorFunctionNode.java index e4c381972e9..ec6af4bb413 100644 --- a/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/rule/TensorFunctionNode.java +++ b/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/rule/TensorFunctionNode.java @@ -117,11 +117,7 @@ public class TensorFunctionNode extends CompositeNode { @Override public Tensor evaluate(EvaluationContext context) { - Value result = expression.evaluate((Context)context); - if ( ! ( result instanceof TensorValue)) - throw new IllegalArgumentException("Attempted to evaluate tensor function '" + expression + "', " + - "but this returns " + result + ", not a tensor"); - return result.asTensor(); + return expression.evaluate((Context)context).asTensor(); } @Override diff --git a/searchlib/src/test/java/com/yahoo/searchlib/rankingexpression/evaluation/EvaluationTestCase.java b/searchlib/src/test/java/com/yahoo/searchlib/rankingexpression/evaluation/EvaluationTestCase.java index 6c7643b37b3..e9030cf5852 100644 --- a/searchlib/src/test/java/com/yahoo/searchlib/rankingexpression/evaluation/EvaluationTestCase.java +++ b/searchlib/src/test/java/com/yahoo/searchlib/rankingexpression/evaluation/EvaluationTestCase.java @@ -294,6 +294,7 @@ public class EvaluationTestCase { "tensor0 != tensor1", "{ {x:0}:3, {x:1}:7 }", "{ {y:0}:7 }"); tester.assertEvaluates("{ {x:0}:1, {x:1}:0 }", "tensor0 in [1,2,3]", "{ {x:0}:3, {x:1}:7 }"); + tester.assertEvaluates("{ {x:0}:0.1 }", "join(tensor0, 0.1, f(x,y) (x*y))", "{ {x:0}:1 }"); // TODO // argmax diff --git a/searchlib/src/vespa/searchlib/transactionlog/translogserverapp.cpp b/searchlib/src/vespa/searchlib/transactionlog/translogserverapp.cpp index ff4a402b438..3c964c2a04d 100644 --- a/searchlib/src/vespa/searchlib/transactionlog/translogserverapp.cpp +++ b/searchlib/src/vespa/searchlib/transactionlog/translogserverapp.cpp @@ -12,7 +12,8 @@ namespace search::transactionlog { TransLogServerApp::TransLogServerApp(const config::ConfigUri & tlsConfigUri, const FileHeaderContext & fileHeaderContext) - : _tls(), + : _lock(), + _tls(), _tlsConfig(), _tlsConfigFetcher(tlsConfigUri.getContext()), _fileHeaderContext(fileHeaderContext) @@ -23,7 +24,8 @@ TransLogServerApp::TransLogServerApp(const config::ConfigUri & tlsConfigUri, namespace { -DomainPart::Crc getCrc(searchlib::TranslogserverConfig::Crcmethod crcType) +DomainPart::Crc +getCrc(searchlib::TranslogserverConfig::Crcmethod crcType) { switch (crcType) { case searchlib::TranslogserverConfig::ccitt_crc32: @@ -36,11 +38,14 @@ DomainPart::Crc getCrc(searchlib::TranslogserverConfig::Crcmethod crcType) } -void TransLogServerApp::start() +void +TransLogServerApp::start() { std::shared_ptr<searchlib::TranslogserverConfig> c = _tlsConfig.get(); - _tls.reset(new TransLogServer(c->servername, c->listenport, c->basedir, _fileHeaderContext, - c->filesizemax, c->maxthreads, getCrc(c->crcmethod))); + auto tls = std::make_shared<TransLogServer>(c->servername, c->listenport, c->basedir, _fileHeaderContext, + c->filesizemax, c->maxthreads, getCrc(c->crcmethod)); + std::lock_guard<std::mutex> guard(_lock); + _tls = std::move(tls); } TransLogServerApp::~TransLogServerApp() @@ -48,11 +53,18 @@ TransLogServerApp::~TransLogServerApp() _tlsConfigFetcher.close(); } -void TransLogServerApp::configure(std::unique_ptr<searchlib::TranslogserverConfig> cfg) +void +TransLogServerApp::configure(std::unique_ptr<searchlib::TranslogserverConfig> cfg) { LOG(config, "configure Transaction Log Server %s at port %d", cfg->servername.c_str(), cfg->listenport); _tlsConfig.set(cfg.release()); _tlsConfig.latch(); } +TransLogServer::SP +TransLogServerApp::getTransLogServer() const { + std::lock_guard<std::mutex> guard(_lock); + return _tls; +} + } diff --git a/searchlib/src/vespa/searchlib/transactionlog/translogserverapp.h b/searchlib/src/vespa/searchlib/transactionlog/translogserverapp.h index 35fa994d1e4..d46805c105c 100644 --- a/searchlib/src/vespa/searchlib/transactionlog/translogserverapp.h +++ b/searchlib/src/vespa/searchlib/transactionlog/translogserverapp.h @@ -14,6 +14,7 @@ namespace search::transactionlog { class TransLogServerApp : public config::IFetcherCallback<searchlib::TranslogserverConfig> { private: + mutable std::mutex _lock; TransLogServer::SP _tls; vespalib::PtrHolder<searchlib::TranslogserverConfig> _tlsConfig; config::ConfigFetcher _tlsConfigFetcher; @@ -28,7 +29,7 @@ public: const common::FileHeaderContext &fileHeaderContext); ~TransLogServerApp(); - TransLogServer::SP getTransLogServer() const { return _tls; } + TransLogServer::SP getTransLogServer() const; void start(); }; diff --git a/vespabase/src/rhel-prestart.sh b/vespabase/src/rhel-prestart.sh index fa7b9a1fe49..ebe9fe16938 100755 --- a/vespabase/src/rhel-prestart.sh +++ b/vespabase/src/rhel-prestart.sh @@ -94,6 +94,7 @@ fixdir ${VESPA_USER} wheel 755 var/db/vespa/tmp fixdir ${VESPA_USER} wheel 755 var/db/vespa/config_server fixdir ${VESPA_USER} wheel 755 var/db/vespa/config_server/serverdb fixdir ${VESPA_USER} wheel 755 var/db/vespa/config_server/serverdb/tenants +fixdir ${VESPA_USER} wheel 755 var/db/vespa/filedistribution fixdir ${VESPA_USER} wheel 755 var/db/vespa/index fixdir ${VESPA_USER} wheel 755 var/db/vespa/logcontrol fixdir ${VESPA_USER} wheel 755 var/db/vespa/search |