- Split rank setup verification code and the binary.

- Accumulate errors during ranksetup. Report them at the end, and also verify them in some tests.

12 files changed, 278 insertions, 146 deletions

diff --git a/searchcore/src/apps/verify_ranksetup/CMakeLists.txt b/searchcore/src/apps/verify_ranksetup/CMakeLists.txt
index cc8434ef46f..536392739d9 100644
--- a/searchcore/src/apps/verify_ranksetup/CMakeLists.txt
+++ b/searchcore/src/apps/verify_ranksetup/CMakeLists.txt
@@ -1,12 +1,20 @@
 # Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-vespa_add_executable(searchcore_verify_ranksetup_app
+vespa_add_library(searchcore_verify_ranksetup
     SOURCES
     verify_ranksetup.cpp
-    OUTPUT_NAME vespa-verify-ranksetup-bin
-    INSTALL bin
+    INSTALL lib64
     DEPENDS
     searchcore_fconfig
     searchcore_matching
     searchcore_documentmetastore
 )
-vespa_generate_config(searchcore_verify_ranksetup_app verify-ranksetup.def)
+vespa_generate_config(searchcore_verify_ranksetup verify-ranksetup.def)
+
+vespa_add_executable(searchcore_verify_ranksetup_app
+    SOURCES
+    verify_ranksetup_app.cpp
+    OUTPUT_NAME vespa-verify-ranksetup-bin
+    INSTALL bin
+    DEPENDS
+    searchcore_verify_ranksetup
+)
diff --git a/searchcore/src/apps/verify_ranksetup/verify_ranksetup.cpp b/searchcore/src/apps/verify_ranksetup/verify_ranksetup.cpp
index e72771f5aa6..72076932dd3 100644
--- a/searchcore/src/apps/verify_ranksetup/verify_ranksetup.cpp
+++ b/searchcore/src/apps/verify_ranksetup/verify_ranksetup.cpp
@@ -21,11 +21,12 @@
 #include <vespa/searchlib/fef/fef.h>
 #include <vespa/searchlib/fef/test/plugin/setup.h>
 #include <vespa/config/subscription/configsubscriber.hpp>
-#include <vespa/vespalib/util/signalhandler.h>
+#include <vespa/vespalib/util/stringfmt.h>
+#include <vespa/vespalib/stllike/asciistream.h>
 #include <optional>
 
 #include <vespa/log/log.h>
-LOG_SETUP("vespa-verify-ranksetup");
+LOG_SETUP(".verify-ranksetup");
 
 using config::ConfigContext;
 using config::ConfigHandle;
@@ -50,8 +51,10 @@ using vespalib::eval::SimpleConstantValue;
 using vespalib::eval::TensorSpec;
 using vespalib::eval::Value;
 using vespalib::eval::ValueType;
+using vespalib::make_string_short::fmt;
 
-std::optional<vespalib::string> get_file(const vespalib::string &ref, const VerifyRanksetupConfig &myCfg) {
+std::optional<vespalib::string>
+get_file(const vespalib::string &ref, const VerifyRanksetupConfig &myCfg) {
     for (const auto &entry: myCfg.file) {
         if (ref == entry.ref) {
             return entry.path;
@@ -60,7 +63,8 @@ std::optional<vespalib::string> get_file(const vespalib::string &ref, const Veri
     return std::nullopt;
 }
 
-RankingExpressions make_expressions(const RankingExpressionsConfig &expressionsCfg, const VerifyRanksetupConfig &myCfg) {
+RankingExpressions
+make_expressions(const RankingExpressionsConfig &expressionsCfg, const VerifyRanksetupConfig &myCfg) {
     RankingExpressions expressions;
     for (const auto &entry: expressionsCfg.expression) {
         if (auto file = get_file(entry.fileref, myCfg)) {
@@ -74,7 +78,8 @@ RankingExpressions make_expressions(const RankingExpressionsConfig &expressionsC
     return expressions;
 }
 
-OnnxModels make_models(const OnnxModelsConfig &modelsCfg, const VerifyRanksetupConfig &myCfg) {
+OnnxModels
+make_models(const OnnxModelsConfig &modelsCfg, const VerifyRanksetupConfig &myCfg) {
     OnnxModels::Vector model_list;
     for (const auto &entry: modelsCfg.model) {
         if (auto file = get_file(entry.fileref, myCfg)) {
@@ -88,9 +93,10 @@ OnnxModels make_models(const OnnxModelsConfig &modelsCfg, const VerifyRanksetupC
     return OnnxModels(model_list);
 }
 
-class App
+class VerifyRankSetup
 {
-public:
+private:
+    std::vector<vespalib::string> _errors;
     bool verify(const search::index::Schema &schema,
                 const search::fef::Properties &props,
                 const IConstantValueRepo &repo,
@@ -105,30 +111,42 @@ public:
                       const RankingExpressionsConfig &expressionsCfg,
                       const OnnxModelsConfig &modelsCfg);
 
-    int usage();
-    int main(int argc, char **argv);
+public:
+    VerifyRankSetup();
+    ~VerifyRankSetup();
+    const std::vector<vespalib::string> & getMessages() const { return _errors; }
+    bool verify(const std::string & configId);
 };
 
 struct DummyConstantValueRepo : IConstantValueRepo {
     const RankingConstantsConfig &cfg;
     DummyConstantValueRepo(const RankingConstantsConfig &cfg_in) : cfg(cfg_in) {}
-    virtual vespalib::eval::ConstantValue::UP getConstant(const vespalib::string &name) const override {
-        for (const auto &entry: cfg.constant) {
-            if (entry.name == name) {
-                try {
-                    auto tensor = vespalib::eval::value_from_spec(TensorSpec(entry.type), FastValueBuilderFactory::get());
-                    return std::make_unique<SimpleConstantValue>(std::move(tensor));
-                } catch (std::exception &) {
-                    return std::make_unique<BadConstantValue>();
-                }
+    vespalib::eval::ConstantValue::UP getConstant(const vespalib::string &name) const override;
+};
+
+vespalib::eval::ConstantValue::UP
+DummyConstantValueRepo::getConstant(const vespalib::string &name) const {
+    for (const auto &entry: cfg.constant) {
+        if (entry.name == name) {
+            try {
+                auto tensor = vespalib::eval::value_from_spec(TensorSpec(entry.type), FastValueBuilderFactory::get());
+                return std::make_unique<SimpleConstantValue>(std::move(tensor));
+            } catch (std::exception &) {
+                return std::make_unique<BadConstantValue>();
             }
         }
-        return vespalib::eval::ConstantValue::UP(nullptr);
     }
-};
+    return vespalib::eval::ConstantValue::UP(nullptr);
+}
+
+VerifyRankSetup::VerifyRankSetup()
+    : _errors()
+{ }
+
+VerifyRankSetup::~VerifyRankSetup() = default;
 
 bool
-App::verify(const search::index::Schema &schema,
+VerifyRankSetup::verify(const search::index::Schema &schema,
             const search::fef::Properties &props,
             const IConstantValueRepo &repo,
             const RankingExpressions &expressions,
@@ -143,25 +161,25 @@ App::verify(const search::index::Schema &schema,
     rankSetup.configure(); // reads config values from the property map
     bool ok = true;
     if (!rankSetup.getFirstPhaseRank().empty()) {
-        ok = verifyFeature(factory, indexEnv, rankSetup.getFirstPhaseRank(), "first phase ranking") && ok;
+        ok = verifyFeature(factory, indexEnv, rankSetup.getFirstPhaseRank(), "first phase ranking", _errors) && ok;
     }
     if (!rankSetup.getSecondPhaseRank().empty()) {
-        ok = verifyFeature(factory, indexEnv, rankSetup.getSecondPhaseRank(), "second phase ranking") && ok;
+        ok = verifyFeature(factory, indexEnv, rankSetup.getSecondPhaseRank(), "second phase ranking", _errors) && ok;
     }
     for (size_t i = 0; i < rankSetup.getSummaryFeatures().size(); ++i) {
-        ok = verifyFeature(factory, indexEnv, rankSetup.getSummaryFeatures()[i], "summary features") && ok;
+        ok = verifyFeature(factory, indexEnv, rankSetup.getSummaryFeatures()[i], "summary features", _errors) && ok;
     }
     for (const auto & feature : rankSetup.get_match_features()) {
-        ok = verifyFeature(factory, indexEnv, feature, "match features") && ok;
+        ok = verifyFeature(factory, indexEnv, feature, "match features", _errors) && ok;
     }
     for (size_t i = 0; i < rankSetup.getDumpFeatures().size(); ++i) {
-        ok = verifyFeature(factory, indexEnv, rankSetup.getDumpFeatures()[i], "dump features") && ok;
+        ok = verifyFeature(factory, indexEnv, rankSetup.getDumpFeatures()[i], "dump features", _errors) && ok;
     }
     return ok;
 }
 
 bool
-App::verifyConfig(const VerifyRanksetupConfig &myCfg,
+VerifyRankSetup::verifyConfig(const VerifyRanksetupConfig &myCfg,
                   const RankProfilesConfig &rankCfg,
                   const IndexschemaConfig &schemaCfg,
                   const AttributesConfig &attributeCfg,
@@ -183,31 +201,22 @@ App::verifyConfig(const VerifyRanksetupConfig &myCfg,
             properties.add(profile.fef.property[j].name,
                            profile.fef.property[j].value);
         }
+        vespalib::string msg;
         if (verify(schema, properties, repo, expressions, models)) {
-            LOG(info, "rank profile '%s': pass", profile.name.c_str());
+            msg = fmt("rank profile '%s': pass", profile.name.c_str());
+            LOG(info, "%s", msg.c_str());
         } else {
-            LOG(error, "rank profile '%s': FAIL", profile.name.c_str());
+            LOG(error, "%s", msg.c_str());
             ok = false;
         }
+        _errors.emplace_back(msg);
     }
     return ok;
 }
 
-int
-App::usage()
-{
-    fprintf(stderr, "Usage: vespa-verify-ranksetup <config-id>\n");
-    return 1;
-}
-
-int
-App::main(int argc, char **argv)
+bool
+VerifyRankSetup::verify(const std::string & configid)
 {
-    if (argc != 2) {
-        return usage();
-    }
-
-    const std::string configid = argv[1];
     LOG(debug, "verifying rank setup for config id '%s' ...",
         configid.c_str());
 
@@ -233,18 +242,25 @@ App::main(int argc, char **argv)
                           *expressionsHandle->getConfig(),
                           *modelsHandle->getConfig());
     } catch (ConfigRuntimeException & e) {
-        LOG(error, "Unable to subscribe to config: %s", e.getMessage().c_str());
+        vespalib::string msg = fmt("Unable to subscribe to config: %s", e.getMessage().c_str());
+        LOG(error, "%s", msg.c_str());
+        _errors.emplace_back(msg);
     } catch (InvalidConfigException & e) {
-        LOG(error, "Error getting config: %s", e.getMessage().c_str());
-    }
-    if (!ok) {
-        return 1;
+        vespalib::string msg = fmt("Error getting config: %s", e.getMessage().c_str());
+        LOG(error, "%s", msg.c_str());
+        _errors.emplace_back(msg);
     }
-    return 0;
+    return ok;
 }
 
-int main(int argc, char **argv) {
-    vespalib::SignalHandler::PIPE.ignore();
-    App app;
-    return app.main(argc, argv);
+bool
+verifyRankSetup(const char * configId, std::string & messages) {
+    VerifyRankSetup verifier;
+    bool ok = verifier.verify(configId);
+    vespalib::asciistream os;
+    for (const auto & m : verifier.getMessages()) {
+        os << m << "\n";
+    }
+    messages = os.str();
+    return ok;
 }
diff --git a/searchcore/src/apps/verify_ranksetup/verify_ranksetup.h b/searchcore/src/apps/verify_ranksetup/verify_ranksetup.h
new file mode 100644
index 00000000000..981a40b9804
--- /dev/null
+++ b/searchcore/src/apps/verify_ranksetup/verify_ranksetup.h
@@ -0,0 +1,7 @@
+// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#pragma once
+
+#include <string>
+
+bool verifyRankSetup(const char * configId, std::string & messages);
diff --git a/searchcore/src/apps/verify_ranksetup/verify_ranksetup_app.cpp b/searchcore/src/apps/verify_ranksetup/verify_ranksetup_app.cpp
new file mode 100644
index 00000000000..5dff78b17a1
--- /dev/null
+++ b/searchcore/src/apps/verify_ranksetup/verify_ranksetup_app.cpp
@@ -0,0 +1,46 @@
+// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#include "verify_ranksetup.h"
+#include <vespa/vespalib/util/signalhandler.h>
+
+#include <vespa/log/log.h>
+LOG_SETUP("vespa-verify-ranksetup");
+
+class App
+{
+public:
+    int usage();
+    int main(int argc, char **argv);
+};
+
+int
+App::usage()
+{
+    fprintf(stderr, "Usage: vespa-verify-ranksetup <config-id>\n");
+    return 1;
+}
+
+int
+App::main(int argc, char **argv)
+{
+    if (argc != 2) {
+        return usage();
+    }
+
+    std::string messages;
+    bool ok = verifyRankSetup(argv[1], messages);
+
+    if ( ! messages.empty() ) {
+        LOG(info, "%s", messages.c_str());
+    }
+    if (!ok) {
+        return 1;
+    }
+    return 0;
+}
+
+int main(int argc, char **argv) {
+    vespalib::SignalHandler::PIPE.ignore();
+    App app;
+    return app.main(argc, argv);
+}
diff --git a/searchlib/src/tests/fef/object_passing/object_passing_test.cpp b/searchlib/src/tests/fef/object_passing/object_passing_test.cpp
index 3639da05b9e..5b16b798754 100644
--- a/searchlib/src/tests/fef/object_passing/object_passing_test.cpp
+++ b/searchlib/src/tests/fef/object_passing/object_passing_test.cpp
@@ -102,7 +102,8 @@ struct Fixture {
     }
 
     bool verify(const vespalib::string &feature) {
-        return verifyFeature(factory, indexEnv, feature, "unit test");
+        std::vector<vespalib::string> errors;
+        return verifyFeature(factory, indexEnv, feature, "unit test", errors);
     }
 };
 
diff --git a/searchlib/src/tests/ranksetup/verify_feature/verify_feature_test.cpp b/searchlib/src/tests/ranksetup/verify_feature/verify_feature_test.cpp
index 24358db8f3a..9b6669a86f7 100644
--- a/searchlib/src/tests/ranksetup/verify_feature/verify_feature_test.cpp
+++ b/searchlib/src/tests/ranksetup/verify_feature/verify_feature_test.cpp
@@ -4,10 +4,13 @@
 #include <vespa/searchlib/fef/test/indexenvironment.h>
 #include <vespa/searchlib/fef/test/plugin/setup.h>
 #include <vespa/searchlib/features/valuefeature.h>
+#include <vespa/vespalib/util/stringfmt.h>
+#include <vespa/vespalib/stllike/asciistream.h>
 
 using namespace search::features;
 using namespace search::fef::test;
 using namespace search::fef;
+using vespalib::make_string_short::fmt;
 
 struct RankFixture {
     BlueprintFactory factory;
@@ -15,43 +18,83 @@ struct RankFixture {
 
     RankFixture() : factory(), indexEnv() {
         setup_fef_test_plugin(factory);
-        factory.addPrototype(Blueprint::SP(new ValueBlueprint()));
+        factory.addPrototype(std::make_shared<ValueBlueprint>());
     }
 
-    bool verify(const std::string &feature) const {
-        return verifyFeature(factory, indexEnv, feature, "feature verification test");
+    bool verify(const std::string &feature, std::vector<vespalib::string> expected_errors) const {
+        std::vector<vespalib::string> errors;
+        bool ok = verifyFeature(factory, indexEnv, feature, "feature verification test", errors);
+        EXPECT_EQUAL(errors.size(), expected_errors.size());
+        for (size_t i(0); i < std::min(errors.size(), expected_errors.size()); i++) {
+            EXPECT_EQUAL(errors[i].size(), expected_errors[i].size());
+            EXPECT_EQUAL(errors[i], expected_errors[i]);
+        }
+        return ok;
     }
 };
 
 TEST_F("verify valid rank feature", RankFixture) {
-    EXPECT_TRUE(f1.verify("value(1, 2, 3).0"));
-    EXPECT_TRUE(f1.verify("value(1, 2, 3).1"));
-    EXPECT_TRUE(f1.verify("value(1, 2, 3).2"));
+    EXPECT_TRUE(f1.verify("value(1, 2, 3).0", {}));
+    EXPECT_TRUE(f1.verify("value(1, 2, 3).1", {}));
+    EXPECT_TRUE(f1.verify("value(1, 2, 3).2", {}));
 }
 
 TEST_F("verify unknown feature", RankFixture) {
-    EXPECT_FALSE(f1.verify("unknown"));
+    EXPECT_FALSE(f1.verify("unknown",
+                           {"invalid rank feature 'unknown': unknown basename: 'unknown'",
+                            "verification failed: rank feature 'unknown' (feature verification test)"}));
 }
 
 TEST_F("verify unknown output", RankFixture) {
-    EXPECT_FALSE(f1.verify("value(1, 2, 3).3"));
+    EXPECT_FALSE(f1.verify("value(1, 2, 3).3",
+                           {"invalid rank feature 'value(1,2,3).3': unknown output: '3'",
+                            "verification failed: rank feature 'value(1, 2, 3).3' (feature verification test)"}));
 }
 
 TEST_F("verify illegal input parameters", RankFixture) {
-    EXPECT_FALSE(f1.verify("value.0"));
+    EXPECT_FALSE(f1.verify("value.0",
+                           {"invalid rank feature 'value.0': The parameter list used for setting up rank feature value is not valid: Expected 1+1x parameter(s), but got 0",
+                            "verification failed: rank feature 'value.0' (feature verification test)"}));
 }
 
 TEST_F("verify illegal feature name", RankFixture) {
-    EXPECT_FALSE(f1.verify("value(2)."));
+    EXPECT_FALSE(f1.verify("value(2).",
+                           {"invalid rank feature 'value(2).': malformed name",
+                            "verification failed: rank feature 'value(2).' (feature verification test)"}));
 }
 
 TEST_F("verify too deep dependency graph", RankFixture) {
-    EXPECT_TRUE(f1.verify("chain(basic, 255, 4)"));
-    EXPECT_FALSE(f1.verify("chain(basic, 256, 4)"));
+    EXPECT_TRUE(f1.verify("chain(basic, 255, 4)", {}));
+    EXPECT_FALSE(f1.verify("chain(basic, 256, 4)",
+                           {"invalid rank feature 'value(4)': dependency graph too deep\n"
+                            "  ... needed by rank feature 'chain(basic,1,4)'\n"
+                            "  ... needed by rank feature 'chain(basic,2,4)'\n"
+                            "  ... needed by rank feature 'chain(basic,3,4)'\n"
+                            "  ... needed by rank feature 'chain(basic,4,4)'\n"
+                            "  ... needed by rank feature 'chain(basic,5,4)'\n"
+                            "  ... needed by rank feature 'chain(basic,6,4)'\n"
+                            "  ... needed by rank feature 'chain(basic,7,4)'\n"
+                            "  ... needed by rank feature 'chain(basic,8,4)'\n"
+                            "  ... needed by rank feature 'chain(basic,9,4)'\n"
+                            "  ... needed by rank feature 'chain(basic,10,4)'\n"
+                            "  (skipped 241 entries)\n"
+                            "  ... needed by rank feature 'chain(basic,252,4)'\n"
+                            "  ... needed by rank feature 'chain(basic,253,4)'\n"
+                            "  ... needed by rank feature 'chain(basic,254,4)'\n"
+                            "  ... needed by rank feature 'chain(basic,255,4)'\n"
+                            "  ... needed by rank feature 'chain(basic,256,4)'\n",
+                            "high stack usage: 360848 bytes",
+                            "verification failed: rank feature 'chain(basic, 256, 4)' (feature verification test)"}));
 }
 
 TEST_F("verify dependency cycle", RankFixture) {
-    EXPECT_FALSE(f1.verify("chain(cycle, 4, 2)"));
+    EXPECT_FALSE(f1.verify("chain(cycle, 4, 2)",
+                           {"invalid rank feature 'chain(cycle,2,2)': dependency cycle detected\n"
+                            "  ... needed by rank feature 'chain(cycle,1,2)'\n"
+                            "  ... needed by rank feature 'chain(cycle,2,2)'\n"
+                            "  ... needed by rank feature 'chain(cycle,3,2)'\n"
+                            "  ... needed by rank feature 'chain(cycle,4,2)'\n",
+                            "verification failed: rank feature 'chain(cycle, 4, 2)' (feature verification test)"}));
 }
 
 TEST_MAIN() { TEST_RUN_ALL(); }
diff --git a/searchlib/src/vespa/searchlib/fef/blueprintresolver.cpp b/searchlib/src/vespa/searchlib/fef/blueprintresolver.cpp
index 731306d1bea..e244c15bc25 100644
--- a/searchlib/src/vespa/searchlib/fef/blueprintresolver.cpp
+++ b/searchlib/src/vespa/searchlib/fef/blueprintresolver.cpp
@@ -7,14 +7,10 @@
 #include <vespa/vespalib/util/stringfmt.h>
 #include <vespa/vespalib/util/size_literals.h>
 #include <vespa/vespalib/util/threadstackexecutor.h>
-#include <stack>
 #include <cassert>
 #include <set>
 #include <thread>
 
-#include <vespa/log/log.h>
-LOG_SETUP(".fef.blueprintresolver");
-
 using vespalib::make_string_short::fmt;
 using vespalib::ThreadStackExecutor;
 using vespalib::makeLambdaTask;
@@ -63,6 +59,7 @@ struct Compiler : public Blueprint::DependencyHandler {
             : spec(std::move(blueprint)), parser(parser_in) {}
     };
     using Stack = std::vector<Frame>;
+    using Errors = std::vector<vespalib::string>;
 
     struct FrameGuard {
         Stack &stack;
@@ -73,15 +70,16 @@ struct Compiler : public Blueprint::DependencyHandler {
         }
     };
 
-    const BlueprintFactory  &factory;
-    const IIndexEnvironment &index_env;
-    Stack                    resolve_stack;
-    ExecutorSpecList        &spec_list;
-    FeatureMap              &feature_map;
-    std::set<vespalib::string> setup_set;
-    std::set<vespalib::string> failed_set;
-    const char *min_stack;
-    const char *max_stack;
+    const BlueprintFactory     &factory;
+    const IIndexEnvironment    &index_env;
+    Stack                       resolve_stack;
+    Errors                      errors;
+    ExecutorSpecList           &spec_list;
+    FeatureMap                 &feature_map;
+    std::set<vespalib::string>  setup_set;
+    std::set<vespalib::string>  failed_set;
+    const char                 *min_stack;
+    const char                 *max_stack;
 
     Compiler(const BlueprintFactory &factory_in,
              const IIndexEnvironment &index_env_in,
@@ -90,6 +88,7 @@ struct Compiler : public Blueprint::DependencyHandler {
         : factory(factory_in),
           index_env(index_env_in),
           resolve_stack(),
+          errors(),
           spec_list(spec_list_out),
           feature_map(feature_map_out),
           setup_set(),
@@ -138,11 +137,13 @@ struct Compiler : public Blueprint::DependencyHandler {
         if (failed_set.count(feature_name) == 0) {
             failed_set.insert(feature_name);
             auto trace = make_trace(skip_self);
+            vespalib::string msg;
             if (trace.empty()) {
-                LOG(warning, "invalid %s: %s", describe(feature_name).c_str(), reason.c_str());
+                msg = fmt("invalid %s: %s", describe(feature_name).c_str(), reason.c_str());
             } else {
-                LOG(warning, "invalid %s: %s\n%s", describe(feature_name).c_str(), reason.c_str(), trace.c_str());
+                msg = fmt("invalid %s: %s\n%s", describe(feature_name).c_str(), reason.c_str(), trace.c_str());
             }
+            errors.emplace_back(msg);
         }
         probe_stack();
         return FeatureRef();
@@ -264,7 +265,8 @@ BlueprintResolver::BlueprintResolver(const BlueprintFactory &factory,
       _seeds(),
       _executorSpecs(),
       _featureMap(),
-      _seedMap()
+      _seedMap(),
+      _compileErrors()
 {
 }
 
@@ -300,6 +302,7 @@ BlueprintResolver::compile()
                                    for (const auto &seed: _seeds) {
                                        auto ref = compiler.resolve_feature(seed, Blueprint::AcceptInput::ANY);
                                        if (compiler.failed()) {
+                                           _compileErrors = std::move(compiler.errors);
                                            return;
                                        }
                                        _seedMap.emplace(FeatureNameParser(seed).featureName(), ref);
@@ -311,27 +314,9 @@ BlueprintResolver::compile()
     executor.shutdown();
     size_t stack_usage = compiler.stack_usage();
     if (stack_usage > (128_Ki)) {
-        LOG(warning, "high stack usage: %zu bytes", stack_usage);
+        _compileErrors.emplace_back(fmt("high stack usage: %zu bytes", stack_usage));
     }
     return !compiler.failed();
 }
 
-const BlueprintResolver::ExecutorSpecList &
-BlueprintResolver::getExecutorSpecs() const
-{
-    return _executorSpecs;
-}
-
-const BlueprintResolver::FeatureMap &
-BlueprintResolver::getFeatureMap() const
-{
-    return _featureMap;
-}
-
-const BlueprintResolver::FeatureMap &
-BlueprintResolver::getSeedMap() const
-{
-    return _seedMap;
-}
-
 }
diff --git a/searchlib/src/vespa/searchlib/fef/blueprintresolver.h b/searchlib/src/vespa/searchlib/fef/blueprintresolver.h
index 3e3b5879518..967f713d32c 100644
--- a/searchlib/src/vespa/searchlib/fef/blueprintresolver.h
+++ b/searchlib/src/vespa/searchlib/fef/blueprintresolver.h
@@ -24,7 +24,8 @@ class FeatureNameParser;
 class BlueprintResolver
 {
 public:
-    typedef std::shared_ptr<BlueprintResolver> SP;
+    using SP = std::shared_ptr<BlueprintResolver>;
+    using Errors = std::vector<vespalib::string>;
 
     /**
      * Low-level reference to a single output from a feature
@@ -44,7 +45,7 @@ public:
             : executor(executor_in), output(output_in) {}
         bool valid() { return (executor != undef); }
     };
-    typedef std::map<vespalib::string, FeatureRef> FeatureMap;
+    using FeatureMap = std::map<vespalib::string, FeatureRef>;
 
     /**
      * Thin blueprint wrapper with additional information about how
@@ -59,7 +60,7 @@ public:
         ExecutorSpec(Blueprint::SP blueprint_in);
         ~ExecutorSpec();
     };
-    typedef std::vector<ExecutorSpec> ExecutorSpecList;
+    using ExecutorSpecList = std::vector<ExecutorSpec>;
 
     /**
      * The maximum dependency depth. This value is defined to protect
@@ -84,6 +85,7 @@ private:
     ExecutorSpecList              _executorSpecs;
     FeatureMap                    _featureMap;
     FeatureMap                    _seedMap;
+    Errors                        _compileErrors;
 
 public:
     BlueprintResolver(const BlueprintResolver &) = delete;
@@ -134,7 +136,7 @@ public:
      *
      * @return feature executor assembly directions
      **/
-    const ExecutorSpecList &getExecutorSpecs() const;
+    const ExecutorSpecList &getExecutorSpecs() const { return _executorSpecs; }
 
     /**
      * Obtain the location of all named features known to this
@@ -145,7 +147,7 @@ public:
      *
      * @return feature locations
      **/
-    const FeatureMap &getFeatureMap() const;
+    const FeatureMap &getFeatureMap() const { return _featureMap; }
 
     /**
      * Obtain the location of all seeds used by this resolver. This
@@ -156,7 +158,13 @@ public:
      *
      * @return seed locations
      **/
-    const FeatureMap &getSeedMap() const;
+    const FeatureMap &getSeedMap() const { return _seedMap; }
+
+    /**
+     * Will return any accumulated errors during compile
+     * @return list of errors
+     **/
+    const Errors & getCompileErrors() const { return _compileErrors; }
 };
 
 }
diff --git a/searchlib/src/vespa/searchlib/fef/ranksetup.cpp b/searchlib/src/vespa/searchlib/fef/ranksetup.cpp
index ed841ae24b3..1c1adfe53fa 100644
--- a/searchlib/src/vespa/searchlib/fef/ranksetup.cpp
+++ b/searchlib/src/vespa/searchlib/fef/ranksetup.cpp
@@ -3,9 +3,9 @@
 #include "ranksetup.h"
 #include "indexproperties.h"
 #include "featurenameparser.h"
+#include <vespa/vespalib/util/stringfmt.h>
 
-#include <vespa/log/log.h>
-LOG_SETUP(".fef.ranksetup");
+using vespalib::make_string_short::fmt;
 
 namespace {
 class VisitorAdapter : public search::fef::IDumpFeatureVisitor
@@ -53,6 +53,7 @@ RankSetup::RankSetup(const BlueprintFactory &factory, const IIndexEnvironment &i
       _match_features(),
       _summaryFeatures(),
       _dumpFeatures(),
+      _compileErrors(),
       _feature_rename_map(),
       _ignoreDefaultRankFeatures(false),
       _compiled(false),
@@ -134,50 +135,59 @@ RankSetup::configure()
 void
 RankSetup::setFirstPhaseRank(const vespalib::string &featureName)
 {
-    LOG_ASSERT(!_compiled);
+    assert(!_compiled);
     _firstPhaseRankFeature = featureName;
 }
 
 void
 RankSetup::setSecondPhaseRank(const vespalib::string &featureName)
 {
-    LOG_ASSERT(!_compiled);
+    assert(!_compiled);
     _secondPhaseRankFeature = featureName;
 }
 
 void
 RankSetup::add_match_feature(const vespalib::string &match_feature)
 {
-    LOG_ASSERT(!_compiled);
+    assert(!_compiled);
     _match_features.push_back(match_feature);
 }
 
 void
 RankSetup::addSummaryFeature(const vespalib::string &summaryFeature)
 {
-    LOG_ASSERT(!_compiled);
+    assert(!_compiled);
     _summaryFeatures.push_back(summaryFeature);
 }
 
 void
 RankSetup::addDumpFeature(const vespalib::string &dumpFeature)
 {
-    LOG_ASSERT(!_compiled);
+    assert(!_compiled);
     _dumpFeatures.push_back(dumpFeature);
 }
 
+void
+RankSetup::compileAndCheckForErrors(BlueprintResolver &bpr) {
+    bool ok = bpr.compile();
+    if ( ! ok ) {
+        _compileError = true;
+        const Errors & errors = bpr.getCompileErrors();
+        _compileErrors.insert(_compileErrors.end(), errors.begin(), errors.end());
+    }
+}
 bool
 RankSetup::compile()
 {
-    LOG_ASSERT(!_compiled);
+    assert(!_compiled);
     if (!_firstPhaseRankFeature.empty()) {
         FeatureNameParser parser(_firstPhaseRankFeature);
         if (parser.valid()) {
             _firstPhaseRankFeature = parser.featureName();
             _first_phase_resolver->addSeed(_firstPhaseRankFeature);
         } else {
-            LOG(warning, "invalid feature name for initial rank: '%s'",
-                _firstPhaseRankFeature.c_str());
+            vespalib::string e = fmt("invalid feature name for initial rank: '%s'", _firstPhaseRankFeature.c_str());
+            _compileErrors.emplace_back(e);
             _compileError = true;
         }
     }
@@ -187,8 +197,8 @@ RankSetup::compile()
             _secondPhaseRankFeature = parser.featureName();
             _second_phase_resolver->addSeed(_secondPhaseRankFeature);
         } else {
-            LOG(warning, "invalid feature name for final rank: '%s'",
-                _secondPhaseRankFeature.c_str());
+            vespalib::string e = fmt("invalid feature name for final rank: '%s'", _secondPhaseRankFeature.c_str());
+            _compileErrors.emplace_back(e);
             _compileError = true;
         }
     }
@@ -206,12 +216,12 @@ RankSetup::compile()
         _dumpResolver->addSeed(feature);
     }
     _indexEnv.hintFeatureMotivation(IIndexEnvironment::RANK);
-    _compileError |= !_first_phase_resolver->compile();
-    _compileError |= !_second_phase_resolver->compile();
-    _compileError |= !_match_resolver->compile();
-    _compileError |= !_summary_resolver->compile();
+    compileAndCheckForErrors(*_first_phase_resolver);
+    compileAndCheckForErrors(*_second_phase_resolver);
+    compileAndCheckForErrors(*_match_resolver);
+    compileAndCheckForErrors(*_summary_resolver);
     _indexEnv.hintFeatureMotivation(IIndexEnvironment::DUMP);
-    _compileError |= !_dumpResolver->compile();
+    compileAndCheckForErrors(*_dumpResolver);
     _compiled = true;
     return !_compileError;
 }
@@ -219,7 +229,7 @@ RankSetup::compile()
 void
 RankSetup::prepareSharedState(const IQueryEnvironment &queryEnv, IObjectStore &objectStore) const
 {
-    LOG_ASSERT(_compiled && !_compileError);
+    assert(_compiled && !_compileError);
     for (const auto &spec : _first_phase_resolver->getExecutorSpecs()) {
         spec.blueprint->prepareSharedState(queryEnv, objectStore);
     }
diff --git a/searchlib/src/vespa/searchlib/fef/ranksetup.h b/searchlib/src/vespa/searchlib/fef/ranksetup.h
index 6a2871827ab..f0973daed88 100644
--- a/searchlib/src/vespa/searchlib/fef/ranksetup.h
+++ b/searchlib/src/vespa/searchlib/fef/ranksetup.h
@@ -23,6 +23,7 @@ namespace search::fef {
 class RankSetup
 {
 public:
+    using Errors = BlueprintResolver::Errors;
     struct MutateOperation {
     public:
         MutateOperation() : MutateOperation("", "") {}
@@ -62,6 +63,7 @@ private:
     std::vector<vespalib::string> _match_features;
     std::vector<vespalib::string> _summaryFeatures;
     std::vector<vespalib::string> _dumpFeatures;
+    Errors                   _compileErrors;
     StringStringMap          _feature_rename_map;
     bool                     _ignoreDefaultRankFeatures;
     bool                     _compiled;
@@ -80,6 +82,8 @@ private:
     MutateOperation          _mutateOnFirstPhase;
     MutateOperation          _mutateOnSecondPhase;
     MutateOperation          _mutateOnSummary;
+
+    void compileAndCheckForErrors(BlueprintResolver &bp);
 public:
     RankSetup(const RankSetup &) = delete;
     RankSetup &operator=(const RankSetup &) = delete;
@@ -438,6 +442,12 @@ public:
      **/
     bool compile();
 
+    /**
+     * Will return any accumulated errors during compile
+     * @return list of errors
+     */
+    const Errors & getCompileErrors() const { return _compileErrors; }
+
     // These functions create rank programs for different tasks. Note
     // that the setup function must be called on rank programs for
     // them to be ready to use. Also keep in mind that creating a rank
diff --git a/searchlib/src/vespa/searchlib/fef/verify_feature.cpp b/searchlib/src/vespa/searchlib/fef/verify_feature.cpp
index 85d1daffd01..2f98e3d4c1e 100644
--- a/searchlib/src/vespa/searchlib/fef/verify_feature.cpp
+++ b/searchlib/src/vespa/searchlib/fef/verify_feature.cpp
@@ -2,28 +2,29 @@
 
 #include "verify_feature.h"
 #include "blueprintresolver.h"
+#include <vespa/vespalib/util/stringfmt.h>
 
-#include <vespa/log/log.h>
-LOG_SETUP(".fef.verify_feature");
+using vespalib::make_string_short::fmt;
 
-namespace search {
-namespace fef {
+namespace search::fef {
 
 bool verifyFeature(const BlueprintFactory &factory,
                    const IIndexEnvironment &indexEnv,
                    const std::string &featureName,
-                   const std::string &desc)
+                   const std::string &desc,
+                   std::vector<vespalib::string> & errors)
 {
     indexEnv.hintFeatureMotivation(IIndexEnvironment::VERIFY_SETUP);
     BlueprintResolver resolver(factory, indexEnv);
     resolver.addSeed(featureName);
     bool result = resolver.compile();
     if (!result) {
-        LOG(error, "verification failed: %s (%s)",
-            BlueprintResolver::describe_feature(featureName).c_str(), desc.c_str());
+        const BlueprintResolver::Errors & compileErrors(resolver.getCompileErrors());
+        errors.insert(errors.end(), compileErrors.begin(), compileErrors.end());
+        vespalib::string msg = fmt("verification failed: %s (%s)",BlueprintResolver::describe_feature(featureName).c_str(), desc.c_str());
+        errors.emplace_back(msg);
     }
     return result;
 }
 
-} // namespace fef
-} // namespace search
+}
diff --git a/searchlib/src/vespa/searchlib/fef/verify_feature.h b/searchlib/src/vespa/searchlib/fef/verify_feature.h
index 5c84403fc8a..f05332baf77 100644
--- a/searchlib/src/vespa/searchlib/fef/verify_feature.h
+++ b/searchlib/src/vespa/searchlib/fef/verify_feature.h
@@ -4,10 +4,8 @@
 
 #include "blueprintfactory.h"
 #include "iindexenvironment.h"
-#include <string>
 
-namespace search {
-namespace fef {
+namespace search::fef {
 
 /**
  * Verify whether a specific feature can be computed. If the feature
@@ -23,8 +21,7 @@ namespace fef {
 bool verifyFeature(const BlueprintFactory &factory,
                    const IIndexEnvironment &indexEnv,
                    const std::string &featureName,
-                   const std::string &desc);
-
-} // namespace fef
-} // namespace search
+                   const std::string &desc,
+                   std::vector<vespalib::string> & errors);
 
+}