Publish

1 files changed, 641 insertions, 0 deletions

diff --git a/searchlib/src/tests/queryeval/termwise_eval/termwise_eval_test.cpp b/searchlib/src/tests/queryeval/termwise_eval/termwise_eval_test.cpp
new file mode 100644
index 00000000000..625d9928048
--- /dev/null
+++ b/searchlib/src/tests/queryeval/termwise_eval/termwise_eval_test.cpp
@@ -0,0 +1,641 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+#include <vespa/vespalib/testkit/test_kit.h>
+#include <vespa/vespalib/util/stringfmt.h>
+#include <vespa/searchlib/queryeval/searchiterator.h>
+#include <vespa/searchlib/queryeval/andnotsearch.h>
+#include <vespa/searchlib/queryeval/andsearch.h>
+#include <vespa/searchlib/queryeval/orsearch.h>
+#include <vespa/searchlib/queryeval/termwise_search.h>
+#include <vespa/searchlib/queryeval/intermediate_blueprints.h>
+#include <vespa/searchlib/queryeval/termwise_blueprint_helper.h>
+#include <vespa/vespalib/test/insertion_operators.h>
+#include <vespa/searchlib/test/initrange.h>
+#include <vespa/searchlib/common/bitvectoriterator.h>
+
+using namespace vespalib;
+using namespace search;
+using namespace search::fef;
+using namespace search::queryeval;
+
+//-----------------------------------------------------------------------------
+
+const uint32_t my_field = 0;
+
+//-----------------------------------------------------------------------------
+
+struct MyTerm : public SearchIterator {
+    size_t pos;
+    bool is_strict;
+    std::vector<uint32_t> hits;
+    MyTerm(const std::vector<uint32_t> &hits_in, bool is_strict_in)
+        : pos(0), is_strict(is_strict_in), hits(hits_in) {}
+    void initRange(uint32_t beginid, uint32_t endid) override {
+        SearchIterator::initRange(beginid, endid);
+        if (is_strict) {
+            doSeek(beginid);
+        }
+    }
+    void resetRange() override {
+        SearchIterator::resetRange();
+        pos = 0;
+    }
+    void doSeek(uint32_t docid) override {
+        while ((pos < hits.size()) && (hits[pos] < docid)) {
+            ++pos;
+        }
+        if (is_strict) {
+            if ((pos == hits.size()) || isAtEnd(hits[pos])) {
+                setAtEnd();
+            } else {
+                setDocId(hits[pos]);
+            }
+        } else {
+            if (isAtEnd(docid)) {
+                setAtEnd();
+            } else if ((pos < hits.size()) && (hits[pos] == docid)) {
+                setDocId(docid);
+            }
+        }
+    }
+    void doUnpack(uint32_t) override {}
+    void visitMembers(vespalib::ObjectVisitor &visitor) const {
+        visit(visitor, "hits", hits);
+        visit(visitor, "strict", is_strict);
+    }
+};
+
+struct MyBlueprint : SimpleLeafBlueprint {
+    std::vector<uint32_t> hits;
+    MyBlueprint(const std::vector<uint32_t> &hits_in)
+        : SimpleLeafBlueprint(FieldSpecBaseList()), hits(hits_in)
+    {
+        setEstimate(HitEstimate(hits.size(), hits.empty()));
+    }
+    MyBlueprint(const std::vector<uint32_t> &hits_in, bool allow_termwise_eval)
+        : SimpleLeafBlueprint(FieldSpecBaseList()), hits(hits_in)
+    {
+        setEstimate(HitEstimate(hits.size(), hits.empty()));
+        set_allow_termwise_eval(allow_termwise_eval);
+    }
+    MyBlueprint(const std::vector<uint32_t> &hits_in, bool allow_termwise_eval, TermFieldHandle handle)
+        : SimpleLeafBlueprint(FieldSpecBase(my_field, handle)), hits(hits_in)
+    {
+        setEstimate(HitEstimate(hits.size(), hits.empty()));
+        set_allow_termwise_eval(allow_termwise_eval);
+    }
+    SearchIterator::UP createLeafSearch(const fef::TermFieldMatchDataArray &,
+                                        bool strict) const override
+    {
+        return SearchIterator::UP(new MyTerm(hits, strict));
+    }
+};
+
+struct MyOr : OrBlueprint {
+    bool use_my_value;
+    bool my_value;
+    MyOr(bool use_my_value_in, bool my_value_in = true)
+        : use_my_value(use_my_value_in), my_value(my_value_in) {}
+    bool supports_termwise_children() const override {
+        if (use_my_value) {
+            return my_value;
+        }
+        // the default value for intermediate blueprints
+        return IntermediateBlueprint::supports_termwise_children();
+    }
+};
+
+//-----------------------------------------------------------------------------
+
+UnpackInfo no_unpack() { return UnpackInfo(); }
+
+UnpackInfo selective_unpack() {
+    UnpackInfo unpack;
+    unpack.add(0); // 'only unpack first child' => trigger selective unpack
+    return unpack;
+}
+
+SearchIterator *TERM(std::initializer_list<uint32_t> hits, bool strict) {
+    return new MyTerm(hits, strict);
+}
+
+SearchIterator *ANDNOT(std::initializer_list<SearchIterator *> children, bool strict) {
+    return AndNotSearch::create(children, strict);
+}
+
+SearchIterator *AND(std::initializer_list<SearchIterator *> children, bool strict) {
+    return AndSearch::create(children, strict);
+}
+
+SearchIterator *ANDz(std::initializer_list<SearchIterator *> children, bool strict) {
+    return AndSearch::create(children, strict, no_unpack());
+}
+
+SearchIterator *ANDs(std::initializer_list<SearchIterator *> children, bool strict) {
+    return AndSearch::create(children, strict, selective_unpack());
+}
+
+SearchIterator *OR(std::initializer_list<SearchIterator *> children, bool strict) {
+    return OrSearch::create(children, strict);
+}
+
+SearchIterator *ORz(std::initializer_list<SearchIterator *> children, bool strict) {
+    return OrSearch::create(children, strict, no_unpack());
+}
+
+SearchIterator *ORs(std::initializer_list<SearchIterator *> children, bool strict) {
+    return OrSearch::create(children, strict, selective_unpack());
+}
+
+//-----------------------------------------------------------------------------
+
+template <typename T>
+std::unique_ptr<T> UP(T *t) { return std::unique_ptr<T>(t); }
+
+//-----------------------------------------------------------------------------
+
+SearchIterator::UP make_search(bool strict) {
+    return UP(AND({OR({TERM({2,7}, true),
+                       TERM({4,8}, true),
+                       TERM({5,6,9}, true)}, true),
+                   OR({TERM({1,4,7}, false),
+                       TERM({2,5,8}, true),
+                       TERM({3,6}, false)}, false),
+                   OR({TERM({1,2,3}, false),
+                       TERM({4,6}, false),
+                       TERM({8,9}, false)}, false)}, strict));
+}
+
+SearchIterator::UP make_filter_search(bool strict) {
+    return UP(ANDNOT({TERM({1,2,3,4,5,6,7,8,9}, true),
+                      TERM({1,9}, false),
+                      TERM({3,7}, true),
+                      TERM({5}, false)}, strict));
+}
+
+void add_if_inside(uint32_t docid, uint32_t begin, uint32_t end, std::vector<uint32_t> &expect) {
+    if (docid >= begin && docid < end) {
+        expect.push_back(docid);
+    }
+}
+
+std::vector<uint32_t> make_expect(uint32_t begin, uint32_t end) {
+    std::vector<uint32_t> expect;
+    add_if_inside(2, begin, end, expect);
+    add_if_inside(4, begin, end, expect);
+    add_if_inside(6, begin, end, expect);
+    add_if_inside(8, begin, end, expect);
+    return expect;
+}
+
+void verify(const std::vector<uint32_t> &expect, SearchIterator &search, uint32_t begin, uint32_t end) {
+    std::vector<uint32_t> actual;
+    search.initRange(begin, end);
+    for (uint32_t docid = begin; docid < end; ++docid) {
+        if (search.seek(docid)) {
+            actual.push_back(docid);
+        }
+    }
+    EXPECT_EQUAL(expect, actual);
+}
+
+//-----------------------------------------------------------------------------
+
+MatchData::UP make_match_data() {
+    uint32_t num_features = 0;
+    uint32_t num_handles = 100;
+    uint32_t num_fields = 1;
+    return MatchData::makeTestInstance(num_features, num_handles, num_fields);
+}
+
+//-----------------------------------------------------------------------------
+
+TEST("require that pseudo term produces correct results") {
+    TEST_DO(verify({1,2,3,4,5}, *UP(TERM({1,2,3,4,5}, true)), 1, 6));
+    TEST_DO(verify({1,2,3,4,5}, *UP(TERM({1,2,3,4,5}, false)), 1, 6));
+    TEST_DO(verify({3,4,5}, *UP(TERM({1,2,3,4,5}, true)), 3, 6));
+    TEST_DO(verify({3,4,5}, *UP(TERM({1,2,3,4,5}, false)), 3, 6));
+    TEST_DO(verify({1,2,3}, *UP(TERM({1,2,3,4,5}, true)), 1, 4));
+    TEST_DO(verify({1,2,3}, *UP(TERM({1,2,3,4,5}, false)), 1, 4));
+}
+
+TEST("require that normal search gives expected results") {
+    auto search = make_search(true);
+    TEST_DO(verify(make_expect(1, 10), *search, 1, 10));
+}
+
+TEST("require that filter search gives expected results") {
+    auto search = make_filter_search(true);
+    TEST_DO(verify(make_expect(1, 10), *search, 1, 10));
+}
+
+TEST("require that termwise AND/OR search produces appropriate results") {
+    for (uint32_t begin: {1, 2, 5}) {
+        for (uint32_t end: {6, 7, 10}) {
+            for (bool strict_search: {true, false}) {
+                for (bool strict_wrapper: {true, false}) {
+                    TEST_STATE(make_string("begin: %u, end: %u, strict_search: %s, strict_wrapper: %s",
+                                    begin, end, strict_search ? "true" : "false",
+                                    strict_wrapper ? "true" : "false").c_str());
+                    auto search = make_termwise(make_search(strict_search), strict_wrapper);
+                    TEST_DO(verify(make_expect(begin, end), *search, begin, end));
+                }
+            }
+        }
+    }
+}
+
+TEST("require that termwise filter search produces appropriate results") {
+    for (uint32_t begin: {1, 2, 5}) {
+        for (uint32_t end: {6, 7, 10}) {
+            for (bool strict_search: {true, false}) {
+                for (bool strict_wrapper: {true, false}) {
+                    TEST_STATE(make_string("begin: %u, end: %u, strict_search: %s, strict_wrapper: %s",
+                                    begin, end, strict_search ? "true" : "false",
+                                    strict_wrapper ? "true" : "false").c_str());
+                    auto search = make_termwise(make_filter_search(strict_search), strict_wrapper);
+                    TEST_DO(verify(make_expect(begin, end), *search, begin, end));
+                }
+            }
+        }
+    }
+}
+
+TEST("require that termwise ANDNOT with single term works") {
+    TEST_DO(verify({2,3,4}, *make_termwise(UP(ANDNOT({TERM({1,2,3,4,5}, true)}, true)), true), 2, 5));
+}
+
+TEST("require that pseudo term is rewindable") {
+    auto search = UP(TERM({1,2,3,4,5}, true));
+    TEST_DO(verify({3,4,5}, *search, 3, 6));
+    search->resetRange();
+    TEST_DO(verify({1,2,3,4}, *search, 1, 5));
+}
+
+TEST("require that termwise wrapper is rewindable") {
+    auto search = make_termwise(make_search(true), true);
+    TEST_DO(verify(make_expect(3, 7), *search, 3, 7));
+    search->resetRange();
+    TEST_DO(verify(make_expect(1, 5), *search, 1, 5));
+}
+
+//-----------------------------------------------------------------------------
+
+TEST("require that leaf blueprints allow termwise evaluation by default") {
+    MyBlueprint bp({});
+    EXPECT_TRUE(bp.getState().allow_termwise_eval());
+}
+
+TEST("require that leaf blueprints can enable/disable termwise evaluation") {
+    MyBlueprint enable({}, true);
+    MyBlueprint disable({}, false);
+    EXPECT_TRUE(enable.getState().allow_termwise_eval());
+    EXPECT_FALSE(disable.getState().allow_termwise_eval());
+}
+
+TEST("require that intermediate blueprints disallow termwise evaluation by default") {
+    MyOr bp(false);
+    bp.addChild(UP(new MyBlueprint({}, true)));
+    bp.addChild(UP(new MyBlueprint({}, true)));
+    EXPECT_FALSE(bp.getState().allow_termwise_eval());
+}
+
+TEST("require that intermediate blueprints can enable/disable termwise evaluation") {
+    MyOr enable(true, true);
+    enable.addChild(UP(new MyBlueprint({}, true)));
+    enable.addChild(UP(new MyBlueprint({}, true)));
+    EXPECT_TRUE(enable.getState().allow_termwise_eval());
+    MyOr disable(true, false);
+    disable.addChild(UP(new MyBlueprint({}, true)));
+    disable.addChild(UP(new MyBlueprint({}, true)));
+    EXPECT_FALSE(disable.getState().allow_termwise_eval());
+}
+
+TEST("require that intermediate blueprints cannot be termwise unless all its children are termwise") {
+    MyOr bp(true, true);
+    bp.addChild(UP(new MyBlueprint({}, true)));
+    bp.addChild(UP(new MyBlueprint({}, false)));
+    EXPECT_FALSE(bp.getState().allow_termwise_eval());
+}
+
+//-----------------------------------------------------------------------------
+
+TEST("require that leafs have tree size 1") {
+    MyBlueprint bp({});
+    EXPECT_EQUAL(1u, bp.getState().tree_size());    
+}
+
+TEST("require that tree size is accumulated correctly by intermediate nodes") {
+    MyOr bp(false);
+    EXPECT_EQUAL(1u, bp.getState().tree_size());    
+    bp.addChild(UP(new MyBlueprint({})));
+    bp.addChild(UP(new MyBlueprint({})));
+    EXPECT_EQUAL(3u, bp.getState().tree_size());
+    auto child = UP(new MyOr(false));
+    child->addChild(UP(new MyBlueprint({})));
+    child->addChild(UP(new MyBlueprint({})));
+    bp.addChild(std::move(child));
+    EXPECT_EQUAL(6u, bp.getState().tree_size());
+}
+
+//-----------------------------------------------------------------------------
+
+TEST("require that any blueprint node can obtain the root") {
+    MyOr bp(false);
+    bp.addChild(UP(new MyBlueprint({1,2,3})));
+    bp.addChild(UP(new MyBlueprint({1,2,3,4,5,6})));
+    EXPECT_TRUE(&bp != &bp.getChild(0));
+    EXPECT_TRUE(&bp != &bp.getChild(1));
+    EXPECT_TRUE(&bp == &bp.getChild(0).root());
+    EXPECT_TRUE(&bp == &bp.getChild(1).root());
+    EXPECT_TRUE(&bp == &bp.root());
+}
+
+//-----------------------------------------------------------------------------
+
+TEST("require that match data keeps track of the termwise limit") {
+    auto md = make_match_data();
+    EXPECT_EQUAL(1.0, md->get_termwise_limit());
+    md->set_termwise_limit(0.03);
+    EXPECT_EQUAL(0.03, md->get_termwise_limit());
+}
+
+//-----------------------------------------------------------------------------
+
+TEST("require that terwise test search string dump is detailed enough") {
+    EXPECT_EQUAL(make_termwise(UP(OR({TERM({1,2,3}, true), TERM({2,3}, true), TERM({3}, true)}, true)), true)->asString(),
+                 make_termwise(UP(OR({TERM({1,2,3}, true), TERM({2,3}, true), TERM({3}, true)}, true)), true)->asString());
+
+    EXPECT_NOT_EQUAL(make_termwise(UP(OR({TERM({1,2,3}, true), TERM({2,3}, true), TERM({3}, true)}, true)), true)->asString(),
+                     make_termwise(UP(OR({TERM({1,2,3}, true), TERM({2,3}, false), TERM({3}, true)}, true)), true)->asString());
+
+    EXPECT_NOT_EQUAL(make_termwise(UP(OR({TERM({1,2,3}, true), TERM({2,3}, true), TERM({3}, true)}, true)), true)->asString(),
+                     make_termwise(UP(OR({TERM({1,2,3}, true), TERM({2,3}, true), TERM({3}, true)}, false)), true)->asString());
+
+    EXPECT_NOT_EQUAL(make_termwise(UP(OR({TERM({1,2,3}, true), TERM({2,3}, true), TERM({3}, true)}, true)), true)->asString(),
+                     make_termwise(UP(OR({TERM({1,2,3}, true), TERM({2,3}, true), TERM({3}, true)}, true)), false)->asString());
+
+    EXPECT_NOT_EQUAL(make_termwise(UP(OR({TERM({1,2,3}, true), TERM({2,3}, true), TERM({3}, true)}, true)), true)->asString(),
+                     make_termwise(UP(OR({TERM({1,2,3}, true), TERM({3}, true), TERM({2,3}, true)}, true)), true)->asString());
+}
+
+//-----------------------------------------------------------------------------
+
+TEST("require that basic termwise evaluation works") {
+    auto md = make_match_data();
+    md->set_termwise_limit(0.0);
+    md->resolveTermField(1)->tagAsNotNeeded();
+    md->resolveTermField(2)->tagAsNotNeeded();
+    OrBlueprint my_or;
+    my_or.addChild(UP(new MyBlueprint({1}, true, 1)));
+    my_or.addChild(UP(new MyBlueprint({2}, true, 2)));
+    for (bool strict: {true, false}) {
+        EXPECT_EQUAL(my_or.createSearch(*md, strict)->asString(),
+                     make_termwise(UP(OR({TERM({1}, strict), TERM({2}, strict)}, strict)), strict)->asString());
+    }
+}
+
+TEST("require that the hit rate must be high enough for termwise evaluation to be activated") {
+    auto md = make_match_data();
+    md->set_termwise_limit(1.0); // <-
+    md->resolveTermField(1)->tagAsNotNeeded();
+    md->resolveTermField(2)->tagAsNotNeeded();
+    OrBlueprint my_or;
+    my_or.addChild(UP(new MyBlueprint({1}, true, 1)));
+    my_or.addChild(UP(new MyBlueprint({2}, true, 2)));
+    for (bool strict: {true, false}) {
+        EXPECT_TRUE(my_or.createSearch(*md, strict)->asString().find("TermwiseSearch") == vespalib::string::npos);
+    }
+}
+
+TEST("require that enough unranked termwise terms are present for termwise evaluation to be activated") {
+    auto md = make_match_data();
+    md->set_termwise_limit(0.0);
+    md->resolveTermField(1)->tagAsNotNeeded();
+    md->resolveTermField(2)->tagAsNotNeeded();
+    OrBlueprint my_or;
+    my_or.addChild(UP(new MyBlueprint({1}, true, 1)));
+    my_or.addChild(UP(new MyBlueprint({2}, false, 2))); // <- not termwise
+    my_or.addChild(UP(new MyBlueprint({3}, true, 3)));  // <- ranked
+    for (bool strict: {true, false}) {
+        EXPECT_TRUE(my_or.createSearch(*md, strict)->asString().find("TermwiseSearch") == vespalib::string::npos);
+    }
+}
+
+TEST("require that termwise evaluation can be multi-level, but not duplicated") {
+    auto md = make_match_data();
+    md->set_termwise_limit(0.0);
+    md->resolveTermField(1)->tagAsNotNeeded();
+    md->resolveTermField(2)->tagAsNotNeeded();
+    md->resolveTermField(3)->tagAsNotNeeded();
+    OrBlueprint my_or;
+    my_or.addChild(UP(new MyBlueprint({1}, true, 1)));    
+    auto child = UP(new OrBlueprint());
+    child->addChild(UP(new MyBlueprint({2}, true, 2)));
+    child->addChild(UP(new MyBlueprint({3}, true, 3)));
+    my_or.addChild(std::move(child));
+    for (bool strict: {true, false}) {
+        EXPECT_EQUAL(my_or.createSearch(*md, strict)->asString(),
+                     make_termwise(UP(OR({TERM({1}, strict), ORz({TERM({2}, strict), TERM({3}, strict)}, strict)}, strict)), strict)->asString());
+    }
+}
+
+//-----------------------------------------------------------------------------
+
+TEST("require that OR can be completely termwise") {
+    auto md = make_match_data();
+    md->set_termwise_limit(0.0);
+    md->resolveTermField(1)->tagAsNotNeeded();
+    md->resolveTermField(2)->tagAsNotNeeded();
+    OrBlueprint my_or;
+    my_or.addChild(UP(new MyBlueprint({1}, true, 1)));
+    my_or.addChild(UP(new MyBlueprint({2}, true, 2)));
+    for (bool strict: {true, false}) {
+        EXPECT_EQUAL(my_or.createSearch(*md, strict)->asString(),
+                     make_termwise(UP(OR({TERM({1}, strict), TERM({2}, strict)}, strict)), strict)->asString());
+    }
+}
+
+TEST("require that OR can be partially termwise") {
+    auto md = make_match_data();
+    md->set_termwise_limit(0.0);
+    md->resolveTermField(1)->tagAsNotNeeded();
+    md->resolveTermField(3)->tagAsNotNeeded();
+    OrBlueprint my_or;
+    my_or.addChild(UP(new MyBlueprint({1}, true, 1)));
+    my_or.addChild(UP(new MyBlueprint({2}, true, 2)));
+    my_or.addChild(UP(new MyBlueprint({3}, true, 3)));
+    for (bool strict: {true, false}) {
+        EXPECT_EQUAL(my_or.createSearch(*md, strict)->asString(),
+                     UP(ORs({make_termwise(UP(OR({TERM({1}, strict), TERM({3}, strict)}, strict)), strict).release(), TERM({2}, strict)}, strict))->asString());
+    }
+}
+
+TEST("require that OR puts termwise subquery at the right place") {
+    auto md = make_match_data();
+    md->set_termwise_limit(0.0);
+    md->resolveTermField(2)->tagAsNotNeeded();
+    md->resolveTermField(3)->tagAsNotNeeded();
+    OrBlueprint my_or;
+    my_or.addChild(UP(new MyBlueprint({1}, true, 1)));
+    my_or.addChild(UP(new MyBlueprint({2}, true, 2)));
+    my_or.addChild(UP(new MyBlueprint({3}, true, 3)));
+    for (bool strict: {true, false}) {
+        EXPECT_EQUAL(my_or.createSearch(*md, strict)->asString(),
+                     UP(ORs({TERM({1}, strict), make_termwise(UP(OR({TERM({2}, strict), TERM({3}, strict)}, strict)), strict).release()}, strict))->asString());
+    }
+}
+
+TEST("require that OR can use termwise eval also when having non-termwise children") {
+    auto md = make_match_data();
+    md->set_termwise_limit(0.0);
+    md->resolveTermField(1)->tagAsNotNeeded();
+    md->resolveTermField(2)->tagAsNotNeeded();
+    md->resolveTermField(3)->tagAsNotNeeded();
+    OrBlueprint my_or;
+    my_or.addChild(UP(new MyBlueprint({1}, false, 1)));
+    my_or.addChild(UP(new MyBlueprint({2}, true, 2)));
+    my_or.addChild(UP(new MyBlueprint({3}, true, 3)));
+    for (bool strict: {true, false}) {
+        EXPECT_EQUAL(my_or.createSearch(*md, strict)->asString(),
+                     UP(ORz({TERM({1}, strict), make_termwise(UP(OR({TERM({2}, strict), TERM({3}, strict)}, strict)), strict).release()}, strict))->asString());
+    }
+}
+
+//-----------------------------------------------------------------------------
+
+TEST("require that AND can be completely termwise") {
+    auto md = make_match_data();
+    md->set_termwise_limit(0.0);
+    md->resolveTermField(1)->tagAsNotNeeded();
+    md->resolveTermField(2)->tagAsNotNeeded();
+    AndBlueprint my_and;
+    my_and.addChild(UP(new MyBlueprint({1}, true, 1)));
+    my_and.addChild(UP(new MyBlueprint({2}, true, 2)));
+    for (bool strict: {true, false}) {
+        EXPECT_EQUAL(my_and.createSearch(*md, strict)->asString(),
+                     make_termwise(UP(AND({TERM({1}, strict), TERM({2}, false)}, strict)), strict)->asString());
+    }
+}
+
+TEST("require that AND can be partially termwise") {
+    auto md = make_match_data();
+    md->set_termwise_limit(0.0);
+    md->resolveTermField(1)->tagAsNotNeeded();
+    md->resolveTermField(3)->tagAsNotNeeded();
+    AndBlueprint my_and;
+    my_and.addChild(UP(new MyBlueprint({1}, true, 1)));
+    my_and.addChild(UP(new MyBlueprint({2}, true, 2)));
+    my_and.addChild(UP(new MyBlueprint({3}, true, 3)));
+    for (bool strict: {true, false}) {
+        EXPECT_EQUAL(my_and.createSearch(*md, strict)->asString(),
+                     UP(ANDs({make_termwise(UP(AND({TERM({1}, strict), TERM({3}, false)}, strict)), strict).release(), TERM({2}, false)}, strict))->asString());
+    }
+}
+
+TEST("require that AND puts termwise subquery at the right place") {
+    auto md = make_match_data();
+    md->set_termwise_limit(0.0);
+    md->resolveTermField(2)->tagAsNotNeeded();
+    md->resolveTermField(3)->tagAsNotNeeded();
+    AndBlueprint my_and;
+    my_and.addChild(UP(new MyBlueprint({1}, true, 1)));
+    my_and.addChild(UP(new MyBlueprint({2}, true, 2)));
+    my_and.addChild(UP(new MyBlueprint({3}, true, 3)));
+    for (bool strict: {true, false}) {
+        EXPECT_EQUAL(my_and.createSearch(*md, strict)->asString(),
+                     UP(ANDs({TERM({1}, strict), make_termwise(UP(AND({TERM({2}, false), TERM({3}, false)}, false)), false).release()}, strict))->asString());
+    }
+}
+
+TEST("require that AND can use termwise eval also when having non-termwise children") {
+    auto md = make_match_data();
+    md->set_termwise_limit(0.0);
+    md->resolveTermField(1)->tagAsNotNeeded();
+    md->resolveTermField(2)->tagAsNotNeeded();
+    md->resolveTermField(3)->tagAsNotNeeded();
+    AndBlueprint my_and;
+    my_and.addChild(UP(new MyBlueprint({1}, false, 1)));
+    my_and.addChild(UP(new MyBlueprint({2}, true, 2)));
+    my_and.addChild(UP(new MyBlueprint({3}, true, 3)));
+    for (bool strict: {true, false}) {
+        EXPECT_EQUAL(my_and.createSearch(*md, strict)->asString(),
+                     UP(ANDz({TERM({1}, strict), make_termwise(UP(AND({TERM({2}, false), TERM({3}, false)}, false)), false).release()}, strict))->asString());
+    }
+}
+
+//-----------------------------------------------------------------------------
+
+TEST("require that ANDNOT can be completely termwise") {
+    auto md = make_match_data();
+    md->set_termwise_limit(0.0);
+    md->resolveTermField(1)->tagAsNotNeeded();
+    AndNotBlueprint my_andnot;
+    my_andnot.addChild(UP(new MyBlueprint({1}, true, 1)));
+    my_andnot.addChild(UP(new MyBlueprint({2}, true, 2)));
+    for (bool strict: {true, false}) {
+        EXPECT_EQUAL(my_andnot.createSearch(*md, strict)->asString(),
+                     make_termwise(UP(ANDNOT({TERM({1}, strict), TERM({2}, false)}, strict)), strict)->asString());
+    }
+}
+
+TEST("require that ANDNOT can be partially termwise") {
+    auto md = make_match_data();
+    md->set_termwise_limit(0.0);
+    AndNotBlueprint my_andnot;
+    my_andnot.addChild(UP(new MyBlueprint({1}, true, 1)));
+    my_andnot.addChild(UP(new MyBlueprint({2}, true, 2)));
+    my_andnot.addChild(UP(new MyBlueprint({3}, true, 3)));
+    for (bool strict: {true, false}) {
+        EXPECT_EQUAL(my_andnot.createSearch(*md, strict)->asString(),
+                     UP(ANDNOT({TERM({1}, strict), make_termwise(UP(OR({TERM({2}, false), TERM({3}, false)}, false)), false).release()}, strict))->asString());
+    }
+}
+
+TEST("require that ANDNOT can be partially termwise with first child being termwise") {
+    auto md = make_match_data();
+    md->set_termwise_limit(0.0);
+    md->resolveTermField(1)->tagAsNotNeeded();
+    AndNotBlueprint my_andnot;
+    my_andnot.addChild(UP(new MyBlueprint({1}, true, 1)));
+    my_andnot.addChild(UP(new MyBlueprint({2}, false, 2)));
+    my_andnot.addChild(UP(new MyBlueprint({3}, true, 3)));
+    for (bool strict: {true, false}) {
+        EXPECT_EQUAL(my_andnot.createSearch(*md, strict)->asString(),
+                     UP(ANDNOT({make_termwise(UP(ANDNOT({TERM({1}, strict), TERM({3}, false)}, strict)), strict).release(), TERM({2}, false)}, strict))->asString());
+    }
+}
+
+//-----------------------------------------------------------------------------
+
+TEST("require that termwise blueprint helper calculates unpack info correctly") {
+    OrBlueprint my_or;
+    my_or.addChild(UP(new MyBlueprint({1}, false, 1))); // termwise not allowed
+    my_or.addChild(UP(new MyBlueprint({2}, false, 2))); // termwise not allowed and ranked
+    my_or.addChild(UP(new MyBlueprint({3}, true, 3)));
+    my_or.addChild(UP(new MyBlueprint({4}, true, 4))); // ranked
+    my_or.addChild(UP(new MyBlueprint({5}, true, 5)));
+    MultiSearch::Children dummy_searches(5, nullptr);
+    UnpackInfo unpack; // non-termwise unpack info
+    unpack.add(1);
+    unpack.add(3);
+    TermwiseBlueprintHelper helper(my_or, dummy_searches, unpack);
+    EXPECT_EQUAL(helper.children.size(), 3u);
+    EXPECT_EQUAL(helper.termwise.size(), 2u);
+    EXPECT_EQUAL(helper.first_termwise, 2u);
+    EXPECT_TRUE(!helper.termwise_unpack.needUnpack(0));
+    EXPECT_TRUE(helper.termwise_unpack.needUnpack(1));
+    EXPECT_TRUE(!helper.termwise_unpack.needUnpack(2));
+    EXPECT_TRUE(helper.termwise_unpack.needUnpack(3));
+    EXPECT_TRUE(!helper.termwise_unpack.needUnpack(4));
+    EXPECT_TRUE(!helper.termwise_unpack.needUnpack(5));
+}
+
+TEST("test that init range works for terwise too.") {
+    search::test::InitRangeVerifier ir;
+    ir.verify(*make_termwise(ir.createIterator(ir.getExpectedDocIds(), false), false));
+    ir.verify(*make_termwise(ir.createIterator(ir.getExpectedDocIds(), true), true));
+}
+
+//-----------------------------------------------------------------------------
+
+TEST_MAIN() { TEST_RUN_ALL(); }