Merge pull request #8021 from vespa-engine/balder/handle-type-25

Ensure that we can print ITEM_WORD_ALTERNATIVES too.

4 files changed, 162 insertions, 209 deletions

diff --git a/searchlib/src/tests/stackdumpiterator/stackdumpiteratortest.cpp b/searchlib/src/tests/stackdumpiterator/stackdumpiteratortest.cpp
index 37a63828667..479ec67813d 100644
--- a/searchlib/src/tests/stackdumpiterator/stackdumpiteratortest.cpp
+++ b/searchlib/src/tests/stackdumpiterator/stackdumpiteratortest.cpp
@@ -5,7 +5,7 @@
 #include <vespa/searchlib/parsequery/stackdumpiterator.h>
 #include <vespa/fastos/time.h>
 
-#define NUMTESTS 5
+#define NUMTESTS 6
 
 int
 StackDumpIteratorTest::Main()
@@ -134,6 +134,10 @@ StackDumpIteratorTest::ShowResult(int testNo,
                                   search::SimpleQueryStack &correct,
                                   unsigned int expected)
 {
+    if (expected == ITERATOR_NOERROR) {
+        vespalib::string query = search::SimpleQueryStack::StackbufToString(actual.getStack());
+        printf("query = %s\n", query.c_str());
+    }
     unsigned int results = 0;
 
     int num = 0;
@@ -282,6 +286,20 @@ StackDumpIteratorTest::RunTest(int testno, bool verify)
         break;
     }
 
+    case 5: {
+        stack.Push(new search::ParseItem(search::ParseItem::ITEM_TERM, "foo", "foobar"));
+        stack.Push(new search::ParseItem(search::ParseItem::ITEM_TERM, "foo1", "foobar1"));
+        stack.Push(new search::ParseItem(search::ParseItem::ITEM_TERM, "foo2", "foobar2"));
+        stack.Push(new search::ParseItem(search::ParseItem::ITEM_WORD_ALTERNATIVES, 3, "wa"));
+
+        stack.AppendBuffer(&buf);
+        search::SimpleQueryStackDumpIterator si(vespalib::stringref(buf.GetDrainPos(), buf.GetUsedLen()));
+
+        if (verify)
+            return ShowResult(testno, si, stack, ITERATOR_NOERROR);
+        break;
+    }
+
 
     default:
     {
diff --git a/searchlib/src/vespa/searchlib/parsequery/parse.cpp b/searchlib/src/vespa/searchlib/parsequery/parse.cpp
index cc6fd6ee915..0911459655c 100644
--- a/searchlib/src/vespa/searchlib/parsequery/parse.cpp
+++ b/searchlib/src/vespa/searchlib/parsequery/parse.cpp
@@ -34,7 +34,7 @@ ParseItem::ParseItem(ItemType type, int arity, const char *idx)
     : PARSEITEM_DEFAULT_CONSTRUCTOR_LIST
 {
     assert(type==ITEM_PHRASE || type==ITEM_SAME_ELEMENT || type==ITEM_WEIGHTED_SET
-           || type==ITEM_DOT_PRODUCT || type==ITEM_WAND);
+           || type==ITEM_DOT_PRODUCT || type==ITEM_WAND || type==ITEM_WORD_ALTERNATIVES);
     SetType(type);
     _arity = arity;
     SetIndex(idx);
@@ -120,6 +120,13 @@ ParseItem::AppendBuffer(RawBuf *buf) const
             buf->append(_indexName.c_str(), indexLen);
         }
         break;
+    case ITEM_WORD_ALTERNATIVES:
+        buf->appendCompressedPositiveNumber(indexLen);
+        if (indexLen != 0) {
+            buf->append(_indexName.c_str(), indexLen);
+        }
+        buf->appendCompressedPositiveNumber(_arity);
+        break;
     case ITEM_WEAK_AND:
         buf->appendCompressedPositiveNumber(_arity);
         buf->appendCompressedPositiveNumber(_arg1);
@@ -167,72 +174,4 @@ ParseItem::AppendBuffer(RawBuf *buf) const
     }
 }
 
-size_t
-ParseItem::GetBufferLen() const
-{
-    // Calculate the length of the buffer.
-    uint32_t indexLen = _indexName.size();
-    uint32_t termLen = _term.size();
-
-    uint32_t len = sizeof(uint8_t); // type field
-    if (Feature_Weight()) {
-        len += sizeof(uint32_t);
-    }
-    if (feature_UniqueId()) {
-        len += sizeof(uint32_t);
-    }
-    if (feature_Flags()) {
-        len += sizeof(uint8_t);
-    }
-
-    // Put the values into the buffer.
-    switch (Type()) {
-    case ITEM_OR:
-    case ITEM_EQUIV:
-    case ITEM_AND:
-    case ITEM_NOT:
-    case ITEM_RANK:
-    case ITEM_ANY:
-        len += sizeof(uint32_t);
-        break;
-    case ITEM_NEAR:
-    case ITEM_ONEAR:
-        len += sizeof(uint32_t) * 2;
-        break;
-    case ITEM_WEAK_AND:
-        len += sizeof(uint32_t) * 3 + indexLen;
-        break;
-    case ITEM_WEIGHTED_SET:
-    case ITEM_DOT_PRODUCT:
-    case ITEM_PHRASE:
-        len += sizeof(uint32_t) * 2 + indexLen;
-        break;
-    case ITEM_SAME_ELEMENT:
-        len += sizeof(uint32_t) * 2 + indexLen;
-        break;
-    case ITEM_WAND:
-        len += sizeof(uint32_t) * 4 + indexLen;
-        break;
-    case ITEM_TERM:
-    case ITEM_NUMTERM:
-    case ITEM_PREFIXTERM:
-    case ITEM_SUBSTRINGTERM:
-    case ITEM_EXACTSTRINGTERM:
-    case ITEM_SUFFIXTERM:
-    case ITEM_REGEXP:
-        len += sizeof(uint32_t) * 2 + indexLen + termLen;
-        break;
-    case ITEM_PURE_WEIGHTED_STRING:
-        len += sizeof(uint32_t) + termLen;
-        break;
-    case ITEM_PURE_WEIGHTED_LONG:
-        len += sizeof(uint64_t);
-        break;
-    case ITEM_UNDEF:
-    default:
-        break;
-    }
-    return len;
-}
-
 }
diff --git a/searchlib/src/vespa/searchlib/parsequery/simplequerystack.cpp b/searchlib/src/vespa/searchlib/parsequery/simplequerystack.cpp
index 9f6ca50e326..54c1073d33e 100644
--- a/searchlib/src/vespa/searchlib/parsequery/simplequerystack.cpp
+++ b/searchlib/src/vespa/searchlib/parsequery/simplequerystack.cpp
@@ -14,7 +14,7 @@ namespace search {
 
 SimpleQueryStack::SimpleQueryStack()
     : _numItems(0),
-      _stack(NULL),
+      _stack(nullptr),
       _FP_queryOK(true)
 {
 }
@@ -44,10 +44,10 @@ ParseItem *
 SimpleQueryStack::Pop()
 {
     ParseItem *item = _stack;
-    if (_stack != NULL) {
+    if (_stack != nullptr) {
         _numItems--;
         _stack = _stack->_next;
-        item->_next = NULL;
+        item->_next = nullptr;
     }
     return item;
 }
@@ -55,24 +55,11 @@ SimpleQueryStack::Pop()
 void
 SimpleQueryStack::AppendBuffer(RawBuf *buf) const
 {
-    for (ParseItem *item = _stack; item != NULL; item = item->_next) {
+    for (ParseItem *item = _stack; item != nullptr; item = item->_next) {
         item->AppendBuffer(buf);
     }
 }
 
-size_t
-SimpleQueryStack::GetBufferLen() const
-{
-    size_t result;
-
-    result = 0;
-    for (const ParseItem *item = _stack;
-         item != NULL; item = item->_next) {
-        result += item->GetBufferLen();
-    }
-
-    return result;
-}
 
 uint32_t
 SimpleQueryStack::GetSize()
@@ -114,6 +101,7 @@ public:
         _name[ParseItem::ITEM_WAND] = 'A';
         _name[ParseItem::ITEM_PREDICATE_QUERY] = 'P';
         _name[ParseItem::ITEM_REGEXP] = '^';
+        _name[ParseItem::ITEM_WORD_ALTERNATIVES] = 'a';
     }
     char operator[] (ParseItem::ItemType i) const { return _name[i]; }
     char operator[] (size_t i) const { return _name[i]; }
@@ -194,133 +182,142 @@ SimpleQueryStack::StackbufToString(vespalib::stringref theBuf)
         result.append(metaStr);
 
         switch (type) {
-        case ParseItem::ITEM_OR:
-        case ParseItem::ITEM_AND:
-        case ParseItem::ITEM_EQUIV:
-        case ParseItem::ITEM_NOT:
-        case ParseItem::ITEM_RANK:
-        case ParseItem::ITEM_ANY:
-            p += vespalib::compress::Integer::decompressPositive(tmp, p);
-            arity = tmp;
-            result.append(make_string("%c/%d~", _G_ItemName[type], arity));
-            break;
-        case ParseItem::ITEM_NEAR:
-        case ParseItem::ITEM_ONEAR:
-            p += vespalib::compress::Integer::decompressPositive(tmp, p);
-            arity = tmp;
-            p += vespalib::compress::Integer::decompressPositive(tmp, p);
-            arg1 = tmp;
-            result.append(make_string("%c/%d/%d~", _G_ItemName[type], arity, arg1));
-            break;
-        case ParseItem::ITEM_WEAK_AND:
-            p += vespalib::compress::Integer::decompressPositive(tmp, p);
-            arity = tmp;
-            p += vespalib::compress::Integer::decompressPositive(tmp, p);
-            arg1 = tmp;
-            p += vespalib::compress::Integer::decompressPositive(tmp, p);
-            idxRefLen = tmp;
-            idxRef = p;
-            p += idxRefLen;
-            result.append(make_string("%c/%d/%d/%d:%.*s~", _G_ItemName[type], arity, arg1, idxRefLen, idxRefLen, idxRef));
-            break;
-        case ParseItem::ITEM_SAME_ELEMENT:
-            p += vespalib::compress::Integer::decompressPositive(tmp, p);
-            arity = tmp;
-            p += vespalib::compress::Integer::decompressPositive(tmp, p);
-            idxRefLen = tmp;
-            idxRef = p;
-            p += idxRefLen;
-            result.append(make_string("%c/%d/%d:%.*s~", _G_ItemName[type], arity, idxRefLen, idxRefLen, idxRef));
-            break;
-
-        case ParseItem::ITEM_NUMTERM:
-        case ParseItem::ITEM_TERM:
-        case ParseItem::ITEM_PREFIXTERM:
-        case ParseItem::ITEM_SUBSTRINGTERM:
-        case ParseItem::ITEM_EXACTSTRINGTERM:
-        case ParseItem::ITEM_SUFFIXTERM:
-        case ParseItem::ITEM_REGEXP:
-            p += vespalib::compress::Integer::decompressPositive(tmp, p);
-            idxRefLen = tmp;
-            idxRef = p;
-            p += idxRefLen;
-            p += vespalib::compress::Integer::decompressPositive(tmp, p);
-            termRefLen = tmp;
-            termRef = p;
-            p += termRefLen;
-            result.append(make_string("%c/%d:%.*s/%d:%.*s~", _G_ItemName[type],
-                                      idxRefLen, idxRefLen, idxRef, termRefLen, termRefLen, termRef));
-            break;
-        case ParseItem::ITEM_PURE_WEIGHTED_STRING:
-            p += vespalib::compress::Integer::decompressPositive(tmp, p);
-            termRefLen = tmp;
-            termRef = p;
-            p += termRefLen;
-            result.append(make_string("%c/%d:%.*s~", _G_ItemName[type], termRefLen, termRefLen, termRef));
-            break;
-
-        case ParseItem::ITEM_PURE_WEIGHTED_LONG:
-            tmpI64 = vespalib::nbo::n2h(*reinterpret_cast<const int64_t *>(p));
-            p += sizeof(uint64_t);
-            result.append(make_string("%c/%" PRId64, _G_ItemName[type], tmpI64));
-            break;
-
-        case ParseItem::ITEM_PHRASE:
-        case ParseItem::ITEM_WEIGHTED_SET:
-        case ParseItem::ITEM_DOT_PRODUCT:
-        case ParseItem::ITEM_WAND:
-            p += vespalib::compress::Integer::decompressPositive(tmp, p);
-            arity = tmp;
-            p += vespalib::compress::Integer::decompressPositive(tmp, p);
-            idxRefLen = tmp;
-            idxRef = p;
-            p += idxRefLen;
-            if (type == ParseItem::ITEM_WAND) {
+            case ParseItem::ITEM_OR:
+            case ParseItem::ITEM_AND:
+            case ParseItem::ITEM_EQUIV:
+            case ParseItem::ITEM_NOT:
+            case ParseItem::ITEM_RANK:
+            case ParseItem::ITEM_ANY:
                 p += vespalib::compress::Integer::decompressPositive(tmp, p);
-                uint32_t targetNumHits = tmp;
-                double scoreThreshold = vespalib::nbo::n2h(*reinterpret_cast<const double *>(p)); 
-                p += sizeof(double);
-                double thresholdBoostFactor = vespalib::nbo::n2h(*reinterpret_cast<const double *>(p));
-                p += sizeof(double);
-                result.append(make_string("%c/%d/%d:%.*s(%u,%f,%f)~", _G_ItemName[type], arity, idxRefLen,
-                                          idxRefLen, idxRef, targetNumHits, scoreThreshold, thresholdBoostFactor));
-            } else {
+                arity = tmp;
+                result.append(make_string("%c/%d~", _G_ItemName[type], arity));
+                break;
+            case ParseItem::ITEM_NEAR:
+            case ParseItem::ITEM_ONEAR:
+                p += vespalib::compress::Integer::decompressPositive(tmp, p);
+                arity = tmp;
+                p += vespalib::compress::Integer::decompressPositive(tmp, p);
+                arg1 = tmp;
+                result.append(make_string("%c/%d/%d~", _G_ItemName[type], arity, arg1));
+                break;
+            case ParseItem::ITEM_WEAK_AND:
+                p += vespalib::compress::Integer::decompressPositive(tmp, p);
+                arity = tmp;
+                p += vespalib::compress::Integer::decompressPositive(tmp, p);
+                arg1 = tmp;
+                p += vespalib::compress::Integer::decompressPositive(tmp, p);
+                idxRefLen = tmp;
+                idxRef = p;
+                p += idxRefLen;
+                result.append(
+                        make_string("%c/%d/%d/%d:%.*s~", _G_ItemName[type], arity, arg1, idxRefLen, idxRefLen, idxRef));
+                break;
+            case ParseItem::ITEM_SAME_ELEMENT:
+                p += vespalib::compress::Integer::decompressPositive(tmp, p);
+                arity = tmp;
+                p += vespalib::compress::Integer::decompressPositive(tmp, p);
+                idxRefLen = tmp;
+                idxRef = p;
+                p += idxRefLen;
                 result.append(make_string("%c/%d/%d:%.*s~", _G_ItemName[type], arity, idxRefLen, idxRefLen, idxRef));
-            }
-            break;
-
-        case ParseItem::ITEM_PREDICATE_QUERY:
-        {
-            idxRefLen = static_cast<uint32_t>(ReadCompressedPositiveInt(p));
-            idxRef = p;
-            p += idxRefLen;
-            size_t feature_count = ReadCompressedPositiveInt(p);
-            result.append(make_string("%c/%d:%.*s/%zu(", _G_ItemName[type], idxRefLen, idxRefLen, idxRef, feature_count));
-            for (size_t i = 0; i < feature_count; ++i) {
-                vespalib::string key = ReadString(p);
-                vespalib::string value = ReadString(p);
-                uint64_t sub_queries = ReadUint64(p);
-                result.append(make_string("%s:%s:%lx", key.c_str(), value.c_str(), sub_queries));
-                if (i < feature_count - 1) {
-                    result.append(',');
+                break;
+
+            case ParseItem::ITEM_NUMTERM:
+            case ParseItem::ITEM_TERM:
+            case ParseItem::ITEM_PREFIXTERM:
+            case ParseItem::ITEM_SUBSTRINGTERM:
+            case ParseItem::ITEM_EXACTSTRINGTERM:
+            case ParseItem::ITEM_SUFFIXTERM:
+            case ParseItem::ITEM_REGEXP:
+                p += vespalib::compress::Integer::decompressPositive(tmp, p);
+                idxRefLen = tmp;
+                idxRef = p;
+                p += idxRefLen;
+                p += vespalib::compress::Integer::decompressPositive(tmp, p);
+                termRefLen = tmp;
+                termRef = p;
+                p += termRefLen;
+                result.append(make_string("%c/%d:%.*s/%d:%.*s~", _G_ItemName[type],
+                                          idxRefLen, idxRefLen, idxRef, termRefLen, termRefLen, termRef));
+                break;
+            case ParseItem::ITEM_PURE_WEIGHTED_STRING:
+                p += vespalib::compress::Integer::decompressPositive(tmp, p);
+                termRefLen = tmp;
+                termRef = p;
+                p += termRefLen;
+                result.append(make_string("%c/%d:%.*s~", _G_ItemName[type], termRefLen, termRefLen, termRef));
+                break;
+
+            case ParseItem::ITEM_PURE_WEIGHTED_LONG:
+                tmpI64 = vespalib::nbo::n2h(*reinterpret_cast<const int64_t *>(p));
+                p += sizeof(uint64_t);
+                result.append(make_string("%c/%" PRId64, _G_ItemName[type], tmpI64));
+                break;
+
+            case ParseItem::ITEM_PHRASE:
+            case ParseItem::ITEM_WEIGHTED_SET:
+            case ParseItem::ITEM_DOT_PRODUCT:
+            case ParseItem::ITEM_WAND:
+                p += vespalib::compress::Integer::decompressPositive(tmp, p);
+                arity = tmp;
+                p += vespalib::compress::Integer::decompressPositive(tmp, p);
+                idxRefLen = tmp;
+                idxRef = p;
+                p += idxRefLen;
+                if (type == ParseItem::ITEM_WAND) {
+                    p += vespalib::compress::Integer::decompressPositive(tmp, p);
+                    uint32_t targetNumHits = tmp;
+                    double scoreThreshold = vespalib::nbo::n2h(*reinterpret_cast<const double *>(p));
+                    p += sizeof(double);
+                    double thresholdBoostFactor = vespalib::nbo::n2h(*reinterpret_cast<const double *>(p));
+                    p += sizeof(double);
+                    result.append(make_string("%c/%d/%d:%.*s(%u,%f,%f)~", _G_ItemName[type], arity, idxRefLen,
+                                              idxRefLen, idxRef, targetNumHits, scoreThreshold, thresholdBoostFactor));
+                } else {
+                    result.append(
+                            make_string("%c/%d/%d:%.*s~", _G_ItemName[type], arity, idxRefLen, idxRefLen, idxRef));
+                }
+                break;
+
+            case ParseItem::ITEM_PREDICATE_QUERY: {
+                idxRefLen = static_cast<uint32_t>(ReadCompressedPositiveInt(p));
+                idxRef = p;
+                p += idxRefLen;
+                size_t feature_count = ReadCompressedPositiveInt(p);
+                result.append(
+                        make_string("%c/%d:%.*s/%zu(", _G_ItemName[type], idxRefLen, idxRefLen, idxRef, feature_count));
+                for (size_t i = 0; i < feature_count; ++i) {
+                    vespalib::string key = ReadString(p);
+                    vespalib::string value = ReadString(p);
+                    uint64_t sub_queries = ReadUint64(p);
+                    result.append(make_string("%s:%s:%lx", key.c_str(), value.c_str(), sub_queries));
+                    if (i < feature_count - 1) {
+                        result.append(',');
+                    }
                 }
-            }
 
-            size_t range_feature_count = ReadCompressedPositiveInt(p);
-            result.append(make_string(")/%zu(", range_feature_count));
-            for (size_t i = 0; i < range_feature_count; ++i) {
-                vespalib::string key = ReadString(p);
-                uint64_t value = ReadUint64(p);
-                uint64_t sub_queries = ReadUint64(p);
-                result.append(make_string("%s:%zu:%lx", key.c_str(), value, sub_queries));
-                if (i < range_feature_count - 1) {
-                    result.append(',');
+                size_t range_feature_count = ReadCompressedPositiveInt(p);
+                result.append(make_string(")/%zu(", range_feature_count));
+                for (size_t i = 0; i < range_feature_count; ++i) {
+                    vespalib::string key = ReadString(p);
+                    uint64_t value = ReadUint64(p);
+                    uint64_t sub_queries = ReadUint64(p);
+                    result.append(make_string("%s:%zu:%lx", key.c_str(), value, sub_queries));
+                    if (i < range_feature_count - 1) {
+                        result.append(',');
+                    }
                 }
+                result.append(")~");
+                break;
+            }
+            case ParseItem::ITEM_WORD_ALTERNATIVES: {
+                idxRefLen = static_cast<uint32_t>(ReadCompressedPositiveInt(p));
+                idxRef = p;
+                p += idxRefLen;
+                arity = ReadCompressedPositiveInt(p);
+                result.append(make_string("%c/%d:%.*s/%d(", _G_ItemName[type], idxRefLen, idxRefLen, idxRef, arity));
+                break;
             }
-            result.append(")~");
-            break;
-        }
-
         default:
             LOG(error, "Unhandled type %d", type);
             LOG_ABORT("should not be reached");
diff --git a/searchlib/src/vespa/searchlib/parsequery/simplequerystack.h b/searchlib/src/vespa/searchlib/parsequery/simplequerystack.h
index 47bb213a8b3..27e85821cf1 100644
--- a/searchlib/src/vespa/searchlib/parsequery/simplequerystack.h
+++ b/searchlib/src/vespa/searchlib/parsequery/simplequerystack.h
@@ -69,7 +69,6 @@ public:
      */
     void AppendBuffer(search::RawBuf *buf) const;
 
-    size_t GetBufferLen() const;
     /**
      * Return the number of items on the stack.
      * @return The number of items on the stack.