Merge pull request #2075 from yahoo/toregge/flush-populated-attributes-to-disk

Flush populated attributes to disk as part of reprocessing.

4 files changed, 76 insertions, 32 deletions

diff --git a/searchcommon/src/vespa/searchcommon/attribute/config.cpp b/searchcommon/src/vespa/searchcommon/attribute/config.cpp
index ac7fe34e06b..e7a4c28b27a 100644
--- a/searchcommon/src/vespa/searchcommon/attribute/config.cpp
+++ b/searchcommon/src/vespa/searchcommon/attribute/config.cpp
@@ -16,13 +16,9 @@ Config::Config() :
     _enableOnlyBitVector(false),
     _isFilter(false),
     _fastAccess(false),
-    _maxInternalBlobSize(defaultMaxInternalBlobSize),
     _growStrategy(),
     _compactionStrategy(),
-    _arity(8),
-    _lower_bound(LLONG_MIN),
-    _upper_bound(LLONG_MAX),
-    _dense_posting_list_threshold(0.4),
+    _predicateParams(),
     _tensorType(vespalib::eval::ValueType::error_type())
 {
 }
@@ -39,13 +35,9 @@ Config::Config(BasicType bt,
       _enableOnlyBitVector(false),
       _isFilter(false),
       _fastAccess(false),
-      _maxInternalBlobSize(defaultMaxInternalBlobSize),
       _growStrategy(),
       _compactionStrategy(),
-      _arity(8),
-      _lower_bound(LLONG_MIN),
-      _upper_bound(LLONG_MAX),
-      _dense_posting_list_threshold(0.4),
+      _predicateParams(),
       _tensorType(vespalib::eval::ValueType::error_type())
 {
 }
diff --git a/searchcommon/src/vespa/searchcommon/attribute/config.h b/searchcommon/src/vespa/searchcommon/attribute/config.h
index d0b2535950f..be2a1d2777d 100644
--- a/searchcommon/src/vespa/searchcommon/attribute/config.h
+++ b/searchcommon/src/vespa/searchcommon/attribute/config.h
@@ -7,12 +7,11 @@
 #include <vespa/searchcommon/common/growstrategy.h>
 #include <vespa/searchcommon/common/compaction_strategy.h>
 #include <vespa/eval/eval/value_type.h>
+#include "predicate_params.h"
 
 namespace search {
 namespace attribute {
 
-const size_t defaultMaxInternalBlobSize = 0x400000000ul;
-
 class Config
 {
 public:
@@ -27,11 +26,7 @@ public:
     CollectionType collectionType()       const { return _type; }
     bool fastSearch()                     const { return _fastSearch; }
     bool huge()                           const { return _huge; }
-    size_t getMaxInternalBlobSize()       const { return _maxInternalBlobSize; }
-    uint32_t arity()                      const { return _arity; }
-    int64_t lower_bound()                 const { return _lower_bound; }
-    int64_t upper_bound()                 const { return _upper_bound; }
-    double dense_posting_list_threshold() const { return _dense_posting_list_threshold; }
+    const PredicateParams &predicateParams() const { return _predicateParams; }
     vespalib::eval::ValueType tensorType() const { return _tensorType; }
 
     /**
@@ -70,11 +65,7 @@ public:
     const CompactionStrategy &getCompactionStrategy() const { return _compactionStrategy; }
     void setHuge(bool v)                         { _huge = v; }
     void setFastSearch(bool v)                   { _fastSearch = v; }
-    void setMaxInternalBlobSize(size_t v)        { _maxInternalBlobSize = v; }
-    void setArity(uint32_t v)                    { _arity = v; }
-    void setBounds(int64_t lower, int64_t upper) { _lower_bound = lower;
-                                                   _upper_bound = upper; }
-    void setDensePostingListThreshold(double v)  { _dense_posting_list_threshold = v; }
+    void setPredicateParams(const PredicateParams &v) { _predicateParams = v; }
     void setTensorType(const vespalib::eval::ValueType &tensorType_in) {
         _tensorType = tensorType_in;
     }
@@ -126,13 +117,9 @@ public:
                _enableOnlyBitVector == b._enableOnlyBitVector &&
                _isFilter == b._isFilter &&
                _fastAccess == b._fastAccess &&
-               _maxInternalBlobSize == b._maxInternalBlobSize &&
                _growStrategy == b._growStrategy &&
                _compactionStrategy == b._compactionStrategy &&
-               _arity == b._arity &&
-               _lower_bound == b._lower_bound &&
-               _upper_bound == b._upper_bound &&
-               _dense_posting_list_threshold == b._dense_posting_list_threshold &&
+               _predicateParams == b._predicateParams &&
             (_basicType.type() != BasicType::Type::TENSOR ||
              _tensorType == b._tensorType);
     }
@@ -146,13 +133,9 @@ private:
     bool           _enableOnlyBitVector;
     bool           _isFilter;
     bool           _fastAccess;
-    size_t         _maxInternalBlobSize;
     GrowStrategy   _growStrategy;
     CompactionStrategy _compactionStrategy;
-    uint32_t       _arity;
-    int64_t        _lower_bound;
-    int64_t        _upper_bound;
-    double         _dense_posting_list_threshold;
+    PredicateParams    _predicateParams;
     vespalib::eval::ValueType _tensorType;
 };
 }  // namespace attribute
diff --git a/searchcommon/src/vespa/searchcommon/attribute/persistent_predicate_params.h b/searchcommon/src/vespa/searchcommon/attribute/persistent_predicate_params.h
new file mode 100644
index 00000000000..1aed2dad72c
--- /dev/null
+++ b/searchcommon/src/vespa/searchcommon/attribute/persistent_predicate_params.h
@@ -0,0 +1,37 @@
+// Copyright 2017 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#pragma once
+
+namespace search {
+namespace attribute {
+
+/*
+ * Persistent parameters for predicate attributes.
+ */
+class PersistentPredicateParams {
+    uint32_t _arity;
+    int64_t  _lower_bound;
+    int64_t  _upper_bound;
+
+public:
+    PersistentPredicateParams()
+        : _arity(8),
+          _lower_bound(std::numeric_limits<int64_t>::min()),
+          _upper_bound(std::numeric_limits<int64_t>::max())
+    {
+    }
+    uint32_t arity()                      const { return _arity; }
+    int64_t lower_bound()                 const { return _lower_bound; }
+    int64_t upper_bound()                 const { return _upper_bound; }
+    void setArity(uint32_t v)                    { _arity = v; }
+    void setBounds(int64_t lower, int64_t upper) { _lower_bound = lower; _upper_bound = upper; }
+
+    bool operator==(const PersistentPredicateParams &rhs) const {
+        return ((_arity == rhs._arity) &&
+                (_lower_bound == rhs._lower_bound) &&
+                (_upper_bound == rhs._upper_bound));
+    }
+};
+
+}  // namespace attribute
+}  // namespace search
diff --git a/searchcommon/src/vespa/searchcommon/attribute/predicate_params.h b/searchcommon/src/vespa/searchcommon/attribute/predicate_params.h
new file mode 100644
index 00000000000..8e8e0605b17
--- /dev/null
+++ b/searchcommon/src/vespa/searchcommon/attribute/predicate_params.h
@@ -0,0 +1,32 @@
+// Copyright 2017 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#pragma once
+
+#include "persistent_predicate_params.h"
+
+namespace search {
+namespace attribute {
+
+/*
+ * Parameters for predicate attributes.
+ */
+class PredicateParams : public PersistentPredicateParams
+{
+    double _dense_posting_list_threshold;
+public:
+    PredicateParams()
+        : PersistentPredicateParams(),
+          _dense_posting_list_threshold(0.4)
+    {
+    }
+
+    double dense_posting_list_threshold() const { return _dense_posting_list_threshold; }
+    void setDensePostingListThreshold(double v)  { _dense_posting_list_threshold = v; }
+    bool operator==(const PredicateParams &rhs) const {
+        return (PersistentPredicateParams::operator==(rhs) &&
+                (_dense_posting_list_threshold == rhs._dense_posting_list_threshold));
+    }
+};
+
+}  // namespace attribute
+}  // namespace search