diff options
Diffstat (limited to 'searchlib')
15 files changed, 156 insertions, 46 deletions
diff --git a/searchlib/CMakeLists.txt b/searchlib/CMakeLists.txt index bb07dc90459..bc342c594d1 100644 --- a/searchlib/CMakeLists.txt +++ b/searchlib/CMakeLists.txt @@ -75,6 +75,7 @@ vespa_define_module( src/tests/attribute/benchmark src/tests/attribute/bitvector src/tests/attribute/changevector + src/tests/attribute/compaction src/tests/attribute/comparator src/tests/attribute/document_weight_iterator src/tests/attribute/enumeratedsave diff --git a/searchlib/src/tests/attribute/attribute_test.cpp b/searchlib/src/tests/attribute/attribute_test.cpp index 3d3aea0baf8..83e99f0a831 100644 --- a/searchlib/src/tests/attribute/attribute_test.cpp +++ b/searchlib/src/tests/attribute/attribute_test.cpp @@ -2,6 +2,8 @@ #include <vespa/fastos/fastos.h> #include <vespa/document/fieldvalue/intfieldvalue.h> #include <vespa/document/fieldvalue/stringfieldvalue.h> +#include <vespa/document/update/arithmeticvalueupdate.h> +#include <vespa/document/update/mapvalueupdate.h> #include <vespa/searchlib/attribute/attribute.h> #include <vespa/searchlib/attribute/attributefile.h> #include <vespa/searchlib/attribute/attributeguard.h> diff --git a/searchlib/src/tests/attribute/compaction/CMakeLists.txt b/searchlib/src/tests/attribute/compaction/CMakeLists.txt new file mode 100644 index 00000000000..994d12761d1 --- /dev/null +++ b/searchlib/src/tests/attribute/compaction/CMakeLists.txt @@ -0,0 +1,8 @@ +# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +vespa_add_executable(searchlib_attribute_compaction_test_app TEST + SOURCES + attribute_compaction_test.cpp + DEPENDS + searchlib +) +vespa_add_test(NAME searchlib_attribute_compaction_test_app COMMAND searchlib_attribute_compaction_test_app) diff --git a/searchlib/src/tests/attribute/compaction/DESC b/searchlib/src/tests/attribute/compaction/DESC new file mode 100644 index 00000000000..dfed48a02fb --- /dev/null +++ b/searchlib/src/tests/attribute/compaction/DESC @@ -0,0 +1 @@ +Unit tests for attribute vector compaction. diff --git a/searchlib/src/tests/attribute/compaction/FILES b/searchlib/src/tests/attribute/compaction/FILES new file mode 100644 index 00000000000..7dbc2016f5c --- /dev/null +++ b/searchlib/src/tests/attribute/compaction/FILES @@ -0,0 +1 @@ +atttribute_compaction_test.cpp diff --git a/searchlib/src/tests/attribute/compaction/attribute_compaction_test.cpp b/searchlib/src/tests/attribute/compaction/attribute_compaction_test.cpp new file mode 100644 index 00000000000..7ba290c967d --- /dev/null +++ b/searchlib/src/tests/attribute/compaction/attribute_compaction_test.cpp @@ -0,0 +1,97 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include <vespa/fastos/fastos.h> +#include <vespa/vespalib/testkit/test_kit.h> +#include <vespa/searchlib/attribute/attribute.h> +#include <vespa/searchlib/attribute/attributefactory.h> +#include <vespa/searchlib/attribute/attributevector.hpp> +#include <vespa/searchlib/attribute/integerbase.h> + +#include <vespa/log/log.h> +LOG_SETUP("attribute_compaction_test"); + +using search::IntegerAttribute; +using search::AttributeVector; +using search::attribute::Config; +using search::attribute::BasicType; +using search::attribute::CollectionType; + +using AttributePtr = AttributeVector::SP; +using AttributeStatus = search::attribute::Status; + +namespace +{ + +template <typename VectorType> +bool is(AttributePtr &v) +{ + return dynamic_cast<VectorType *>(v.get()); +} + +template <typename VectorType> +VectorType &as(AttributePtr &v) +{ + return dynamic_cast<VectorType &>(*v); +} + +void populateAttribute(IntegerAttribute &v, uint32_t docIdLimit) +{ + for(uint32_t docId = 0; docId < docIdLimit; ++docId) { + uint32_t checkDocId = 0; + EXPECT_TRUE(v.addDoc(checkDocId)); + EXPECT_EQUAL(docId, checkDocId); + v.clearDoc(docId); + for (size_t vi = 0; vi <= 40; ++vi) { + EXPECT_TRUE(v.append(docId, 42, 1) ); + } + } + v.commit(true); + v.incGeneration(); +} + +void populateAttribute(AttributePtr &v, uint32_t docIdLimit) +{ + if (is<IntegerAttribute>(v)) { + populateAttribute(as<IntegerAttribute>(v), docIdLimit); + } +} + +void cleanAttribute(AttributeVector &v, uint32_t docIdLimit) +{ + for (uint32_t docId = 0; docId < docIdLimit; ++docId) { + v.clearDoc(docId); + } + v.commit(true); + v.incGeneration(); +} + +} + +class Fixture { +public: + AttributePtr _v; + + Fixture(Config cfg) + : _v() + { _v = search::AttributeFactory::createAttribute("test", cfg); } + ~Fixture() { } + void populate(uint32_t docIdLimit) { populateAttribute(_v, docIdLimit); } + void clean(uint32_t docIdLimit) { cleanAttribute(*_v, docIdLimit); } + AttributeStatus getStatus() { _v->commit(true); return _v->getStatus(); } + AttributeStatus getStatus(const vespalib::string &prefix) { + AttributeStatus status(getStatus()); + LOG(info, "status %s: used=%zu, dead=%zu, onHold=%zu", + prefix.c_str(), status.getUsed(), status.getDead(), status.getOnHold()); + return status; + } +}; + +TEST_F("Test that compaction of integer array attribute reduces memory usage", Fixture({ BasicType::INT64, CollectionType::ARRAY })) +{ + f.populate(3000); + AttributeStatus beforeStatus = f.getStatus("before"); + f.clean(2000); + AttributeStatus afterStatus = f.getStatus("after"); + EXPECT_LESS(afterStatus.getUsed(), beforeStatus.getUsed()); +} + +TEST_MAIN() { TEST_RUN_ALL(); } diff --git a/searchlib/src/vespa/searchlib/attribute/attributevector.cpp b/searchlib/src/vespa/searchlib/attribute/attributevector.cpp index 00aaf73c5df..f5f3257c2eb 100644 --- a/searchlib/src/vespa/searchlib/attribute/attributevector.cpp +++ b/searchlib/src/vespa/searchlib/attribute/attributevector.cpp @@ -18,12 +18,14 @@ #include <vespa/vespalib/data/fileheader.h> #include <functional> #include <stdexcept> -#include <vespa/log/log.h> #include "ipostinglistsearchcontext.h" #include "ipostinglistattributebase.h" #include <vespa/searchlib/queryeval/emptysearch.h> #include "interlock.h" #include "attributesaver.h" +#include <vespa/document/update/mapvalueupdate.h> + +#include <vespa/log/log.h> LOG_SETUP(".searchlib.attribute.attributevector"); using vespalib::getLastErrorString; diff --git a/searchlib/src/vespa/searchlib/attribute/attributevector.h b/searchlib/src/vespa/searchlib/attribute/attributevector.h index b5f7731f1a1..631b9e11932 100644 --- a/searchlib/src/vespa/searchlib/attribute/attributevector.h +++ b/searchlib/src/vespa/searchlib/attribute/attributevector.h @@ -4,8 +4,6 @@ #include "address_space_usage.h" #include "iattributesavetarget.h" -#include <vespa/document/update/arithmeticvalueupdate.h> -#include <vespa/document/update/mapvalueupdate.h> #include <vespa/fastlib/io/bufferedfile.h> #include <vespa/fastlib/text/normwordfolder.h> #include <vespa/fastos/fastos.h> @@ -30,47 +28,46 @@ #include <shared_mutex> #include <string> -using document::ArithmeticValueUpdate; -using document::MapValueUpdate; -using document::FieldValue; - -namespace vespalib -{ - -class GenericHeader; +namespace document { + class ArithmeticValueUpdate; + class MapValueUpdate; + class FieldValue; +} +namespace vespalib { + class GenericHeader; } namespace search { -template <typename T> class ComponentGuard; -class AttributeReadGuard; -class AttributeWriteGuard; -class AttributeSaver; -class EnumStoreBase; + template <typename T> class ComponentGuard; + class AttributeReadGuard; + class AttributeWriteGuard; + class AttributeSaver; + class EnumStoreBase; + class IDocumentWeightAttribute; -class IDocumentWeightAttribute; + namespace fef { + class TermFieldMatchData; + } -namespace fef { -class TermFieldMatchData; + namespace attribute { + class IPostingListSearchContext; + class IPostingListAttributeBase; + class Interlock; + class InterlockGuard; + class MultiValueMapping2Base; + } } -namespace attribute -{ - -class IPostingListSearchContext; - -class IPostingListAttributeBase; - -class Interlock; -class InterlockGuard; -class MultiValueMapping2Base; - -} +namespace search { using search::attribute::WeightedType; using search::attribute::Status; +using document::ArithmeticValueUpdate; +using document::MapValueUpdate; +using document::FieldValue; template <typename T> class UnWeightedType diff --git a/searchlib/src/vespa/searchlib/attribute/attributevector.hpp b/searchlib/src/vespa/searchlib/attribute/attributevector.hpp index 20edc0826ad..7c8b1004731 100644 --- a/searchlib/src/vespa/searchlib/attribute/attributevector.hpp +++ b/searchlib/src/vespa/searchlib/attribute/attributevector.hpp @@ -3,6 +3,7 @@ #include <vespa/searchlib/attribute/attributevector.h> #include <vespa/searchlib/attribute/integerbase.h> +#include <vespa/document/update/arithmeticvalueupdate.h> #include <cmath> namespace search { diff --git a/searchlib/src/vespa/searchlib/attribute/floatbase.cpp b/searchlib/src/vespa/searchlib/attribute/floatbase.cpp index 8532302a70c..daf07a1879c 100644 --- a/searchlib/src/vespa/searchlib/attribute/floatbase.cpp +++ b/searchlib/src/vespa/searchlib/attribute/floatbase.cpp @@ -1,14 +1,13 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include <vespa/fastos/fastos.h> -#include <vespa/log/log.h> #include "floatbase.h" #include <vespa/searchlib/common/sort.h> - -LOG_SETUP(".searchlib.attribute.floatbase"); - #include <vespa/searchlib/attribute/attributevector.hpp> +#include <vespa/document/fieldvalue/fieldvalue.h> +#include <vespa/log/log.h> +LOG_SETUP(".searchlib.attribute.floatbase"); namespace search { diff --git a/searchlib/src/vespa/searchlib/attribute/integerbase.cpp b/searchlib/src/vespa/searchlib/attribute/integerbase.cpp index af49b222b04..46ecf789ae5 100644 --- a/searchlib/src/vespa/searchlib/attribute/integerbase.cpp +++ b/searchlib/src/vespa/searchlib/attribute/integerbase.cpp @@ -3,12 +3,12 @@ #include <vespa/fastos/fastos.h> #include "integerbase.h" #include <vespa/searchlib/common/sort.h> -#include <vespa/log/log.h> +#include <vespa/searchlib/attribute/attributevector.hpp> +#include <vespa/document/fieldvalue/fieldvalue.h> +#include <vespa/log/log.h> LOG_SETUP(".searchlib.attribute.integerbase"); -#include <vespa/searchlib/attribute/attributevector.hpp> - namespace search { IMPLEMENT_IDENTIFIABLE_ABSTRACT(IntegerAttribute, NumericAttribute); diff --git a/searchlib/src/vespa/searchlib/attribute/stringbase.cpp b/searchlib/src/vespa/searchlib/attribute/stringbase.cpp index 6024a68fa33..d4ad5d84004 100644 --- a/searchlib/src/vespa/searchlib/attribute/stringbase.cpp +++ b/searchlib/src/vespa/searchlib/attribute/stringbase.cpp @@ -2,18 +2,17 @@ #include <vespa/fastos/fastos.h> #include "stringbase.h" -#include <vespa/log/log.h> #include <vespa/vespalib/util/array.h> #include <vespa/vespalib/text/utf8.h> #include <vespa/vespalib/text/lowercase.h> #include <vespa/searchlib/common/sort.h> +#include <vespa/searchlib/attribute/attributevector.hpp> +#include <vespa/document/fieldvalue/fieldvalue.h> +#include <vespa/log/log.h> LOG_SETUP(".searchlib.attribute.stringbase"); -#include <vespa/searchlib/attribute/attributevector.hpp> - -namespace search -{ +namespace search { IMPLEMENT_IDENTIFIABLE_ABSTRACT(StringAttribute, AttributeVector); diff --git a/searchlib/src/vespa/searchlib/queryeval/fake_requestcontext.h b/searchlib/src/vespa/searchlib/queryeval/fake_requestcontext.h index a0dee812b51..f204710c011 100644 --- a/searchlib/src/vespa/searchlib/queryeval/fake_requestcontext.h +++ b/searchlib/src/vespa/searchlib/queryeval/fake_requestcontext.h @@ -4,6 +4,7 @@ #include <vespa/searchlib/queryeval/irequestcontext.h> #include <vespa/searchcommon/attribute/iattributecontext.h> +#include <vespa/searchlib/attribute/attributevector.h> #include <limits> namespace search { diff --git a/searchlib/src/vespa/searchlib/queryeval/irequestcontext.h b/searchlib/src/vespa/searchlib/queryeval/irequestcontext.h index 3ebfb4f148c..ee2d9cdbac7 100644 --- a/searchlib/src/vespa/searchlib/queryeval/irequestcontext.h +++ b/searchlib/src/vespa/searchlib/queryeval/irequestcontext.h @@ -3,9 +3,12 @@ #pragma once #include <vespa/vespalib/util/doom.h> -#include <vespa/searchlib/attribute/attributevector.h> +#include <vespa/vespalib/stllike/string.h> namespace search { + +class AttributeVector; + namespace queryeval { /** diff --git a/searchlib/src/vespa/searchlib/queryeval/searchable.cpp b/searchlib/src/vespa/searchlib/queryeval/searchable.cpp index be1487eb7fd..26c8ede1eb1 100644 --- a/searchlib/src/vespa/searchlib/queryeval/searchable.cpp +++ b/searchlib/src/vespa/searchlib/queryeval/searchable.cpp @@ -1,8 +1,6 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include <vespa/fastos/fastos.h> -#include <vespa/log/log.h> -LOG_SETUP(".searchable"); #include "searchable.h" #include "leaf_blueprints.h" #include "intermediate_blueprints.h" |