1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
|
// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
#include "serialized_fast_value_attribute.h"
#include "streamed_value_saver.h"
#include <vespa/eval/eval/value.h>
#include <vespa/fastlib/io/bufferedfile.h>
#include <vespa/searchcommon/attribute/config.h>
#include <vespa/log/log.h>
LOG_SETUP(".searchlib.tensor.serialized_fast_value_attribute");
#include "blob_sequence_reader.h"
using namespace vespalib;
using namespace vespalib::eval;
namespace search::tensor {
SerializedFastValueAttribute::SerializedFastValueAttribute(stringref name, const Config &cfg)
: TensorAttribute(name, cfg, _tensorBufferStore),
_tensor_type(cfg.tensorType()),
_tensorBufferStore(_tensor_type, {}, 1000u)
{
}
SerializedFastValueAttribute::~SerializedFastValueAttribute()
{
getGenerationHolder().clearHoldLists();
_tensorStore.clearHoldLists();
}
void
SerializedFastValueAttribute::setTensor(DocId docId, const vespalib::eval::Value &tensor)
{
checkTensorType(tensor);
EntryRef ref = _tensorBufferStore.store_tensor(tensor);
assert(ref.valid());
setTensorRef(docId, ref);
}
std::unique_ptr<Value>
SerializedFastValueAttribute::getTensor(DocId docId) const
{
EntryRef ref;
if (docId < getCommittedDocIdLimit()) {
ref = acquire_entry_ref(docId);
}
return _tensorBufferStore.get_tensor(ref);
}
bool
SerializedFastValueAttribute::onLoad(vespalib::Executor *)
{
BlobSequenceReader tensorReader(*this);
if (!tensorReader.hasData()) {
return false;
}
setCreateSerialNum(tensorReader.getCreateSerialNum());
assert(tensorReader.getVersion() == getVersion());
uint32_t numDocs(tensorReader.getDocIdLimit());
_refVector.reset();
_refVector.unsafe_reserve(numDocs);
vespalib::Array<char> buffer(1024);
for (uint32_t lid = 0; lid < numDocs; ++lid) {
uint32_t tensorSize = tensorReader.getNextSize();
if (tensorSize != 0) {
if (tensorSize > buffer.size()) {
buffer.resize(tensorSize + 1024);
}
tensorReader.readBlob(&buffer[0], tensorSize);
vespalib::nbostream source(&buffer[0], tensorSize);
EntryRef ref = _tensorBufferStore.store_encoded_tensor(source);
_refVector.push_back(AtomicEntryRef(ref));
} else {
EntryRef invalid;
_refVector.push_back(AtomicEntryRef(invalid));
}
}
setNumDocs(numDocs);
setCommittedDocIdLimit(numDocs);
return true;
}
std::unique_ptr<AttributeSaver>
SerializedFastValueAttribute::onInitSave(vespalib::stringref fileName)
{
vespalib::GenerationHandler::Guard guard(getGenerationHandler().
takeGuard());
return std::make_unique<StreamedValueSaver>
(std::move(guard),
this->createAttributeHeader(fileName),
getRefCopy(),
_tensorBufferStore);
}
}
|
