diff options
12 files changed, 137 insertions, 74 deletions
diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionZooKeeperClient.java b/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionZooKeeperClient.java index 4bbb8a63974..120eb967a73 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionZooKeeperClient.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionZooKeeperClient.java @@ -132,17 +132,12 @@ public class SessionZooKeeperClient { return sessionPath.append(barrierName); } - /** Returns the number of node members needed in a barrier */ - private int getNumberOfMembers() { - return (curator.zooKeeperEnsembleCount() / 2) + 1; // majority - } - private CompletionWaiter createCompletionWaiter(String waiterNode) { - return curator.createCompletionWaiter(sessionPath, waiterNode, getNumberOfMembers(), serverId); + return curator.createCompletionWaiter(sessionPath, waiterNode, serverId); } private CompletionWaiter getCompletionWaiter(Path path) { - return curator.getCompletionWaiter(path, getNumberOfMembers(), serverId); + return curator.getCompletionWaiter(path, serverId); } /** Returns a transaction deleting this session on commit */ diff --git a/model-evaluation/src/main/java/ai/vespa/models/evaluation/Model.java b/model-evaluation/src/main/java/ai/vespa/models/evaluation/Model.java index 66612b7ccc3..e0be208bf31 100644 --- a/model-evaluation/src/main/java/ai/vespa/models/evaluation/Model.java +++ b/model-evaluation/src/main/java/ai/vespa/models/evaluation/Model.java @@ -95,9 +95,11 @@ public class Model implements AutoCloseable { logger.fine("Replacing " + node + " => " + expr); node = expr; for (var inputSpec : m.inputSpecs) { - var old = declaredTypes.put(inputSpec.source, inputSpec.wantedType); - if (old != null && ! old.equals(inputSpec.wantedType)) { - throw new IllegalArgumentException("Conflicting types needed for " + inputSpec.source + "; " + old + " != " + inputSpec.wantedType); + var old = declaredTypes.get(inputSpec.source); + if (old == null) { + declaredTypes.put(inputSpec.source, inputSpec.wantedType); + } else if (! old.isAssignableTo(inputSpec.wantedType)) { + throw new IllegalArgumentException("Conflicting types needed for " + inputSpec.source + "; " + old + " cannot be assigned to " + inputSpec.wantedType); } } } else { diff --git a/screwdriver.yaml b/screwdriver.yaml index c43b58d37b5..00e68d4b046 100644 --- a/screwdriver.yaml +++ b/screwdriver.yaml @@ -180,8 +180,8 @@ jobs: set +x buildah login --username aressem --password "$DOCKER_HUB_DEPLOY_KEY" docker.io eval "$OPT_STATE" - buildah push docker.io/vespaengine/vespa-systemtest-centos-stream8:$VESPA_VERSION docker://docker.io/vespaengine/vespa-systemtest-centos-stream8:$VESPA_VERSION - buildah push docker.io/vespaengine/vespa-systemtest-centos-stream8:$VESPA_VERSION docker://docker.io/vespaengine/vespa-systemtest-centos-stream8:latest + buildah push --format v2s2 docker.io/vespaengine/vespa-systemtest-centos-stream8:$VESPA_VERSION docker://docker.io/vespaengine/vespa-systemtest-centos-stream8:$VESPA_VERSION + buildah push --format v2s2 docker.io/vespaengine/vespa-systemtest-centos-stream8:$VESPA_VERSION docker://docker.io/vespaengine/vespa-systemtest-centos-stream8:latest fi - *save-cache @@ -292,7 +292,7 @@ jobs: buildah tag docker.io/$IMAGE_NAME:$VESPA_VERSION vespaengine/vespa:latest # Run quick start guide $SD_SOURCE_DIR/screwdriver/test-quick-start-guide.sh - - publish-test-image: | + - publish-image: | if [[ -z $SD_PULL_REQUEST ]]; then if curl -fsSL https://index.docker.io/v1/repositories/$IMAGE_NAME/tags/$VESPA_VERSION &> /dev/null; then echo "Container image docker.io/$IMAGE_NAME:$VESPA_VERSION aldready exists." diff --git a/searchlib/src/vespa/searchlib/attribute/CMakeLists.txt b/searchlib/src/vespa/searchlib/attribute/CMakeLists.txt index c966c4f81b6..db5cf43050e 100644 --- a/searchlib/src/vespa/searchlib/attribute/CMakeLists.txt +++ b/searchlib/src/vespa/searchlib/attribute/CMakeLists.txt @@ -105,6 +105,7 @@ vespa_add_library(searchlib_attribute OBJECT postinglisttraits.cpp postingstore.cpp predicate_attribute.cpp + raw_attribute.cpp raw_buffer_store.cpp raw_buffer_store_reader.cpp raw_buffer_store_writer.cpp diff --git a/searchlib/src/vespa/searchlib/attribute/raw_attribute.cpp b/searchlib/src/vespa/searchlib/attribute/raw_attribute.cpp new file mode 100644 index 00000000000..65841f59827 --- /dev/null +++ b/searchlib/src/vespa/searchlib/attribute/raw_attribute.cpp @@ -0,0 +1,42 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include "raw_attribute.h" + +namespace search::attribute { + +RawAttribute::RawAttribute(const vespalib::string& name, const Config& config) + : NotImplementedAttribute(name, config) +{ +} + +RawAttribute::~RawAttribute() = default; + +long +RawAttribute::onSerializeForAscendingSort(DocId doc, void* serTo, long available, const common::BlobConverter*) const +{ + auto raw = get_raw(doc); + if (available >= (long)raw.size()) { + memcpy(serTo, raw.data(), raw.size()); + } else { + return -1; + } + return raw.size(); +} + +long +RawAttribute::onSerializeForDescendingSort(DocId doc, void* serTo, long available, const common::BlobConverter*) const +{ + auto raw = get_raw(doc); + if (available >= (long)raw.size()) { + auto *dst = static_cast<unsigned char *>(serTo); + const auto * src(reinterpret_cast<const uint8_t *>(raw.data())); + for (size_t i(0); i < raw.size(); ++i) { + dst[i] = 0xff - src[i]; + } + } else { + return -1; + } + return raw.size(); +} + +} diff --git a/searchlib/src/vespa/searchlib/attribute/raw_attribute.h b/searchlib/src/vespa/searchlib/attribute/raw_attribute.h new file mode 100644 index 00000000000..6ba709786e5 --- /dev/null +++ b/searchlib/src/vespa/searchlib/attribute/raw_attribute.h @@ -0,0 +1,22 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#pragma once + +#include "not_implemented_attribute.h" + +namespace search::attribute { + +/** + * Base class for all raw attributes. + */ +class RawAttribute : public NotImplementedAttribute +{ +public: + RawAttribute(const vespalib::string& name, const Config& config); + ~RawAttribute() override; + + long onSerializeForAscendingSort(DocId doc, void* serTo, long available, const common::BlobConverter*) const override; + long onSerializeForDescendingSort(DocId doc, void* serTo, long available, const common::BlobConverter*) const override; +}; + +} diff --git a/searchlib/src/vespa/searchlib/attribute/single_raw_attribute.cpp b/searchlib/src/vespa/searchlib/attribute/single_raw_attribute.cpp index 67bbd3c945d..f3be30b44d1 100644 --- a/searchlib/src/vespa/searchlib/attribute/single_raw_attribute.cpp +++ b/searchlib/src/vespa/searchlib/attribute/single_raw_attribute.cpp @@ -20,7 +20,7 @@ constexpr uint32_t max_small_buffer_type_id = 500u; namespace search::attribute { SingleRawAttribute::SingleRawAttribute(const vespalib::string& name, const Config& config) - : NotImplementedAttribute(name, config), + : RawAttribute(name, config), _ref_vector(config.getGrowStrategy(), getGenerationHolder()), _raw_store(get_memory_allocator(), max_small_buffer_type_id, mapper_grow_factor) { @@ -137,42 +137,6 @@ SingleRawAttribute::clearDoc(DocId docId) return 0u; } -long -SingleRawAttribute::onSerializeForAscendingSort(DocId doc, void * serTo, long available, const common::BlobConverter * bc) const -{ - auto raw = get_raw(doc); - vespalib::ConstBufferRef buf(raw.data(), raw.size()); - if (bc != nullptr) { - buf = bc->convert(buf); - } - if (available >= (long)buf.size()) { - memcpy(serTo, buf.data(), buf.size()); - } else { - return -1; - } - return buf.size(); -} - -long -SingleRawAttribute::onSerializeForDescendingSort(DocId doc, void * serTo, long available, const common::BlobConverter * bc) const -{ - auto raw = get_raw(doc); - vespalib::ConstBufferRef buf(raw.data(), raw.size()); - if (bc != nullptr) { - buf = bc->convert(buf); - } - if (available >= (long)buf.size()) { - auto *dst = static_cast<unsigned char *>(serTo); - const auto * src(static_cast<const uint8_t *>(buf.data())); - for (size_t i(0); i < buf.size(); ++i) { - dst[i] = 0xff - src[i]; - } - } else { - return -1; - } - return buf.size(); -} - std::unique_ptr<AttributeSaver> SingleRawAttribute::onInitSave(vespalib::stringref fileName) { diff --git a/searchlib/src/vespa/searchlib/attribute/single_raw_attribute.h b/searchlib/src/vespa/searchlib/attribute/single_raw_attribute.h index fe7c873c277..681cb0397d2 100644 --- a/searchlib/src/vespa/searchlib/attribute/single_raw_attribute.h +++ b/searchlib/src/vespa/searchlib/attribute/single_raw_attribute.h @@ -2,7 +2,7 @@ #pragma once -#include "not_implemented_attribute.h" +#include "raw_attribute.h" #include "raw_buffer_store.h" #include <vespa/vespalib/util/rcuvector.h> @@ -11,7 +11,7 @@ namespace search::attribute { /** * Attribute vector storing a single raw value per document. */ -class SingleRawAttribute : public NotImplementedAttribute +class SingleRawAttribute : public RawAttribute { using AtomicEntryRef = vespalib::datastore::AtomicEntryRef; using EntryRef = vespalib::datastore::EntryRef; @@ -43,8 +43,6 @@ public: } bool isUndefined(DocId docid) const override; uint32_t clearDoc(DocId docId) override; - long onSerializeForAscendingSort(DocId, void *, long, const common::BlobConverter *) const override; - long onSerializeForDescendingSort(DocId, void *, long, const common::BlobConverter *) const override; }; } diff --git a/vespajlib/src/main/java/com/yahoo/slime/ArrayValue.java b/vespajlib/src/main/java/com/yahoo/slime/ArrayValue.java index 9f455a5b7d4..7346f2585d7 100644 --- a/vespajlib/src/main/java/com/yahoo/slime/ArrayValue.java +++ b/vespajlib/src/main/java/com/yahoo/slime/ArrayValue.java @@ -10,9 +10,11 @@ final class ArrayValue extends Value { static final Impl initial_impl = new EmptyImpl(); private interface Impl { - public void prepareFor(ArrayValue self, Type type); - public Value add(Value value, int used); - public Value get(int index); + void prepareFor(ArrayValue self, Type type); + Value add(Value value, int used); + Value add(long value, int used); + Value add(double value, int used); + Value get(int index); } private static final class EmptyImpl implements Impl { @@ -26,6 +28,8 @@ final class ArrayValue extends Value { } } public Value add(Value value, int used) { return NixValue.invalid(); } + public Value add(long value, int used) { return NixValue.invalid(); } + public Value add(double value, int used) { return NixValue.invalid(); } public Value get(int index) { return NixValue.invalid(); } } @@ -36,15 +40,22 @@ final class ArrayValue extends Value { self.impl = new GenericImpl(this, self.used); } } + private static long[] grow(long[] arr) { + long[] v = new long[arr.length << 1]; + System.arraycopy(arr, 0, v, 0, arr.length); + return v; + } public Value add(Value value, int used) { + return NixValue.invalid(); + } + public Value add(long value, int used) { if (used == values.length) { - long[] v = values; - values = new long[v.length << 1]; - System.arraycopy(v, 0, values, 0, used); + values = grow(values); } - values[used] = value.asLong(); + values[used] = value; return get(used); } + public Value add(double value, int used) { return NixValue.invalid(); } public Value get(int index) { return new LongValue(values[index]); } } @@ -55,15 +66,22 @@ final class ArrayValue extends Value { self.impl = new GenericImpl(this, self.used); } } + private static double[] grow(double[] arr) { + double[] v = new double[arr.length << 1]; + System.arraycopy(arr, 0, v, 0, arr.length); + return v; + } public Value add(Value value, int used) { + return NixValue.invalid(); + } + public Value add(double value, int used) { if (used == values.length) { - double[] v = values; - values = new double[v.length << 1]; - System.arraycopy(v, 0, values, 0, used); + values = grow(values); } - values[used] = value.asDouble(); + values[used] = value; return get(used); } + public Value add(long value, int used) { return NixValue.invalid(); } public Value get(int index) { return new DoubleValue(values[index]); } } @@ -80,11 +98,20 @@ final class ArrayValue extends Value { } } public void prepareFor(ArrayValue self, Type type) {} + private static Value[] grow(Value[] arr) { + Value[] v = new Value[arr.length << 1]; + System.arraycopy(arr, 0, v, 0, arr.length); + return v; + } + public Value add(long value, int used) { + return add(new LongValue(value), used); + } + public Value add(double value, int used) { + return add(new DoubleValue(value), used); + } public Value add(Value value, int used) { if (used == values.length) { - Value[] v = values; - values = new Value[v.length << 1]; - System.arraycopy(v, 0, values, 0, used); + values = grow(values); } values[used] = value; return get(used); @@ -112,6 +139,18 @@ final class ArrayValue extends Value { } } + @Override + public Cursor addLong(long value) { + impl.prepareFor(this, Type.LONG); + return impl.add(value, used++); + } + + @Override + public Cursor addDouble(double value) { + impl.prepareFor(this, Type.DOUBLE); + return impl.add(value, used++); + } + protected Value addLeaf(Value value) { impl.prepareFor(this, value.type()); return impl.add(value, used++); diff --git a/vespajlib/src/main/java/com/yahoo/slime/Value.java b/vespajlib/src/main/java/com/yahoo/slime/Value.java index 053b06b97c4..985fed6c6bb 100644 --- a/vespajlib/src/main/java/com/yahoo/slime/Value.java +++ b/vespajlib/src/main/java/com/yahoo/slime/Value.java @@ -48,8 +48,8 @@ abstract class Value implements Cursor { public final Cursor addNix() { return addLeaf(NixValue.instance()); } public final Cursor addBool(boolean bit) { return addLeaf(BoolValue.instance(bit)); } - public final Cursor addLong(long l) { return addLeaf(new LongValue(l)); } - public final Cursor addDouble(double d) { return addLeaf(new DoubleValue(d)); } + public Cursor addLong(long l) { return NixValue.invalid(); } + public Cursor addDouble(double d) { return NixValue.invalid(); } public final Cursor addString(String str) { return addLeaf(StringValue.create(str)); } public final Cursor addString(byte[] utf8) { return addLeaf(Utf8Value.create(utf8)); } public final Cursor addData(byte[] data) { return addLeaf(DataValue.create(data)); } diff --git a/zkfacade/src/main/java/com/yahoo/vespa/curator/Curator.java b/zkfacade/src/main/java/com/yahoo/vespa/curator/Curator.java index 80159624eed..9a7dd469fa1 100644 --- a/zkfacade/src/main/java/com/yahoo/vespa/curator/Curator.java +++ b/zkfacade/src/main/java/com/yahoo/vespa/curator/Curator.java @@ -173,11 +173,11 @@ public class Curator extends AbstractComponent implements AutoCloseable { }); } - public CompletionWaiter getCompletionWaiter(Path waiterPath, int numMembers, String id) { + public CompletionWaiter getCompletionWaiter(Path waiterPath, String id) { return CuratorCompletionWaiter.create(this, waiterPath, id); } - public CompletionWaiter createCompletionWaiter(Path parentPath, String waiterNode, int numMembers, String id) { + public CompletionWaiter createCompletionWaiter(Path parentPath, String waiterNode, String id) { return CuratorCompletionWaiter.createAndInitialize(this, parentPath, waiterNode, id); } diff --git a/zkfacade/src/main/java/com/yahoo/vespa/curator/mock/MockCurator.java b/zkfacade/src/main/java/com/yahoo/vespa/curator/mock/MockCurator.java index e61e4f3add9..5ebd33801ed 100644 --- a/zkfacade/src/main/java/com/yahoo/vespa/curator/mock/MockCurator.java +++ b/zkfacade/src/main/java/com/yahoo/vespa/curator/mock/MockCurator.java @@ -82,12 +82,12 @@ public class MockCurator extends Curator { } @Override - public CompletionWaiter getCompletionWaiter(Path parentPath, int numMembers, String id) { + public CompletionWaiter getCompletionWaiter(Path parentPath, String id) { return mockFramework().createCompletionWaiter(); } @Override - public CompletionWaiter createCompletionWaiter(Path parentPath, String waiterNode, int numMembers, String id) { + public CompletionWaiter createCompletionWaiter(Path parentPath, String waiterNode, String id) { return mockFramework().createCompletionWaiter(); } |