diff options
22 files changed, 444 insertions, 54 deletions
diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/ServiceRegistry.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/ServiceRegistry.java index b2fd4a5d76c..98591ba41e2 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/ServiceRegistry.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/ServiceRegistry.java @@ -25,6 +25,7 @@ import com.yahoo.vespa.hosted.controller.api.integration.resource.CostReportCons import com.yahoo.vespa.hosted.controller.api.integration.resource.MeteringClient; import com.yahoo.vespa.hosted.controller.api.integration.routing.GlobalRoutingService; import com.yahoo.vespa.hosted.controller.api.integration.secrets.TenantSecretService; +import com.yahoo.vespa.hosted.controller.api.integration.vcmr.ChangeRequestClient; import com.yahoo.vespa.hosted.controller.api.integration.zone.ZoneRegistry; import java.time.Clock; @@ -90,4 +91,6 @@ public interface ServiceRegistry { TenantSecretService tenantSecretService(); ArchiveService archiveService(); + + ChangeRequestClient changeRequestClient(); } diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/vcmr/ChangeRequest.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/vcmr/ChangeRequest.java new file mode 100644 index 00000000000..a56f96955de --- /dev/null +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/vcmr/ChangeRequest.java @@ -0,0 +1,139 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.api.integration.vcmr; + +import java.util.List; +import java.util.Objects; + +/** + * @author olaa + */ +public class ChangeRequest { + + private final String id; + private final ChangeRequestSource changeRequestSource; + private final List<String> impactedSwitches; + private final List<String> impactedHosts; + private final boolean isApproved; + private final Impact impact; + + private ChangeRequest(String id, ChangeRequestSource changeRequestSource, List<String> impactedSwitches, List<String> impactedHosts, boolean isApproved, Impact impact) { + this.id = Objects.requireNonNull(id); + this.changeRequestSource = Objects.requireNonNull(changeRequestSource); + this.impactedSwitches = Objects.requireNonNull(impactedSwitches); + this.impactedHosts = Objects.requireNonNull(impactedHosts); + this.isApproved = Objects.requireNonNull(isApproved); + this.impact = Objects.requireNonNull(impact); + } + + public String getId() { + return id; + } + + public ChangeRequestSource getChangeRequestSource() { + return changeRequestSource; + } + + public List<String> getImpactedSwitches() { + return impactedSwitches; + } + + public List<String> getImpactedHosts() { + return impactedHosts; + } + + public boolean isApproved() { + return isApproved; + } + + public Impact getImpact() { + return impact; + } + + @Override + public String toString() { + return "ChangeRequest{" + + "id='" + id + '\'' + + ", changeRequestSource=" + changeRequestSource + + ", impactedSwitches=" + impactedSwitches + + ", impactedHosts=" + impactedHosts + + ", isApproved=" + isApproved + + ", impact=" + impact + + '}'; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + ChangeRequest that = (ChangeRequest) o; + return isApproved == that.isApproved && + Objects.equals(id, that.id) && + Objects.equals(changeRequestSource, that.changeRequestSource) && + Objects.equals(impactedSwitches, that.impactedSwitches) && + Objects.equals(impactedHosts, that.impactedHosts) && + impact == that.impact; + } + + @Override + public int hashCode() { + return Objects.hash(id, changeRequestSource, impactedSwitches, impactedHosts, isApproved, impact); + } + + public static class Builder { + private String id; + private ChangeRequestSource changeRequestSource; + private List<String> impactedSwitches; + private List<String> impactedHosts; + private boolean isApproved; + private Impact impact; + + + public Builder id(String id) { + this.id = id; + return this; + } + + public Builder changeRequestSource(ChangeRequestSource changeRequestSource) { + this.changeRequestSource = changeRequestSource; + return this; + } + + public Builder impactedSwitches(List<String> impactedSwitches) { + this.impactedSwitches = impactedSwitches; + return this; + } + + public Builder impactedHosts(List<String> impactedHosts) { + this.impactedHosts = impactedHosts; + return this; + } + + public Builder isApproved(boolean isApproved) { + this.isApproved = isApproved; + return this; + } + + public Builder impact(Impact impact) { + this.impact = impact; + return this; + } + + public ChangeRequest build() { + return new ChangeRequest(id, changeRequestSource, impactedSwitches, impactedHosts, isApproved, impact); + } + + public String getId() { + return this.id; + } + } + + public enum Impact { + NONE, + LOW, + MODERATE, + HIGH, + VERY_HIGH, + UNKNOWN + } + +} diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/vcmr/ChangeRequestClient.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/vcmr/ChangeRequestClient.java new file mode 100644 index 00000000000..e8ff768927f --- /dev/null +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/vcmr/ChangeRequestClient.java @@ -0,0 +1,15 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.api.integration.vcmr; + +import java.util.List; + +/** + * @author olaa + */ +public interface ChangeRequestClient { + + List<ChangeRequest> getUpcomingChangeRequests(); + + void approveChangeRequests(List<ChangeRequest> changeRequests); + +} diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/vcmr/ChangeRequestSource.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/vcmr/ChangeRequestSource.java new file mode 100644 index 00000000000..63f6c256766 --- /dev/null +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/vcmr/ChangeRequestSource.java @@ -0,0 +1,137 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.api.integration.vcmr; + +import java.time.ZonedDateTime; +import java.util.Objects; + +/** + * @author olaa + */ +public class ChangeRequestSource { + + private final String system; + private final String id; + private final Status status; + private final String url; + private final ZonedDateTime plannedStartTime; + private final ZonedDateTime plannedEndTime; + + + private ChangeRequestSource(String system, String id, String url, Status status, ZonedDateTime plannedStartTime, ZonedDateTime plannedEndTime) { + this.system = Objects.requireNonNull(system); + this.id = Objects.requireNonNull(id); + this.url = Objects.requireNonNull(url); + this.status = Objects.requireNonNull(status); + this.plannedStartTime = Objects.requireNonNull(plannedStartTime); + this.plannedEndTime = Objects.requireNonNull(plannedEndTime); + } + + public String getSystem() { + return system; + } + + public String getId() { + return id; + } + + public Status getStatus() { + return status; + } + + public String getUrl() { + return url; + } + + public ZonedDateTime getPlannedStartTime() { + return plannedStartTime; + } + + public ZonedDateTime getPlannedEndTime() { + return plannedEndTime; + } + + @Override + public String toString() { + return "ChangeRequestSource{" + + "system='" + system + '\'' + + ", id='" + id + '\'' + + ", status=" + status + + ", url='" + url + '\'' + + ", plannedStartTime=" + plannedStartTime + + ", plannedEndTime=" + plannedEndTime + + '}'; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + ChangeRequestSource that = (ChangeRequestSource) o; + return Objects.equals(system, that.system) && + Objects.equals(id, that.id) && + status == that.status && + Objects.equals(url, that.url) && + Objects.equals(plannedStartTime, that.plannedStartTime) && + Objects.equals(plannedEndTime, that.plannedEndTime); + } + + @Override + public int hashCode() { + return Objects.hash(system, id, status, url, plannedStartTime, plannedEndTime); + } + + public enum Status { + DRAFT, + WAITING_FOR_APPROVAL, + APPROVED, + STARTED, + COMPLETE, + CLOSED, + CANCELED, + UNKNOWN_STATUS + } + + public static class Builder { + private String system; + private String id; + private String url; + private Status status; + private ZonedDateTime plannedStartTime; + private ZonedDateTime plannedEndTime; + + public Builder system(String system) { + this.system = system; + return this; + } + + public Builder id(String id) { + this.id = id; + return this; + } + + public Builder url(String url) { + this.url = url; + return this; + } + + public Builder status(Status status) { + this.status = status; + return this; + } + + public Builder plannedStartTime(ZonedDateTime plannedStartTime) { + this.plannedStartTime = plannedStartTime; + return this; + } + + public Builder plannedEndTime(ZonedDateTime plannedEndTime) { + this.plannedEndTime = plannedEndTime; + return this; + } + + public ChangeRequestSource build() { + return new ChangeRequestSource(system, id, url, status, plannedStartTime, plannedEndTime); + } + } + +} diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/vcmr/NoopChangeRequestClient.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/vcmr/NoopChangeRequestClient.java new file mode 100644 index 00000000000..d548620b062 --- /dev/null +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/vcmr/NoopChangeRequestClient.java @@ -0,0 +1,19 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.api.integration.vcmr; + +import java.util.List; + +/** + * @author olaa + */ +public class NoopChangeRequestClient implements ChangeRequestClient { + + @Override + public List<ChangeRequest> getUpcomingChangeRequests() { + return List.of(); + } + + @Override + public void approveChangeRequests(List<ChangeRequest> changeRequests) {} + +} diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/vcmr/package-info.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/vcmr/package-info.java new file mode 100644 index 00000000000..8962d8e79f4 --- /dev/null +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/vcmr/package-info.java @@ -0,0 +1,5 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +@ExportPackage +package com.yahoo.vespa.hosted.controller.api.integration.vcmr; + +import com.yahoo.osgi.annotation.ExportPackage;
\ No newline at end of file diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ChangeRequestMaintainer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ChangeRequestMaintainer.java new file mode 100644 index 00000000000..71291e5d766 --- /dev/null +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ChangeRequestMaintainer.java @@ -0,0 +1,46 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.maintenance; + +import com.yahoo.config.provision.SystemName; +import com.yahoo.vespa.hosted.controller.Controller; +import com.yahoo.vespa.hosted.controller.api.integration.vcmr.ChangeRequestClient; + +import java.time.Duration; +import java.util.function.Predicate; +import java.util.logging.Logger; +import java.util.stream.Collectors; + +/** + * @author olaa + */ +public class ChangeRequestMaintainer extends ControllerMaintainer { + + private final Logger logger = Logger.getLogger(ChangeRequestMaintainer.class.getName()); + private final ChangeRequestClient changeRequestClient; + + public ChangeRequestMaintainer(Controller controller, Duration interval) { + super(controller, interval, null, SystemName.allOf(Predicate.not(SystemName::isPublic))); + this.changeRequestClient = controller.serviceRegistry().changeRequestClient(); + } + + + @Override + protected boolean maintain() { + var changeRequests = changeRequestClient.getUpcomingChangeRequests(); + + if (!changeRequests.isEmpty()) { + logger.fine(() -> "Found the following upcoming change requests:"); + changeRequests.forEach(changeRequest -> logger.fine(changeRequest::toString)); + } + + var unapprovedRequests = changeRequests + .stream() + .filter(changeRequest -> !changeRequest.isApproved()) + .collect(Collectors.toList()); + + changeRequestClient.approveChangeRequests(unapprovedRequests); + + // TODO: Store in curator? + return true; + } +} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintenance.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintenance.java index d9a233eb475..18960de16af 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintenance.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintenance.java @@ -67,6 +67,7 @@ public class ControllerMaintenance extends AbstractComponent { maintainers.add(new TrafficShareUpdater(controller, intervals.trafficFractionUpdater)); maintainers.add(new ArchiveUriUpdater(controller, intervals.archiveUriUpdater)); maintainers.add(new TenantRoleMaintainer(controller, intervals.tenantRoleMaintainer)); + maintainers.add(new ChangeRequestMaintainer(controller, intervals.changeRequestMaintainer)); } public Upgrader upgrader() { return upgrader; } @@ -119,6 +120,7 @@ public class ControllerMaintenance extends AbstractComponent { private final Duration trafficFractionUpdater; private final Duration archiveUriUpdater; private final Duration tenantRoleMaintainer; + private final Duration changeRequestMaintainer; public Intervals(SystemName system) { this.system = Objects.requireNonNull(system); @@ -148,6 +150,7 @@ public class ControllerMaintenance extends AbstractComponent { this.trafficFractionUpdater = duration(5, MINUTES); this.archiveUriUpdater = duration(5, MINUTES); this.tenantRoleMaintainer = duration(5, MINUTES); + this.changeRequestMaintainer = duration(12, HOURS); } private Duration duration(long amount, TemporalUnit unit) { diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ServiceRegistryMock.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ServiceRegistryMock.java index 6ffac36fd6a..e96eed2b6d9 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ServiceRegistryMock.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ServiceRegistryMock.java @@ -35,6 +35,8 @@ import com.yahoo.vespa.hosted.controller.api.integration.stubs.MockMailer; import com.yahoo.vespa.hosted.controller.api.integration.stubs.MockMeteringClient; import com.yahoo.vespa.hosted.controller.api.integration.stubs.MockRunDataStore; import com.yahoo.vespa.hosted.controller.api.integration.stubs.MockTesterCloud; +import com.yahoo.vespa.hosted.controller.api.integration.vcmr.ChangeRequestClient; +import com.yahoo.vespa.hosted.controller.api.integration.vcmr.NoopChangeRequestClient; /** * A mock implementation of a {@link ServiceRegistry} for testing purposes. @@ -70,6 +72,7 @@ public class ServiceRegistryMock extends AbstractComponent implements ServiceReg private final ContainerRegistryMock containerRegistry = new ContainerRegistryMock(); private final NoopTenantSecretService tenantSecretService = new NoopTenantSecretService(); private final ArchiveService archiveService = new MockArchiveService(); + private final ChangeRequestClient changeRequestClient = new NoopChangeRequestClient(); public ServiceRegistryMock(SystemName system) { this.zoneRegistryMock = new ZoneRegistryMock(system); @@ -221,6 +224,11 @@ public class ServiceRegistryMock extends AbstractComponent implements ServiceReg return archiveService; } + @Override + public ChangeRequestClient changeRequestClient() { + return changeRequestClient; + } + public ConfigServerMock configServerMock() { return configServerMock; } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/controller/responses/maintenance.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/controller/responses/maintenance.json index e1e44bd0943..08741e7f38a 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/controller/responses/maintenance.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/controller/responses/maintenance.json @@ -10,6 +10,9 @@ "name": "ArchiveUriUpdater" }, { + "name": "ChangeRequestMaintainer" + }, + { "name": "CloudEventReporter" }, { diff --git a/eval/src/vespa/eval/eval/cell_type.h b/eval/src/vespa/eval/eval/cell_type.h index 4eec639478d..57f707c2aa8 100644 --- a/eval/src/vespa/eval/eval/cell_type.h +++ b/eval/src/vespa/eval/eval/cell_type.h @@ -121,17 +121,14 @@ struct CellMeta { // convenience functions to be used for specific operations constexpr CellMeta map() const { return decay(); } - static constexpr CellMeta reduce(CellType input_cell_type, bool output_is_scalar) { - return normalize(input_cell_type, output_is_scalar).decay(); - } constexpr CellMeta reduce(bool output_is_scalar) const { - return CellMeta::reduce(cell_type, output_is_scalar); + return normalize(cell_type, output_is_scalar).decay(); } static constexpr CellMeta join(CellMeta a, CellMeta b) { return unify(a, b).decay(); } static constexpr CellMeta merge(CellMeta a, CellMeta b) { return unify(a, b).decay(); } static constexpr CellMeta concat(CellMeta a, CellMeta b) { return unify(a, b); } - static constexpr CellMeta peek(CellType input_cell_type, bool output_is_scalar) { - return normalize(input_cell_type, output_is_scalar); + constexpr CellMeta peek(bool output_is_scalar) const { + return normalize(cell_type, output_is_scalar); } constexpr CellMeta rename() const { return self(); } }; diff --git a/eval/src/vespa/eval/eval/value_type.cpp b/eval/src/vespa/eval/eval/value_type.cpp index fbd04babc70..440aecce76e 100644 --- a/eval/src/vespa/eval/eval/value_type.cpp +++ b/eval/src/vespa/eval/eval/value_type.cpp @@ -275,7 +275,7 @@ ValueType ValueType::reduce(const std::vector<vespalib::string> &dimensions_in) const { MyReduce result(_dimensions, dimensions_in); - auto meta = CellMeta::reduce(_cell_type, result.dimensions.empty()); + auto meta = cell_meta().reduce(result.dimensions.empty()); return error_if(_error || result.has_error, make_type(meta.cell_type, std::move(result.dimensions))); } @@ -284,7 +284,7 @@ ValueType ValueType::peek(const std::vector<vespalib::string> &dimensions_in) const { MyReduce result(_dimensions, dimensions_in); - auto meta = CellMeta::peek(_cell_type, result.dimensions.empty()); + auto meta = cell_meta().peek(result.dimensions.empty()); return error_if(_error || result.has_error || dimensions_in.empty(), make_type(meta.cell_type, std::move(result.dimensions))); } diff --git a/eval/src/vespa/eval/instruction/dense_single_reduce_function.cpp b/eval/src/vespa/eval/instruction/dense_single_reduce_function.cpp index 014f36eaa2c..87f02f16345 100644 --- a/eval/src/vespa/eval/instruction/dense_single_reduce_function.cpp +++ b/eval/src/vespa/eval/instruction/dense_single_reduce_function.cpp @@ -120,7 +120,7 @@ void my_single_reduce_op(InterpretedFunction::State &state, uint64_t param) { struct MyGetFun { template <typename ICM, typename AGGR, typename GE8, typename I1> static auto invoke() { using ICT = CellValueType<ICM::value.cell_type>; - using OCT = CellValueType<CellMeta::reduce(ICM::value.cell_type, false).cell_type>; + using OCT = CellValueType<ICM::value.reduce(false).cell_type>; using AggrType = typename AGGR::template templ<OCT>; return my_single_reduce_op<ICT, OCT, AggrType, GE8::value, I1::value>; } diff --git a/eval/src/vespa/eval/instruction/generic_peek.cpp b/eval/src/vespa/eval/instruction/generic_peek.cpp index ae5897257df..da61de33f3a 100644 --- a/eval/src/vespa/eval/instruction/generic_peek.cpp +++ b/eval/src/vespa/eval/instruction/generic_peek.cpp @@ -348,7 +348,7 @@ void my_generic_peek_op(State &state, uint64_t param_in) { struct SelectGenericPeekOp { template <typename ICM, typename OutIsScalar> static auto invoke() { using ICT = CellValueType<ICM::value.cell_type>; - constexpr CellMeta ocm = CellMeta::peek(ICM::value.cell_type, OutIsScalar::value); + constexpr CellMeta ocm = ICM::value.peek(OutIsScalar::value); using OCT = CellValueType<ocm.cell_type>; return my_generic_peek_op<ICT,OCT>; } diff --git a/eval/src/vespa/eval/instruction/generic_reduce.cpp b/eval/src/vespa/eval/instruction/generic_reduce.cpp index 23e53e9734b..7f48976048c 100644 --- a/eval/src/vespa/eval/instruction/generic_reduce.cpp +++ b/eval/src/vespa/eval/instruction/generic_reduce.cpp @@ -191,7 +191,7 @@ void my_full_reduce_op(State &state, uint64_t) { struct SelectGenericReduceOp { template <typename ICM, typename OIS, typename AGGR> static auto invoke(const ReduceParam ¶m) { using ICT = CellValueType<ICM::value.cell_type>; - using OCT = CellValueType<CellMeta::reduce(ICM::value.cell_type, OIS::value).cell_type>; + using OCT = CellValueType<ICM::value.reduce(OIS::value).cell_type>; using AggrType = typename AGGR::template templ<OCT>; if constexpr (OIS::value) { return my_full_reduce_op<ICT, AggrType>; @@ -299,7 +299,7 @@ GenericReduce::make_instruction(const ValueType &result_type, { auto ¶m = stash.create<ReduceParam>(input_type, dimensions, factory); assert(result_type == param.res_type); - assert(result_type.cell_meta().eq(CellMeta::reduce(input_type.cell_type(), result_type.is_double()))); + assert(result_type.cell_meta().eq(input_type.cell_meta().reduce(result_type.is_double()))); auto fun = typify_invoke<3,ReduceTypify,SelectGenericReduceOp>(input_type.cell_meta(), result_type.cell_meta().is_scalar, aggr, param); return Instruction(fun, wrap_param<ReduceParam>(param)); } diff --git a/messagebus/src/main/java/com/yahoo/messagebus/MessageBus.java b/messagebus/src/main/java/com/yahoo/messagebus/MessageBus.java index feec51643c4..99d9a6d6483 100644 --- a/messagebus/src/main/java/com/yahoo/messagebus/MessageBus.java +++ b/messagebus/src/main/java/com/yahoo/messagebus/MessageBus.java @@ -63,7 +63,7 @@ public class MessageBus implements ConfigHandler, NetworkOwner, MessageHandler, private final AtomicBoolean destroyed = new AtomicBoolean(false); private final ProtocolRepository protocolRepository = new ProtocolRepository(); private final AtomicReference<Map<String, RoutingTable>> tablesRef = new AtomicReference<>(null); - private final CopyOnWriteHashMap<String, MessageHandler> sessions = new CopyOnWriteHashMap<>(); + private final Map<String, MessageHandler> sessions = new ConcurrentHashMap<>(); private final Network net; private final Messenger msn; private final Resender resender; @@ -72,7 +72,7 @@ public class MessageBus implements ConfigHandler, NetworkOwner, MessageHandler, private int pendingCount = 0; private int pendingSize = 0; private final Thread careTaker = new Thread(this::sendBlockedMessages); - private final ConcurrentHashMap<SendBlockedMessages, Long> blockedSenders = new ConcurrentHashMap<>(); + private final Map<SendBlockedMessages, Long> blockedSenders = new ConcurrentHashMap<>(); public interface SendBlockedMessages { /** diff --git a/searchlib/src/vespa/searchlib/attribute/enum_store_dictionary.cpp b/searchlib/src/vespa/searchlib/attribute/enum_store_dictionary.cpp index e71aa98b725..f7a2a507aec 100644 --- a/searchlib/src/vespa/searchlib/attribute/enum_store_dictionary.cpp +++ b/searchlib/src/vespa/searchlib/attribute/enum_store_dictionary.cpp @@ -131,6 +131,33 @@ EnumStoreDictionary<DictionaryT>::find_matching_enums(const vespalib::datastore: template <> void +EnumStoreDictionary<EnumTree>::clear_all_posting_lists(std::function<void(EntryRef)>) +{ + LOG_ABORT("should not be reached"); +} + +template <> +void +EnumStoreDictionary<EnumPostingTree>::clear_all_posting_lists(std::function<void(EntryRef)> clearer) +{ + auto& dict = this->_dict; + auto itr = dict.begin(); + EntryRef prev; + while (itr.valid()) { + EntryRef ref(itr.getData()); + if (ref.ref() != prev.ref()) { + if (ref.valid()) { + clearer(ref); + } + prev = ref; + } + itr.writeData(EntryRef().ref()); + ++itr; + } +} + +template <> +void EnumStoreDictionary<EnumTree>::update_posting_list(Index, const vespalib::datastore::EntryComparator&, std::function<EntryRef(EntryRef)>) { LOG_ABORT("should not be reached"); diff --git a/searchlib/src/vespa/searchlib/attribute/enum_store_dictionary.h b/searchlib/src/vespa/searchlib/attribute/enum_store_dictionary.h index 363bd519538..67bbb1cd3e9 100644 --- a/searchlib/src/vespa/searchlib/attribute/enum_store_dictionary.h +++ b/searchlib/src/vespa/searchlib/attribute/enum_store_dictionary.h @@ -49,6 +49,7 @@ public: std::vector<attribute::IAttributeVector::EnumHandle> find_matching_enums(const vespalib::datastore::EntryComparator& cmp) const override; + void clear_all_posting_lists(std::function<void(EntryRef)> clearer) override; void update_posting_list(Index idx, const vespalib::datastore::EntryComparator& cmp, std::function<EntryRef(EntryRef)> updater) override; EnumPostingTree& get_posting_dictionary() override; const EnumPostingTree& get_posting_dictionary() const override; diff --git a/searchlib/src/vespa/searchlib/attribute/i_enum_store_dictionary.h b/searchlib/src/vespa/searchlib/attribute/i_enum_store_dictionary.h index 7712cdd3dad..8bbd7677595 100644 --- a/searchlib/src/vespa/searchlib/attribute/i_enum_store_dictionary.h +++ b/searchlib/src/vespa/searchlib/attribute/i_enum_store_dictionary.h @@ -47,6 +47,7 @@ public: virtual std::vector<attribute::IAttributeVector::EnumHandle> find_matching_enums(const vespalib::datastore::EntryComparator& cmp) const = 0; + virtual void clear_all_posting_lists(std::function<void(EntryRef)> clearer) = 0; virtual void update_posting_list(Index idx, const vespalib::datastore::EntryComparator& cmp, std::function<EntryRef(EntryRef)> updater) = 0; virtual EnumPostingTree& get_posting_dictionary() = 0; virtual const EnumPostingTree& get_posting_dictionary() const = 0; diff --git a/searchlib/src/vespa/searchlib/attribute/postinglistattribute.cpp b/searchlib/src/vespa/searchlib/attribute/postinglistattribute.cpp index ccf248a7c15..d3223901a71 100644 --- a/searchlib/src/vespa/searchlib/attribute/postinglistattribute.cpp +++ b/searchlib/src/vespa/searchlib/attribute/postinglistattribute.cpp @@ -29,20 +29,11 @@ PostingListAttributeBase<P>::clearAllPostings() { _postingList.clearBuilder(); _attr.incGeneration(); // Force freeze - auto& dict = _dictionary.get_posting_dictionary(); - auto itr = dict.begin(); - EntryRef prev; - while (itr.valid()) { - EntryRef ref(itr.getData()); - if (ref.ref() != prev.ref()) { - if (ref.valid()) { - _postingList.clear(ref); - } - prev = ref; - } - itr.writeData(EntryRef().ref()); - ++itr; - } + auto clearer = [this](EntryRef posting_idx) + { + _postingList.clear(posting_idx); + }; + _dictionary.clear_all_posting_lists(clearer); _attr.incGeneration(); // Force freeze } @@ -168,22 +159,16 @@ clearPostings(attribute::IAttributeVector::EnumHandle eidx, } EntryRef er(eidx); - auto& dict = _dictionary.get_posting_dictionary(); - auto itr = dict.lowerBound(er, cmp); - assert(itr.valid()); - - EntryRef newPosting(itr.getData()); - assert(newPosting.valid()); - - _postingList.apply(newPosting, - &postings._additions[0], - &postings._additions[0] + - postings._additions.size(), - &postings._removals[0], - &postings._removals[0] + - postings._removals.size()); - dict.thaw(itr); - itr.writeData(newPosting.ref()); + auto updater = [this, &postings](EntryRef posting_idx) -> EntryRef + { + _postingList.apply(posting_idx, + &postings._additions[0], + &postings._additions[0] + postings._additions.size(), + &postings._removals[0], + &postings._removals[0] + postings._removals.size()); + return posting_idx; + }; + _dictionary.update_posting_list(er, cmp, updater); } template <typename P> diff --git a/travis/travis-build.sh b/travis/travis-build.sh index 849f807042a..9537118ae67 100755 --- a/travis/travis-build.sh +++ b/travis/travis-build.sh @@ -56,13 +56,13 @@ if [[ $SHOULD_BUILD == systemtest ]]; then devtoolset-9-libatomic-devel \ devtoolset-9-binutils \ libxml2-devel \ - rh-ruby25-rubygems-devel \ - rh-ruby25-ruby-devel \ - rh-ruby25 \ - rh-ruby25-rubygem-net-telnet + rh-ruby27-rubygems-devel \ + rh-ruby27-ruby-devel \ + rh-ruby27 \ + rh-ruby27-rubygem-net-telnet - source /opt/rh/rh-ruby25/enable - gem install libxml-ruby gnuplot distribution test-unit builder concurrent-ruby + source /opt/rh/rh-ruby27/enable + gem install libxml-ruby gnuplot distribution test-unit builder concurrent-ruby ffi cd $HOME git clone https://github.com/vespa-engine/system-test diff --git a/vespaclient-container-plugin/src/main/java/com/yahoo/document/restapi/resource/DocumentV1ApiHandler.java b/vespaclient-container-plugin/src/main/java/com/yahoo/document/restapi/resource/DocumentV1ApiHandler.java index 9b198ce8df9..faf15d397c5 100644 --- a/vespaclient-container-plugin/src/main/java/com/yahoo/document/restapi/resource/DocumentV1ApiHandler.java +++ b/vespaclient-container-plugin/src/main/java/com/yahoo/document/restapi/resource/DocumentV1ApiHandler.java @@ -259,6 +259,7 @@ public class DocumentV1ApiHandler extends AbstractRequestHandler { // This blocks until all visitors are done. These, in turn, may require the asyncSession to be alive // to be able to run, as well as dispatch of operations against it, which is done by visitDispatcher. + visits.values().forEach(VisitorSession::abort); visits.values().forEach(VisitorSession::destroy); // Shut down both dispatchers, so only we empty the queues of outstanding operations, and can be sure they're empty. @@ -1102,10 +1103,10 @@ public class DocumentV1ApiHandler extends AbstractRequestHandler { response.respond(Response.Status.INTERNAL_SERVER_ERROR); } - visitDispatcher.execute(() -> { - phaser.arriveAndAwaitAdvance(); // We may get here while dispatching thread is still putting us in the map. - visits.remove(this).destroy(); - }); + }); + visitDispatcher.execute(() -> { + phaser.arriveAndAwaitAdvance(); // We may get here while dispatching thread is still putting us in the map. + visits.remove(this).destroy(); }); } }; |