diff options
193 files changed, 4133 insertions, 1268 deletions
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/VespaMetricSet.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/VespaMetricSet.java index db716866f15..d8c59ebda65 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/VespaMetricSet.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/VespaMetricSet.java @@ -797,6 +797,9 @@ public class VespaMetricSet { metrics.add(new Metric("vds.mergethrottler.queuesize.max")); metrics.add(new Metric("vds.mergethrottler.queuesize.sum")); metrics.add(new Metric("vds.mergethrottler.queuesize.count")); + metrics.add(new Metric("vds.mergethrottler.active_window_size.max")); + metrics.add(new Metric("vds.mergethrottler.active_window_size.sum")); + metrics.add(new Metric("vds.mergethrottler.active_window_size.count")); metrics.add(new Metric("vds.mergethrottler.bounced_due_to_back_pressure.rate")); metrics.add(new Metric("vds.mergethrottler.locallyexecutedmerges.ok.rate")); metrics.add(new Metric("vds.mergethrottler.mergechains.ok.rate")); diff --git a/configdefinitions/src/vespa/stor-filestor.def b/configdefinitions/src/vespa/stor-filestor.def index 88028cef394..66700eff3e6 100644 --- a/configdefinitions/src/vespa/stor-filestor.def +++ b/configdefinitions/src/vespa/stor-filestor.def @@ -43,6 +43,10 @@ common_merge_chain_optimalization_minimum_size int default=64 restart ## Note that this will gradually be increased to reach stor-distributormanager:splitsize which is currently at 32M bucket_merge_chunk_size int default=33554432 restart +## If set, portions of apply bucket diff handling will be performed asynchronously +## with persistence thread not waiting for local writes to complete. +async_apply_bucket_diff bool default=false + ## When merging, it is possible to send more metadata than needed in order to ## let local nodes in merge decide which entries fits best to add this time ## based on disk location. Toggle this option on to use it. Note that memory diff --git a/container-search/src/main/java/com/yahoo/search/dispatch/rpc/Client.java b/container-search/src/main/java/com/yahoo/search/dispatch/rpc/Client.java index ca40683a887..0a7357f4a86 100644 --- a/container-search/src/main/java/com/yahoo/search/dispatch/rpc/Client.java +++ b/container-search/src/main/java/com/yahoo/search/dispatch/rpc/Client.java @@ -84,9 +84,6 @@ interface Client { } interface NodeConnection { - void getDocsums(List<FastHit> hits, CompressionType compression, int uncompressedLength, byte[] compressedSlime, - RpcFillInvoker.GetDocsumsResponseReceiver responseReceiver, double timeoutSeconds); - void request(String rpcMethod, CompressionType compression, int uncompressedLength, byte[] compressedPayload, ResponseReceiver responseReceiver, double timeoutSeconds); diff --git a/container-search/src/main/java/com/yahoo/search/dispatch/rpc/RpcClient.java b/container-search/src/main/java/com/yahoo/search/dispatch/rpc/RpcClient.java index bb0bbf4b529..918d9566913 100644 --- a/container-search/src/main/java/com/yahoo/search/dispatch/rpc/RpcClient.java +++ b/container-search/src/main/java/com/yahoo/search/dispatch/rpc/RpcClient.java @@ -60,18 +60,6 @@ class RpcClient implements Client { } @Override - public void getDocsums(List<FastHit> hits, CompressionType compression, int uncompressedLength, - byte[] compressedSlime, RpcFillInvoker.GetDocsumsResponseReceiver responseReceiver, double timeoutSeconds) { - Request request = new Request("proton.getDocsums"); - request.parameters().add(new Int8Value(compression.getCode())); - request.parameters().add(new Int32Value(uncompressedLength)); - request.parameters().add(new DataValue(compressedSlime)); - - request.setContext(hits); - invokeAsync(request, timeoutSeconds, new RpcDocsumResponseWaiter(this, responseReceiver)); - } - - @Override public void request(String rpcMethod, CompressionType compression, int uncompressedLength, byte[] compressedPayload, ResponseReceiver responseReceiver, double timeoutSeconds) { Request request = new Request(rpcMethod); @@ -104,44 +92,6 @@ class RpcClient implements Client { } - private static class RpcDocsumResponseWaiter implements RequestWaiter { - - /** The node to which we made the request we are waiting for - for error messages only */ - private final RpcNodeConnection node; - - /** The handler to which the response is forwarded */ - private final RpcFillInvoker.GetDocsumsResponseReceiver handler; - - public RpcDocsumResponseWaiter(RpcNodeConnection node, RpcFillInvoker.GetDocsumsResponseReceiver handler) { - this.node = node; - this.handler = handler; - } - - @Override - public void handleRequestDone(Request requestWithResponse) { - if (requestWithResponse.isError()) { - handler.receive(ResponseOrError.fromError("Error response from " + node + ": " + requestWithResponse.errorMessage())); - return; - } - - Values returnValues = requestWithResponse.returnValues(); - if (returnValues.size() < 3) { - handler.receive(ResponseOrError.fromError( - "Invalid getDocsums response from " + node + ": Expected 3 return arguments, got " + returnValues.size())); - return; - } - - byte compression = returnValues.get(0).asInt8(); - int uncompressedSize = returnValues.get(1).asInt32(); - byte[] compressedSlimeBytes = returnValues.get(2).asData(); - @SuppressWarnings("unchecked") // TODO: Non-protobuf rpc docsums to be removed soon - List<FastHit> hits = (List<FastHit>) requestWithResponse.getContext(); - handler.receive( - ResponseOrError.fromResponse(new GetDocsumsResponse(compression, uncompressedSize, compressedSlimeBytes, hits))); - } - - } - private static class RpcProtobufResponseWaiter implements RequestWaiter { /** The node to which we made the request we are waiting for - for error messages only */ diff --git a/container-search/src/main/java/com/yahoo/search/dispatch/rpc/RpcFillInvoker.java b/container-search/src/main/java/com/yahoo/search/dispatch/rpc/RpcFillInvoker.java deleted file mode 100644 index ad5d129ef6d..00000000000 --- a/container-search/src/main/java/com/yahoo/search/dispatch/rpc/RpcFillInvoker.java +++ /dev/null @@ -1,260 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.search.dispatch.rpc; - -import com.yahoo.collections.ListMap; -import com.yahoo.compress.CompressionType; -import com.yahoo.compress.Compressor; -import com.yahoo.container.protect.Error; -import com.yahoo.data.access.Inspector; -import com.yahoo.data.access.slime.SlimeAdapter; -import com.yahoo.prelude.Location; -import com.yahoo.prelude.fastsearch.DocumentDatabase; -import com.yahoo.prelude.fastsearch.FastHit; -import com.yahoo.prelude.fastsearch.TimeoutException; -import com.yahoo.search.Query; -import com.yahoo.search.Result; -import com.yahoo.search.dispatch.FillInvoker; -import com.yahoo.search.dispatch.rpc.Client.GetDocsumsResponse; -import com.yahoo.search.query.SessionId; -import com.yahoo.search.result.ErrorMessage; -import com.yahoo.search.result.Hit; -import com.yahoo.slime.ArrayTraverser; -import com.yahoo.slime.BinaryFormat; -import com.yahoo.slime.Cursor; -import com.yahoo.slime.Slime; - -import java.util.Iterator; -import java.util.List; -import java.util.Map; -import java.util.concurrent.BlockingQueue; -import java.util.concurrent.LinkedBlockingQueue; -import java.util.concurrent.TimeUnit; -import java.util.logging.Level; -import java.util.logging.Logger; - -/** - * {@link FillInvoker} implementation using RPC - * - * @author bratseth - * @author ollivir - */ -public class RpcFillInvoker extends FillInvoker { - private static final Logger log = Logger.getLogger(RpcFillInvoker.class.getName()); - - private final DocumentDatabase documentDb; - private final RpcResourcePool resourcePool; - private GetDocsumsResponseReceiver responseReceiver; - - RpcFillInvoker(RpcResourcePool resourcePool, DocumentDatabase documentDb) { - this.documentDb = documentDb; - this.resourcePool = resourcePool; - } - - @Override - protected void sendFillRequest(Result result, String summaryClass) { - ListMap<Integer, FastHit> hitsByNode = hitsByNode(result); - Query query = result.getQuery(); - - CompressionType compression = CompressionType - .valueOf(query.properties().getString(RpcResourcePool.dispatchCompression, "LZ4").toUpperCase()); - - if (query.getTraceLevel() >= 3) { - query.trace("Sending " + hitsByNode.size() + " summary fetch RPC requests", 3); - query.trace("RpcSlime: Not resending query during document summary fetching", 3); - } - - responseReceiver = new GetDocsumsResponseReceiver(hitsByNode.size(), resourcePool.compressor(), result); - for (Map.Entry<Integer, List<FastHit>> nodeHits : hitsByNode.entrySet()) { - sendGetDocsumsRequest(nodeHits.getKey(), nodeHits.getValue(), summaryClass, compression, result, responseReceiver); - } - } - - @Override - protected void getFillResults(Result result, String summaryClass) { - try { - responseReceiver.processResponses(result.getQuery(), summaryClass, documentDb); - result.hits().setSorted(false); - result.analyzeHits(); - } catch (TimeoutException e) { - result.hits().addError(ErrorMessage.createTimeout("Summary data is incomplete: " + e.getMessage())); - } - } - - @Override - protected void release() { - // nothing to release - } - - /** Return a map of hits by their search node (partition) id */ - private static ListMap<Integer, FastHit> hitsByNode(Result result) { - ListMap<Integer, FastHit> hitsByNode = new ListMap<>(); - for (Iterator<Hit> i = result.hits().unorderedDeepIterator(); i.hasNext();) { - Hit h = i.next(); - if (!(h instanceof FastHit)) - continue; - FastHit hit = (FastHit) h; - - hitsByNode.put(hit.getDistributionKey(), hit); - } - return hitsByNode; - } - - /** Send a getDocsums request to a node. Responses will be added to the given receiver. */ - private void sendGetDocsumsRequest(int nodeId, List<FastHit> hits, String summaryClass, CompressionType compression, - Result result, GetDocsumsResponseReceiver responseReceiver) { - Client.NodeConnection node = resourcePool.getConnection(nodeId); - if (node == null) { - String error = "Could not fill hits from unknown node " + nodeId; - responseReceiver.receive(Client.ResponseOrError.fromError(error)); - result.hits().addError(ErrorMessage.createEmptyDocsums(error)); - log.warning("Got hits with partid " + nodeId + ", which is not included in the current dispatch config"); - return; - } - - Query query = result.getQuery(); - String rankProfile = query.getRanking().getProfile(); - byte[] serializedSlime = BinaryFormat - .encode(toSlime(rankProfile, summaryClass, query.getModel().getDocumentDb(), - query.getSessionId(), query.getRanking().getLocation(), hits)); - double timeoutSeconds = ((double) query.getTimeLeft() - 3.0) / 1000.0; - Compressor.Compression compressionResult = resourcePool.compress(query, serializedSlime); - node.getDocsums(hits, compressionResult.type(), serializedSlime.length, compressionResult.data(), responseReceiver, timeoutSeconds); - } - - static private Slime toSlime(String rankProfile, String summaryClass, String docType, SessionId sessionId, Location location, List<FastHit> hits) { - Slime slime = new Slime(); - Cursor root = slime.setObject(); - if (summaryClass != null) { - root.setString("class", summaryClass); - } - if (sessionId != null) { - root.setData("sessionid", sessionId.asUtf8String().getBytes()); - } - if (docType != null) { - root.setString("doctype", docType); - } - if (rankProfile != null) { - root.setString("ranking", rankProfile); - } - if (location != null) { - root.setString("location", location.backendString()); - } - Cursor gids = root.setArray("gids"); - for (FastHit hit : hits) { - gids.addData(hit.getRawGlobalId()); - } - return slime; - } - - /** Receiver of the responses to a set of getDocsums requests */ - public static class GetDocsumsResponseReceiver { - - private final BlockingQueue<Client.ResponseOrError<GetDocsumsResponse>> responses; - private final Compressor compressor; - private final Result result; - - /** Whether we have already logged/notified about an error - to avoid spamming */ - private boolean hasReportedError = false; - - /** The number of responses we should receive (and process) before this is complete */ - private int outstandingResponses; - - GetDocsumsResponseReceiver(int requestCount, Compressor compressor, Result result) { - this.compressor = compressor; - responses = new LinkedBlockingQueue<>(requestCount); - outstandingResponses = requestCount; - this.result = result; - } - - /** Called by a thread belonging to the client when a valid response becomes available */ - public void receive(Client.ResponseOrError<GetDocsumsResponse> response) { - responses.add(response); - } - - private void throwTimeout() throws TimeoutException { - throw new TimeoutException("Timed out waiting for summary data. " + outstandingResponses + " responses outstanding."); - } - - /** - * Call this from the dispatcher thread to initiate and complete processing of responses. - * This will block until all responses are available and processed, or to timeout. - */ - void processResponses(Query query, String summaryClass, DocumentDatabase documentDb) throws TimeoutException { - try { - int skippedHits = 0; - while (outstandingResponses > 0) { - long timeLeftMs = query.getTimeLeft(); - if (timeLeftMs <= 0) { - throwTimeout(); - } - Client.ResponseOrError<GetDocsumsResponse> response = responses.poll(timeLeftMs, TimeUnit.MILLISECONDS); - if (response == null) - throwTimeout(); - skippedHits += processResponse(response, summaryClass, documentDb); - outstandingResponses--; - } - if (skippedHits != 0) { - result.hits().addError(com.yahoo.search.result.ErrorMessage.createEmptyDocsums("Missing hit summary data for summary " + - summaryClass + " for " + skippedHits + " hits")); - } - } - catch (InterruptedException e) { - // TODO: Add error - } - } - - private int processResponse(Client.ResponseOrError<GetDocsumsResponse> responseOrError, - String summaryClass, - DocumentDatabase documentDb) { - if (responseOrError.error().isPresent()) { - if (hasReportedError) return 0; - String error = responseOrError.error().get(); - result.hits().addError(ErrorMessage.createBackendCommunicationError(error)); - log.log(Level.WARNING, "Error fetching summary data: "+ error); - } - else { - Client.GetDocsumsResponse response = responseOrError.response().get(); - CompressionType compression = CompressionType.valueOf(response.compression()); - byte[] slimeBytes = compressor.decompress(response.compressedSlimeBytes(), compression, response.uncompressedSize()); - return fill(response.hitsContext(), summaryClass, documentDb, slimeBytes); - } - return 0; - } - - private void addErrors(com.yahoo.slime.Inspector errors) { - errors.traverse((ArrayTraverser) (int index, com.yahoo.slime.Inspector value) -> { - int errorCode = ("timeout".equalsIgnoreCase(value.field("type").asString())) - ? Error.TIMEOUT.code - : Error.UNSPECIFIED.code; - result.hits().addError(new ErrorMessage(errorCode, - value.field("message").asString(), value.field("details").asString())); - }); - } - - private int fill(List<FastHit> hits, String summaryClass, DocumentDatabase documentDb, byte[] slimeBytes) { - com.yahoo.slime.Inspector root = BinaryFormat.decode(slimeBytes).get(); - com.yahoo.slime.Inspector errors = root.field("errors"); - boolean hasErrors = errors.valid() && (errors.entries() > 0); - if (hasErrors) { - addErrors(errors); - } - - Inspector summaries = new SlimeAdapter(root.field("docsums")); - if ( ! summaries.valid()) - return 0; // No summaries; Perhaps we requested a non-existing summary class - int skippedHits = 0; - for (int i = 0; i < hits.size(); i++) { - Inspector summary = summaries.entry(i).field("docsum"); - if (summary.valid()) { - hits.get(i).setField(Hit.SDDOCNAME_FIELD, documentDb.getName()); - hits.get(i).addSummary(documentDb.getDocsumDefinitionSet().getDocsum(summaryClass), summary); - hits.get(i).setFilled(summaryClass); - } else { - skippedHits++; - } - } - return skippedHits; - } - - } -} diff --git a/container-search/src/main/java/com/yahoo/search/dispatch/rpc/RpcInvokerFactory.java b/container-search/src/main/java/com/yahoo/search/dispatch/rpc/RpcInvokerFactory.java index ba68847e0ab..25514ae4a23 100644 --- a/container-search/src/main/java/com/yahoo/search/dispatch/rpc/RpcInvokerFactory.java +++ b/container-search/src/main/java/com/yahoo/search/dispatch/rpc/RpcInvokerFactory.java @@ -43,13 +43,7 @@ public class RpcInvokerFactory extends InvokerFactory { Query query = result.getQuery(); boolean summaryNeedsQuery = searcher.summaryNeedsQuery(query); - return new RpcProtobufFillInvoker(rpcResourcePool, searcher.getDocumentDatabase(query), searcher.getServerId(), summaryNeedsQuery); } - // for testing - public FillInvoker createFillInvoker(DocumentDatabase documentDb) { - return new RpcFillInvoker(rpcResourcePool, documentDb); - } - } diff --git a/container-search/src/test/java/com/yahoo/search/dispatch/rpc/FillTestCase.java b/container-search/src/test/java/com/yahoo/search/dispatch/rpc/FillTestCase.java deleted file mode 100644 index 288167022d8..00000000000 --- a/container-search/src/test/java/com/yahoo/search/dispatch/rpc/FillTestCase.java +++ /dev/null @@ -1,209 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.search.dispatch.rpc; - -import com.yahoo.prelude.fastsearch.DocsumDefinition; -import com.yahoo.prelude.fastsearch.DocsumDefinitionSet; -import com.yahoo.prelude.fastsearch.DocsumField; -import com.yahoo.prelude.fastsearch.DocumentDatabase; -import com.yahoo.prelude.fastsearch.FastHit; -import com.yahoo.search.Query; -import com.yahoo.search.Result; - -import org.junit.Test; - -import java.util.ArrayList; -import java.util.Collections; -import java.util.HashMap; -import java.util.List; -import java.util.Map; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertNull; - -/** - * Tests using a dispatcher to fill a result - * - * @author bratseth - */ -public class FillTestCase { - - private MockClient client = new MockClient(); - - @Test - public void testFilling() { - Map<Integer, Client.NodeConnection> nodes = new HashMap<>(); - nodes.put(0, client.createConnection("host0", 123)); - nodes.put(1, client.createConnection("host1", 123)); - nodes.put(2, client.createConnection("host2", 123)); - RpcResourcePool rpcResourcePool = new RpcResourcePool(nodes); - RpcInvokerFactory factory = new RpcInvokerFactory(rpcResourcePool, null); - - Query query = new Query(); - Result result = new Result(query); - result.hits().add(createHit(0, 0)); - result.hits().add(createHit(2, 1)); - result.hits().add(createHit(1, 2)); - result.hits().add(createHit(2, 3)); - result.hits().add(createHit(0, 4)); - - client.setDocsumReponse("host0", 0, "summaryClass1", map("field1", "s.0.0", "field2", 0)); - client.setDocsumReponse("host2", 1, "summaryClass1", map("field1", "s.2.1", "field2", 1)); - client.setDocsumReponse("host1", 2, "summaryClass1", map("field1", "s.1.2", "field2", 2)); - client.setDocsumReponse("host2", 3, "summaryClass1", map("field1", "s.2.3", "field2", 3)); - client.setDocsumReponse("host0", 4, "summaryClass1", map("field1", "s.0.4", "field2", 4)); - - factory.createFillInvoker(db()).fill(result, "summaryClass1"); - - assertEquals("s.0.0", result.hits().get("hit:0").getField("field1").toString()); - assertEquals("s.2.1", result.hits().get("hit:1").getField("field1").toString()); - assertEquals("s.1.2", result.hits().get("hit:2").getField("field1").toString()); - assertEquals("s.2.3", result.hits().get("hit:3").getField("field1").toString()); - assertEquals("s.0.4", result.hits().get("hit:4").getField("field1").toString()); - assertEquals(0L, result.hits().get("hit:0").getField("field2")); - assertEquals(1L, result.hits().get("hit:1").getField("field2")); - assertEquals(2L, result.hits().get("hit:2").getField("field2")); - assertEquals(3L, result.hits().get("hit:3").getField("field2")); - assertEquals(4L, result.hits().get("hit:4").getField("field2")); - } - - @Test - public void testEmptyHits() { - Map<Integer, Client.NodeConnection> nodes = new HashMap<>(); - nodes.put(0, client.createConnection("host0", 123)); - nodes.put(1, client.createConnection("host1", 123)); - nodes.put(2, client.createConnection("host2", 123)); - RpcResourcePool rpcResourcePool = new RpcResourcePool(nodes); - RpcInvokerFactory factory = new RpcInvokerFactory(rpcResourcePool, null); - - Query query = new Query(); - Result result = new Result(query); - result.hits().add(createHit(0, 0)); - result.hits().add(createHit(2, 1)); - result.hits().add(createHit(1, 2)); - result.hits().add(createHit(2, 3)); - result.hits().add(createHit(0, 4)); - - client.setDocsumReponse("host0", 0, "summaryClass1", map("field1", "s.0.0", "field2", 0)); - client.setDocsumReponse("host2", 1, "summaryClass1", map("field1", "s.2.1", "field2", 1)); - client.setDocsumReponse("host1", 2, "summaryClass1", new HashMap<>()); - client.setDocsumReponse("host2", 3, "summaryClass1", map("field1", "s.2.3", "field2", 3)); - client.setDocsumReponse("host0", 4, "summaryClass1", new HashMap<>()); - - factory.createFillInvoker(db()).fill(result, "summaryClass1"); - - assertEquals("s.0.0", result.hits().get("hit:0").getField("field1").toString()); - assertEquals("s.2.1", result.hits().get("hit:1").getField("field1").toString()); - assertNull(result.hits().get("hit:2").getField("field1")); - assertEquals("s.2.3", result.hits().get("hit:3").getField("field1").toString()); - assertNull(result.hits().get("hit:4").getField("field1")); - - assertEquals(0L, result.hits().get("hit:0").getField("field2")); - assertEquals(1L, result.hits().get("hit:1").getField("field2")); - assertNull(result.hits().get("hit:2").getField("field2")); - assertEquals(3L, result.hits().get("hit:3").getField("field2")); - assertNull(result.hits().get("hit:4").getField("field2")); - - assertNull(result.hits().getError()); - } - - @Test - public void testMissingHits() { - Map<Integer, Client.NodeConnection> nodes = new HashMap<>(); - nodes.put(0, client.createConnection("host0", 123)); - nodes.put(1, client.createConnection("host1", 123)); - nodes.put(2, client.createConnection("host2", 123)); - RpcResourcePool rpcResourcePool = new RpcResourcePool(nodes); - RpcInvokerFactory factory = new RpcInvokerFactory(rpcResourcePool, null); - - Query query = new Query(); - Result result = new Result(query); - result.hits().add(createHit(0, 0)); - result.hits().add(createHit(2, 1)); - result.hits().add(createHit(1, 2)); - result.hits().add(createHit(2, 3)); - result.hits().add(createHit(0, 4)); - - client.setDocsumReponse("host0", 0, "summaryClass1", map("field1", "s.0.0", "field2", 0)); - client.setDocsumReponse("host2", 1, "summaryClass1", map("field1", "s.2.1", "field2", 1)); - client.setDocsumReponse("host1", 2, "summaryClass1", null); - client.setDocsumReponse("host2", 3, "summaryClass1", map("field1", "s.2.3", "field2", 3)); - client.setDocsumReponse("host0", 4, "summaryClass1", null); - - factory.createFillInvoker(db()).fill(result, "summaryClass1"); - - assertEquals("s.0.0", result.hits().get("hit:0").getField("field1").toString()); - assertEquals("s.2.1", result.hits().get("hit:1").getField("field1").toString()); - assertNull(result.hits().get("hit:2").getField("field1")); - assertEquals("s.2.3", result.hits().get("hit:3").getField("field1").toString()); - assertNull(result.hits().get("hit:4").getField("field1")); - - assertEquals(0L, result.hits().get("hit:0").getField("field2")); - assertEquals(1L, result.hits().get("hit:1").getField("field2")); - assertNull(result.hits().get("hit:2").getField("field2")); - assertEquals(3L, result.hits().get("hit:3").getField("field2")); - assertNull(result.hits().get("hit:4").getField("field2")); - - assertEquals("Missing hit summary data for summary summaryClass1 for 2 hits", result.hits().getError().getDetailedMessage()); - } - - @Test - public void testErrorHandling() { - client.setMalfunctioning(true); - - Map<Integer, Client.NodeConnection> nodes = new HashMap<>(); - nodes.put(0, client.createConnection("host0", 123)); - RpcResourcePool rpcResourcePool = new RpcResourcePool(nodes); - RpcInvokerFactory factory = new RpcInvokerFactory(rpcResourcePool, null); - - Query query = new Query(); - Result result = new Result(query); - result.hits().add(createHit(0, 0)); - - factory.createFillInvoker(db()).fill(result, "summaryClass1"); - - assertEquals("Malfunctioning", result.hits().getError().getDetailedMessage()); - } - - @Test - public void testSendingFill2UnknownNode() { - client.setMalfunctioning(true); - - Map<Integer, Client.NodeConnection> nodes = new HashMap<>(); - nodes.put(0, client.createConnection("host0", 123)); - RpcResourcePool rpcResourcePool = new RpcResourcePool(nodes); - RpcInvokerFactory factory = new RpcInvokerFactory(rpcResourcePool, null); - - Query query = new Query(); - Result result = new Result(query); - result.hits().add(createHit(0, 0)); - result.hits().add(createHit(1, 1)); - - factory.createFillInvoker(db()).fill(result, "summaryClass1"); - - assertEquals("Could not fill hits from unknown node 1", result.hits().getError().getDetailedMessage()); - } - - private DocumentDatabase db() { - List<DocsumField> fields = new ArrayList<>(); - fields.add(DocsumField.create("field1", "string")); - fields.add(DocsumField.create("field2", "int64")); - DocsumDefinitionSet docsums = new DocsumDefinitionSet(Collections.singleton(new DocsumDefinition("summaryClass1", fields))); - return new DocumentDatabase("default", docsums, Collections.emptySet()); - } - - private FastHit createHit(int sourceNodeId, int hitId) { - FastHit hit = new FastHit("hit:" + hitId, 1.0); - hit.setPartId(sourceNodeId); - hit.setDistributionKey(sourceNodeId); - hit.setGlobalId(client.globalIdFrom(hitId).getRawId()); - return hit; - } - - private Map<String, Object> map(String stringKey, String stringValue, String intKey, int intValue) { - Map<String, Object> map = new HashMap<>(); - map.put(stringKey, stringValue); - map.put(intKey, intValue); - return map; - } - -} diff --git a/container-search/src/test/java/com/yahoo/search/dispatch/rpc/MockClient.java b/container-search/src/test/java/com/yahoo/search/dispatch/rpc/MockClient.java index 8ebdfcc1a12..61971e975e5 100644 --- a/container-search/src/test/java/com/yahoo/search/dispatch/rpc/MockClient.java +++ b/container-search/src/test/java/com/yahoo/search/dispatch/rpc/MockClient.java @@ -55,46 +55,6 @@ public class MockClient implements Client { } @Override - public void getDocsums(List<FastHit> hitsContext, CompressionType compression, int uncompressedSize, byte[] compressedSlime, - RpcFillInvoker.GetDocsumsResponseReceiver responseReceiver, double timeoutSeconds) { - if (malfunctioning) { - responseReceiver.receive(ResponseOrError.fromError("Malfunctioning")); - return; - } - - Inspector request = BinaryFormat.decode(compressor.decompress(compressedSlime, compression, uncompressedSize)).get(); - String docsumClass = request.field("class").asString(); - List<Map<String, Object>> docsumsToReturn = new ArrayList<>(); - request.field("gids").traverse((ArrayTraverser) (index, gid) -> { - GlobalId docId = new GlobalId(gid.asData()); - docsumsToReturn.add(docsums.get(new DocsumKey(toString(), docId, docsumClass))); - }); - Slime responseSlime = new Slime(); - Cursor root = responseSlime.setObject(); - Cursor docsums = root.setArray("docsums"); - for (Map<String, Object> docsumFields : docsumsToReturn) { - if (docsumFields == null) continue; - - Cursor docsumItem = docsums.addObject(); - Cursor docsum = docsumItem.setObject("docsum"); - for (Map.Entry<String, Object> field : docsumFields.entrySet()) { - if (field.getValue() instanceof Integer) - docsum.setLong(field.getKey(), (Integer) field.getValue()); - else if (field.getValue() instanceof String) - docsum.setString(field.getKey(), (String) field.getValue()); - else - throw new RuntimeException(); - } - } - byte[] slimeBytes = BinaryFormat.encode(responseSlime); - CompressionType responseCompressionType = compression == CompressionType.INCOMPRESSIBLE ? CompressionType.NONE : compression; - Compressor.Compression compressionResult = compressor.compress(responseCompressionType, slimeBytes); - GetDocsumsResponse response = new GetDocsumsResponse(compressionResult.type().getCode(), slimeBytes.length, - compressionResult.data(), hitsContext); - responseReceiver.receive(ResponseOrError.fromResponse(response)); - } - - @Override public void request(String rpcMethod, CompressionType compression, int uncompressedLength, byte[] compressedPayload, ResponseReceiver responseReceiver, double timeoutSeconds) { if (malfunctioning) { diff --git a/container-search/src/test/java/com/yahoo/search/dispatch/rpc/MockRpcResourcePoolBuilder.java b/container-search/src/test/java/com/yahoo/search/dispatch/rpc/MockRpcResourcePoolBuilder.java index dbef9d819e8..23d6ae6bf2b 100644 --- a/container-search/src/test/java/com/yahoo/search/dispatch/rpc/MockRpcResourcePoolBuilder.java +++ b/container-search/src/test/java/com/yahoo/search/dispatch/rpc/MockRpcResourcePoolBuilder.java @@ -5,7 +5,6 @@ import com.yahoo.compress.CompressionType; import com.yahoo.prelude.fastsearch.FastHit; import com.yahoo.search.dispatch.rpc.Client.NodeConnection; import com.yahoo.search.dispatch.rpc.Client.ResponseReceiver; -import com.yahoo.search.dispatch.rpc.RpcFillInvoker.GetDocsumsResponseReceiver; import java.util.HashMap; import java.util.List; @@ -35,12 +34,6 @@ public class MockRpcResourcePoolBuilder { } @Override - public void getDocsums(List<FastHit> hits, CompressionType compression, int uncompressedLength, byte[] compressedSlime, - GetDocsumsResponseReceiver responseReceiver, double timeoutSeconds) { - responseReceiver.receive(Client.ResponseOrError.fromError("getDocsums(..) attempted for node " + key)); - } - - @Override public void request(String rpcMethod, CompressionType compression, int uncompressedLength, byte[] compressedPayload, ResponseReceiver responseReceiver, double timeoutSeconds) { responseReceiver.receive(Client.ResponseOrError.fromError("request('"+rpcMethod+"', ..) attempted for node " + key)); diff --git a/container-search/src/test/java/com/yahoo/search/dispatch/rpc/RpcSearchInvokerTest.java b/container-search/src/test/java/com/yahoo/search/dispatch/rpc/RpcSearchInvokerTest.java index 27fc3f85136..45ad361a214 100644 --- a/container-search/src/test/java/com/yahoo/search/dispatch/rpc/RpcSearchInvokerTest.java +++ b/container-search/src/test/java/com/yahoo/search/dispatch/rpc/RpcSearchInvokerTest.java @@ -9,7 +9,6 @@ import com.yahoo.prelude.fastsearch.FastHit; import com.yahoo.prelude.fastsearch.VespaBackEndSearcher; import com.yahoo.search.Query; import com.yahoo.search.Result; -import com.yahoo.search.dispatch.rpc.RpcFillInvoker.GetDocsumsResponseReceiver; import com.yahoo.search.dispatch.searchcluster.Node; import com.yahoo.search.searchchain.Execution; import org.junit.Test; @@ -85,12 +84,6 @@ public class RpcSearchInvokerTest { public NodeConnection createConnection(String hostname, int port) { return new NodeConnection() { @Override - public void getDocsums(List<FastHit> hits, CompressionType compression, int uncompressedLength, byte[] compressedSlime, - GetDocsumsResponseReceiver responseReceiver, double timeoutSeconds) { - fail("Unexpected call"); - } - - @Override public void request(String rpcMethod, CompressionType compression, int uncompressedLength, byte[] compressedPayload, ResponseReceiver responseReceiver, double timeoutSeconds) { compressionTypeHolder.set(compression); diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java index 1774694853b..59d18fcf17b 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java @@ -470,10 +470,10 @@ public class ApplicationApiHandler extends AuditLoggingRequestHandler { TenantInfo mergedInfo = TenantInfo.EMPTY .withName(getString(insp.field("name"), oldInfo.name())) .withEmail(getString(insp.field("email"), oldInfo.email())) - .withWebsite(getString(insp.field("website"), oldInfo.email())) + .withWebsite(getString(insp.field("website"), oldInfo.website())) .withInvoiceEmail(getString(insp.field("invoiceEmail"), oldInfo.invoiceEmail())) .withContactName(getString(insp.field("contactName"), oldInfo.contactName())) - .withContactEmail(getString(insp.field("contactEmail"), oldInfo.contactName())) + .withContactEmail(getString(insp.field("contactEmail"), oldInfo.contactEmail())) .withAddress(updateTenantInfoAddress(insp.field("address"), oldInfo.address())) .withBillingContact(updateTenantInfoBillingContact(insp.field("billingContact"), oldInfo.billingContact())); diff --git a/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/FileDownloader.java b/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/FileDownloader.java index 1d638a427f9..0678509dc68 100644 --- a/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/FileDownloader.java +++ b/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/FileDownloader.java @@ -11,7 +11,6 @@ import com.yahoo.yolean.Exceptions; import java.io.File; import java.time.Duration; import java.util.Map; -import java.util.Objects; import java.util.Optional; import java.util.concurrent.CompletableFuture; import java.util.concurrent.ExecutionException; @@ -37,6 +36,7 @@ public class FileDownloader implements AutoCloseable { private final Supervisor supervisor; private final File downloadDirectory; private final Duration timeout; + private final Duration sleepBetweenRetries; private final FileReferenceDownloader fileReferenceDownloader; private final Downloads downloads = new Downloads(); @@ -61,7 +61,8 @@ public class FileDownloader implements AutoCloseable { this.supervisor = supervisor; this.downloadDirectory = downloadDirectory; this.timeout = timeout; - // Needed to receive RPC receiveFile* calls from server after asking for files + this.sleepBetweenRetries = sleepBetweenRetries; + // Needed to receive RPC receiveFile* calls from server after starting download of file reference new FileReceiver(supervisor, downloads, downloadDirectory); this.fileReferenceDownloader = new FileReferenceDownloader(connectionPool, downloads, timeout, sleepBetweenRetries); } @@ -83,12 +84,11 @@ public class FileDownloader implements AutoCloseable { Future<Optional<File>> getFutureFile(FileReferenceDownload fileReferenceDownload) { FileReference fileReference = fileReferenceDownload.fileReference(); - Objects.requireNonNull(fileReference, "file reference cannot be null"); Optional<File> file = getFileFromFileSystem(fileReference); return (file.isPresent()) ? CompletableFuture.completedFuture(file) - : download(fileReferenceDownload); + : startDownload(fileReferenceDownload); } public Map<FileReference, Double> downloadStatus() { return downloads.downloadStatus(); } @@ -119,18 +119,24 @@ public class FileDownloader implements AutoCloseable { } } + boolean fileReferenceExists(FileReference fileReference) { + return getFileFromFileSystem(fileReference).isPresent(); + } + boolean isDownloading(FileReference fileReference) { return downloads.get(fileReference).isPresent(); } - /** Start a download, don't wait for result */ + /** Start a download if needed, don't wait for result */ public void downloadIfNeeded(FileReferenceDownload fileReferenceDownload) { - getFutureFile(fileReferenceDownload); + if (fileReferenceExists(fileReferenceDownload.fileReference())) return; + + startDownload(fileReferenceDownload); } - /** Download, the future returned will be complete()d by receiving method in {@link FileReceiver} */ - private synchronized Future<Optional<File>> download(FileReferenceDownload fileReferenceDownload) { - return fileReferenceDownloader.download(fileReferenceDownload); + /** Start downloading, the future returned will be complete()d by receiving method in {@link FileReceiver} */ + private synchronized Future<Optional<File>> startDownload(FileReferenceDownload fileReferenceDownload) { + return fileReferenceDownloader.startDownload(fileReferenceDownload); } public void close() { diff --git a/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/FileReferenceDownload.java b/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/FileReferenceDownload.java index 470d94ce749..21e35bf67af 100644 --- a/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/FileReferenceDownload.java +++ b/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/FileReferenceDownload.java @@ -5,6 +5,7 @@ package com.yahoo.vespa.filedistribution; import com.yahoo.config.FileReference; import java.io.File; +import java.util.Objects; import java.util.Optional; import java.util.concurrent.CompletableFuture; @@ -22,6 +23,7 @@ public class FileReferenceDownload { } public FileReferenceDownload(FileReference fileReference, boolean downloadFromOtherSourceIfNotFound, String client) { + Objects.requireNonNull(fileReference, "file reference cannot be null"); this.fileReference = fileReference; this.future = new CompletableFuture<>(); this.downloadFromOtherSourceIfNotFound = downloadFromOtherSourceIfNotFound; diff --git a/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/FileReferenceDownloader.java b/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/FileReferenceDownloader.java index 952684b7b0b..740bf23796f 100644 --- a/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/FileReferenceDownloader.java +++ b/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/FileReferenceDownloader.java @@ -11,7 +11,6 @@ import com.yahoo.vespa.config.ConnectionPool; import java.io.File; import java.time.Duration; -import java.time.Instant; import java.util.Optional; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; @@ -50,39 +49,29 @@ public class FileReferenceDownloader { this.rpcTimeout = Duration.ofSeconds(timeoutString == null ? 30 : Integer.parseInt(timeoutString)); } - private void startDownload(FileReferenceDownload fileReferenceDownload) { + private void waitUntilDownloadStarted(FileReferenceDownload fileReferenceDownload) { FileReference fileReference = fileReferenceDownload.fileReference(); - Instant end = Instant.now().plus(downloadTimeout); - boolean downloadStarted = false; int retryCount = 0; do { - try { - if (startDownloadRpc(fileReferenceDownload, retryCount)) { - downloadStarted = true; - } else { - retryCount++; - long sleepTime = Math.min(sleepBetweenRetries.toMillis() * retryCount, - Math.max(0, Duration.between(Instant.now(), end).toMillis())); - Thread.sleep(sleepTime); - } - } - catch (InterruptedException e) { /* ignored */} - } while (Instant.now().isBefore(end) && !downloadStarted); + if (startDownloadRpc(fileReferenceDownload, retryCount)) + return; - if ( !downloadStarted) { - fileReferenceDownload.future().completeExceptionally(new RuntimeException("Failed getting file reference '" + fileReference.value() + "'")); - downloads.remove(fileReference); - } + try { Thread.sleep(sleepBetweenRetries.toMillis()); } catch (InterruptedException e) { /* ignored */} + retryCount++; + } while (retryCount < 5); + + fileReferenceDownload.future().completeExceptionally(new RuntimeException("Failed getting " + fileReference)); + downloads.remove(fileReference); } - Future<Optional<File>> download(FileReferenceDownload fileReferenceDownload) { + Future<Optional<File>> startDownload(FileReferenceDownload fileReferenceDownload) { FileReference fileReference = fileReferenceDownload.fileReference(); Optional<FileReferenceDownload> inProgress = downloads.get(fileReference); if (inProgress.isPresent()) return inProgress.get().future(); log.log(Level.FINE, () -> "Will download file reference '" + fileReference.value() + "' with timeout " + downloadTimeout); downloads.add(fileReferenceDownload); - downloadExecutor.submit(() -> startDownload(fileReferenceDownload)); + downloadExecutor.submit(() -> waitUntilDownloadStarted(fileReferenceDownload)); return fileReferenceDownload.future(); } @@ -99,7 +88,7 @@ public class FileReferenceDownloader { double timeoutSecs = (double) rpcTimeout.getSeconds(); timeoutSecs += retryCount * 10.0; connection.invokeSync(request, timeoutSecs); - Level logLevel = (retryCount > 5 ? Level.INFO : Level.FINE); + Level logLevel = (retryCount > 3 ? Level.INFO : Level.FINE); if (validateResponse(request)) { log.log(Level.FINE, () -> "Request callback, OK. Req: " + request + "\nSpec: " + connection + ", retry count " + retryCount); if (request.returnValues().get(0).asInt32() == 0) { diff --git a/flags/src/main/java/com/yahoo/vespa/flags/Flags.java b/flags/src/main/java/com/yahoo/vespa/flags/Flags.java index 9393b0305b0..9ca143a0425 100644 --- a/flags/src/main/java/com/yahoo/vespa/flags/Flags.java +++ b/flags/src/main/java/com/yahoo/vespa/flags/Flags.java @@ -309,14 +309,6 @@ public class Flags { ZONE_ID, TENANT_ID ); - public static final UnboundBooleanFlag USE_APPLICATION_LOCK_IN_MAINTENANCE_DEPLOYMENT = defineFeatureFlag( - "use-application-lock-in-maintenance-deployment", true, - List.of("hmusum"), "2021-09-16", "2021-11-01", - "Whether to use application node repository lock when doing maintenance deployment.", - "Takes effect immediately", - APPLICATION_ID - ); - public static final UnboundBooleanFlag DELETE_UNMAINTAINED_CERTIFICATES = defineFeatureFlag( "delete-unmaintained-certificates", false, List.of("andreer"), "2021-09-23", "2021-11-11", diff --git a/metrics/src/vespa/metrics/metricvalueset.h b/metrics/src/vespa/metrics/metricvalueset.h index 82c01343472..bb2a7409ce7 100644 --- a/metrics/src/vespa/metrics/metricvalueset.h +++ b/metrics/src/vespa/metrics/metricvalueset.h @@ -68,7 +68,7 @@ public: ValueClass getValues() const; /** - * Get the current values from the metric. This function should not be + * Set the current values for the metric. This function should not be * called in parallel. Only call it from a single thread or use external * locking. If it returns false, it means the metric have just been reset. * In which case, redo getValues(), apply the update again, and call diff --git a/metrics/src/vespa/metrics/valuemetric.h b/metrics/src/vespa/metrics/valuemetric.h index 3470aa067e0..9d5f0a82735 100644 --- a/metrics/src/vespa/metrics/valuemetric.h +++ b/metrics/src/vespa/metrics/valuemetric.h @@ -37,7 +37,6 @@ protected: void logWarning(const char* msg, const char *op) const; void logNonFiniteValueWarning() const; - void sendLogEvent(Metric::String name, double value) const; }; template<typename AvgVal, typename TotVal, bool SumOnAdd> @@ -86,7 +85,7 @@ public: ~ValueMetric(); MetricValueClass::UP getValues() const override { - return MetricValueClass::UP(new Values(_values.getValues())); + return std::make_unique<Values>(_values.getValues()); } void unsetOnZeroValue() { _values.setFlag(UNSET_ON_ZERO_VALUE); } diff --git a/metrics/src/vespa/metrics/valuemetric.hpp b/metrics/src/vespa/metrics/valuemetric.hpp index a9522dcec29..5e0ef95e9e5 100644 --- a/metrics/src/vespa/metrics/valuemetric.hpp +++ b/metrics/src/vespa/metrics/valuemetric.hpp @@ -26,7 +26,7 @@ ValueMetric<AvgVal, TotVal, SumOnAdd>::ValueMetric( {} template<typename AvgVal, typename TotVal, bool SumOnAdd> -ValueMetric<AvgVal, TotVal, SumOnAdd>::~ValueMetric() { } +ValueMetric<AvgVal, TotVal, SumOnAdd>::~ValueMetric() = default; template<typename AvgVal, typename TotVal, bool SumOnAdd> void ValueMetric<AvgVal, TotVal, SumOnAdd>::inc(AvgVal incVal) @@ -239,8 +239,7 @@ ValueMetric<AvgVal, TotVal, SumOnAdd>::getDoubleValue(stringref id) const template<typename AvgVal, typename TotVal, bool SumOnAdd> void -ValueMetric<AvgVal, TotVal, SumOnAdd>::addMemoryUsage( - MemoryConsumption& mc) const +ValueMetric<AvgVal, TotVal, SumOnAdd>::addMemoryUsage(MemoryConsumption& mc) const { ++mc._valueMetricCount; mc._valueMetricValues += _values.getMemoryUsageAllocatedInternally(); diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/servicedump/VespaServiceDumperImpl.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/servicedump/VespaServiceDumperImpl.java index 59f5e0f3f40..0a9496be0a6 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/servicedump/VespaServiceDumperImpl.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/servicedump/VespaServiceDumperImpl.java @@ -100,7 +100,9 @@ public class VespaServiceDumperImpl implements VespaServiceDumper { unixPathDirectory.deleteRecursively(); } context.log(log, Level.INFO, "Creating '" + unixPathDirectory +"'."); - unixPathDirectory.createDirectory("rwxrwxrwx"); + unixPathDirectory.createDirectory("rwxr-x---") + .setOwner(context.userNamespace().vespaUser()) + .setGroup(context.userNamespace().vespaGroup()); URI destination = serviceDumpDestination(nodeSpec, createDumpId(request)); ProducerContext producerCtx = new ProducerContext(context, directory, request); List<Artifact> producedArtifacts = new ArrayList<>(); diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/fs/ContainerFileSystemProvider.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/fs/ContainerFileSystemProvider.java index a44f90b164b..909c6c9cbc1 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/fs/ContainerFileSystemProvider.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/fs/ContainerFileSystemProvider.java @@ -114,7 +114,6 @@ class ContainerFileSystemProvider extends FileSystemProvider { // Only called when both 'source' and 'target' have 'this' as the FS provider Path targetPathOnHost = pathOnHost(target); provider(targetPathOnHost).copy(pathOnHost(source), targetPathOnHost, options); - fixOwnerToContainerRoot(toContainerPath(target)); } @Override @@ -122,7 +121,6 @@ class ContainerFileSystemProvider extends FileSystemProvider { // Only called when both 'source' and 'target' have 'this' as the FS provider Path targetPathOnHost = pathOnHost(target); provider(targetPathOnHost).move(pathOnHost(source), targetPathOnHost, options); - fixOwnerToContainerRoot(toContainerPath(target)); } @Override diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/fs/ContainerFileSystemTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/fs/ContainerFileSystemTest.java index a5fc6a1373f..4e85052a176 100644 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/fs/ContainerFileSystemTest.java +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/fs/ContainerFileSystemTest.java @@ -85,7 +85,7 @@ class ContainerFileSystemTest { new UnixPath(destination).setOwnerId(500).setGroupId(200); ContainerPath destination2 = ContainerPath.fromPathInContainer(containerFs, Path.of("/dest2")); Files.copy(destination, destination2, StandardCopyOption.COPY_ATTRIBUTES, StandardCopyOption.REPLACE_EXISTING); - assertOwnership(destination2, 0, 0, 10000, 11000); + assertOwnership(destination2, 500, 200, 10500, 11200); } @Test @@ -116,7 +116,7 @@ class ContainerFileSystemTest { new UnixPath(destination).setOwnerId(500).setGroupId(200); ContainerPath destination2 = ContainerPath.fromPathInContainer(containerFs, Path.of("/dest2")); Files.move(destination, destination2, StandardCopyOption.COPY_ATTRIBUTES, StandardCopyOption.REPLACE_EXISTING); - assertOwnership(destination2, 0, 0, 10000, 11000); + assertOwnership(destination2, 500, 200, 10500, 11200); } @Test diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/MaintenanceDeployment.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/MaintenanceDeployment.java index ff696e727fa..1f9a37d517a 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/MaintenanceDeployment.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/MaintenanceDeployment.java @@ -24,8 +24,6 @@ import java.util.function.Supplier; import java.util.logging.Level; import java.util.logging.Logger; -import static com.yahoo.vespa.flags.Flags.USE_APPLICATION_LOCK_IN_MAINTENANCE_DEPLOYMENT; - /** * A wrapper of a deployment suitable for maintenance. * This is a single-use, single-thread object. @@ -50,10 +48,7 @@ class MaintenanceDeployment implements Closeable { this.application = application; this.metric = metric; - Optional<Mutex> lock = USE_APPLICATION_LOCK_IN_MAINTENANCE_DEPLOYMENT.bindTo(nodeRepository.flagSource()).value() - ? tryLock(application, nodeRepository) - : Optional.of(() -> { }); - + Optional<Mutex> lock = tryLock(application, nodeRepository); try { deployment = tryDeployment(lock, application, deployer, nodeRepository); this.lock = lock; diff --git a/persistence/src/vespa/persistence/dummyimpl/dummypersistence.cpp b/persistence/src/vespa/persistence/dummyimpl/dummypersistence.cpp index 66f03edafa2..6e4f38fe564 100644 --- a/persistence/src/vespa/persistence/dummyimpl/dummypersistence.cpp +++ b/persistence/src/vespa/persistence/dummyimpl/dummypersistence.cpp @@ -15,7 +15,6 @@ #include <vespa/vespalib/util/exceptions.h> #include <vespa/vespalib/util/idestructorcallback.h> #include <vespa/vespalib/stllike/hash_map.hpp> -#include <algorithm> #include <cassert> #include <vespa/log/log.h> @@ -74,11 +73,9 @@ BucketContent::getBucketInfo() const uint32_t totalSize = 0; uint32_t checksum = 0; - for (std::vector<BucketEntry>::const_iterator - it = _entries.begin(); it != _entries.end(); ++it) - { - const DocEntry& entry(*it->entry); - const GlobalId& gid(it->gid); + for (const BucketEntry & bucketEntry : _entries) { + const DocEntry& entry(*bucketEntry.entry); + const GlobalId& gid(bucketEntry.gid); GidMapType::const_iterator gidIt(_gidMap.find(gid)); assert(gidIt != _gidMap.end()); @@ -94,7 +91,7 @@ BucketContent::getBucketInfo() const ++unique; uniqueSize += entry.getSize(); - checksum ^= computeEntryChecksum(*it); + checksum ^= computeEntryChecksum(bucketEntry); } if (!unique) { checksum = 0; @@ -115,12 +112,6 @@ BucketContent::getBucketInfo() const } namespace { -struct HasDocId { - const DocumentId &_did; - HasDocId(const DocumentId &did) : _did(did) {} - bool operator()(const DocEntry &entry) - { return *entry.getDocumentId() == _did; } -}; struct TimestampLess { bool operator()(const BucketEntry &bucketEntry, Timestamp t) @@ -129,15 +120,6 @@ struct TimestampLess { { return t < bucketEntry.entry->getTimestamp(); } }; -template <typename Iter> -typename std::iterator_traits<Iter>::value_type -dereferenceOrDefaultIfAtEnd(Iter it, Iter end) { - if (it == end) { - return typename std::iterator_traits<Iter>::value_type(); - } - return *it; -} - } // namespace bool @@ -405,9 +387,8 @@ DummyPersistence::setClusterState(BucketSpace bucketSpace, const ClusterState& c return Result(); } -Result -DummyPersistence::setActiveState(const Bucket& b, - BucketInfo::ActiveState newState) +void +DummyPersistence::setActiveStateAsync(const Bucket& b, BucketInfo::ActiveState newState, OperationComplete::UP onComplete) { DUMMYPERSISTENCE_VERIFY_INITIALIZED; LOG(debug, "setCurrentState(%s, %s)", @@ -416,11 +397,12 @@ DummyPersistence::setActiveState(const Bucket& b, assert(b.getBucketSpace() == FixedBucketSpaces::default_space()); BucketContentGuard::UP bc(acquireBucketWithLock(b)); - if (!bc.get()) { - return BucketInfoResult(Result::ErrorType::TRANSIENT_ERROR, "Bucket not found"); + if ( ! bc ) { + onComplete->onComplete(std::make_unique<BucketInfoResult>(Result::ErrorType::TRANSIENT_ERROR, "Bucket not found")); + } else { + (*bc)->setActive(newState == BucketInfo::ACTIVE); + onComplete->onComplete(std::make_unique<Result>()); } - (*bc)->setActive(newState == BucketInfo::ACTIVE); - return Result(); } BucketInfoResult @@ -442,51 +424,52 @@ DummyPersistence::getBucketInfo(const Bucket& b) const return BucketInfoResult(info); } -Result -DummyPersistence::put(const Bucket& b, Timestamp t, Document::SP doc, Context&) +void +DummyPersistence::putAsync(const Bucket& b, Timestamp t, Document::SP doc, Context&, OperationComplete::UP onComplete) { DUMMYPERSISTENCE_VERIFY_INITIALIZED; LOG(debug, "put(%s, %" PRIu64 ", %s)", - b.toString().c_str(), - uint64_t(t), - doc->getId().toString().c_str()); + b.toString().c_str(), uint64_t(t), doc->getId().toString().c_str()); assert(b.getBucketSpace() == FixedBucketSpaces::default_space()); BucketContentGuard::UP bc(acquireBucketWithLock(b)); if (!bc.get()) { - return BucketInfoResult(Result::ErrorType::TRANSIENT_ERROR, "Bucket not found"); - } - - DocEntry::SP existing = (*bc)->getEntry(t); - if (existing.get()) { - if (doc->getId() == *existing->getDocumentId()) { - return Result(); + bc.reset(); + onComplete->onComplete(std::make_unique<BucketInfoResult>(Result::ErrorType::TRANSIENT_ERROR, "Bucket not found")); + } else { + DocEntry::SP existing = (*bc)->getEntry(t); + if (existing) { + bc.reset(); + if (doc->getId() == *existing->getDocumentId()) { + onComplete->onComplete(std::make_unique<Result>()); + } else { + onComplete->onComplete(std::make_unique<Result>(Result::ErrorType::TIMESTAMP_EXISTS, + "Timestamp already existed")); + } } else { - return Result(Result::ErrorType::TIMESTAMP_EXISTS, - "Timestamp already existed"); + LOG(spam, "Inserting document %s", doc->toString(true).c_str()); + auto entry = std::make_unique<DocEntry>(t, NONE, Document::UP(doc->clone())); + (*bc)->insert(std::move(entry)); + bc.reset(); + onComplete->onComplete(std::make_unique<Result>()); } } - - LOG(spam, "Inserting document %s", doc->toString(true).c_str()); - - auto entry = std::make_unique<DocEntry>(t, NONE, Document::UP(doc->clone())); - (*bc)->insert(std::move(entry)); - return Result(); } -UpdateResult -DummyPersistence::update(const Bucket& bucket, Timestamp ts, DocumentUpdateSP upd, Context& context) +void +DummyPersistence::updateAsync(const Bucket& bucket, Timestamp ts, DocumentUpdateSP upd, Context& context, OperationComplete::UP onComplete) { GetResult getResult = get(bucket, document::AllFields(), upd->getId(), context); if (getResult.hasError()) { - return UpdateResult(getResult.getErrorCode(), getResult.getErrorMessage()); + onComplete->onComplete(std::make_unique<UpdateResult>(getResult.getErrorCode(), getResult.getErrorMessage())); + return; } - auto docToUpdate = getResult.getDocumentPtr(); Timestamp updatedTs = getResult.getTimestamp(); if (!docToUpdate) { if (!upd->getCreateIfNonExistent()) { - return UpdateResult(); + onComplete->onComplete(std::make_unique<UpdateResult>()); + return; } else { docToUpdate = std::make_shared<document::Document>(upd->getType(), upd->getId()); updatedTs = ts; @@ -498,14 +481,14 @@ DummyPersistence::update(const Bucket& bucket, Timestamp ts, DocumentUpdateSP up Result putResult = put(bucket, ts, std::move(docToUpdate), context); if (putResult.hasError()) { - return UpdateResult(putResult.getErrorCode(), putResult.getErrorMessage()); + onComplete->onComplete(std::make_unique<UpdateResult>(putResult.getErrorCode(), putResult.getErrorMessage())); + } else { + onComplete->onComplete(std::make_unique<UpdateResult>(updatedTs)); } - - return UpdateResult(updatedTs); } -RemoveResult -DummyPersistence::remove(const Bucket& b, Timestamp t, const DocumentId& did, Context&) +void +DummyPersistence::removeAsync(const Bucket& b, Timestamp t, const DocumentId& did, Context &, OperationComplete::UP onComplete) { DUMMYPERSISTENCE_VERIFY_INITIALIZED; LOG(debug, "remove(%s, %" PRIu64 ", %s)", @@ -515,19 +498,21 @@ DummyPersistence::remove(const Bucket& b, Timestamp t, const DocumentId& did, Co assert(b.getBucketSpace() == FixedBucketSpaces::default_space()); BucketContentGuard::UP bc(acquireBucketWithLock(b)); - if (!bc.get()) { - return RemoveResult(Result::ErrorType::TRANSIENT_ERROR, "Bucket not found"); - } - - DocEntry::SP entry((*bc)->getEntry(did)); - bool foundPut(entry.get() && !entry->isRemove()); - DocEntry::UP remEntry(new DocEntry(t, REMOVE_ENTRY, did)); + if ( ! bc ) { + bc.reset(); + onComplete->onComplete(std::make_unique<RemoveResult>(Result::ErrorType::TRANSIENT_ERROR, "Bucket not found")); + } else { + DocEntry::SP entry((*bc)->getEntry(did)); + bool foundPut(entry.get() && !entry->isRemove()); + auto remEntry = std::make_unique<DocEntry>(t, REMOVE_ENTRY, did); - if ((*bc)->hasTimestamp(t)) { - (*bc)->eraseEntry(t); + if ((*bc)->hasTimestamp(t)) { + (*bc)->eraseEntry(t); + } + (*bc)->insert(std::move(remEntry)); + bc.reset(); + onComplete->onComplete(std::make_unique<RemoveResult>(foundPut)); } - (*bc)->insert(std::move(remEntry)); - return RemoveResult(foundPut); } GetResult @@ -745,8 +730,8 @@ DummyPersistence::createBucket(const Bucket& b, Context&) return Result(); } -Result -DummyPersistence::deleteBucket(const Bucket& b, Context&) +void +DummyPersistence::deleteBucketAsync(const Bucket& b, Context&, OperationComplete::UP onComplete) { DUMMYPERSISTENCE_VERIFY_INITIALIZED; LOG(debug, "deleteBucket(%s)", b.toString().c_str()); @@ -756,7 +741,7 @@ DummyPersistence::deleteBucket(const Bucket& b, Context&) assert(!_content[b]->_inUse); } _content.erase(b); - return Result(); + onComplete->onComplete(std::make_unique<Result>()); } Result diff --git a/persistence/src/vespa/persistence/dummyimpl/dummypersistence.h b/persistence/src/vespa/persistence/dummyimpl/dummypersistence.h index 486f4cec2f2..99d6ba717b7 100644 --- a/persistence/src/vespa/persistence/dummyimpl/dummypersistence.h +++ b/persistence/src/vespa/persistence/dummyimpl/dummypersistence.h @@ -155,12 +155,12 @@ public: BucketIdListResult getModifiedBuckets(BucketSpace bucketSpace) const override; Result setClusterState(BucketSpace bucketSpace, const ClusterState& newState) override; - Result setActiveState(const Bucket& bucket, BucketInfo::ActiveState newState) override; + void setActiveStateAsync(const Bucket&, BucketInfo::ActiveState, OperationComplete::UP) override; BucketInfoResult getBucketInfo(const Bucket&) const override; - Result put(const Bucket&, Timestamp, DocumentSP, Context&) override; GetResult get(const Bucket&, const document::FieldSet&, const DocumentId&, Context&) const override; - RemoveResult remove(const Bucket& b, Timestamp t, const DocumentId& did, Context&) override; - UpdateResult update(const Bucket&, Timestamp, DocumentUpdateSP, Context&) override; + void putAsync(const Bucket&, Timestamp, DocumentSP, Context&, OperationComplete::UP) override; + void removeAsync(const Bucket& b, Timestamp t, const DocumentId& did, Context&, OperationComplete::UP) override; + void updateAsync(const Bucket&, Timestamp, DocumentUpdateSP, Context&, OperationComplete::UP) override; CreateIteratorResult createIterator(const Bucket &bucket, FieldSetSP fs, const Selection &, IncludedVersions, Context &context) override; @@ -169,7 +169,7 @@ public: Result destroyIterator(IteratorId, Context&) override; Result createBucket(const Bucket&, Context&) override; - Result deleteBucket(const Bucket&, Context&) override; + void deleteBucketAsync(const Bucket&, Context&, OperationComplete::UP) override; Result split(const Bucket& source, const Bucket& target1, const Bucket& target2, Context&) override; diff --git a/persistence/src/vespa/persistence/spi/abstractpersistenceprovider.cpp b/persistence/src/vespa/persistence/spi/abstractpersistenceprovider.cpp index 23a8f600024..951dbf97cff 100644 --- a/persistence/src/vespa/persistence/spi/abstractpersistenceprovider.cpp +++ b/persistence/src/vespa/persistence/spi/abstractpersistenceprovider.cpp @@ -9,13 +9,6 @@ namespace storage::spi { -RemoveResult -AbstractPersistenceProvider::removeIfFound(const Bucket& b, Timestamp timestamp, - const DocumentId& id, Context& context) -{ - return remove(b, timestamp, id, context); -} - void AbstractPersistenceProvider::removeIfFoundAsync(const Bucket& b, Timestamp timestamp, const DocumentId& id, Context& context, OperationComplete::UP onComplete) diff --git a/persistence/src/vespa/persistence/spi/abstractpersistenceprovider.h b/persistence/src/vespa/persistence/spi/abstractpersistenceprovider.h index 2332c05b57f..e287bdc5252 100644 --- a/persistence/src/vespa/persistence/spi/abstractpersistenceprovider.h +++ b/persistence/src/vespa/persistence/spi/abstractpersistenceprovider.h @@ -14,39 +14,11 @@ namespace storage::spi { class AbstractPersistenceProvider : public PersistenceProvider { public: - /** - * Default impl is empty. - */ Result initialize() override { return Result(); }; - - /** - * Default impl empty. - */ Result createBucket(const Bucket&, Context&) override { return Result(); } - - /** - * Default impl is empty. - */ Result removeEntry(const Bucket&, Timestamp, Context&) override { return Result(); } - - /** - * Default impl is remove(). - */ - RemoveResult removeIfFound(const Bucket&, Timestamp, const DocumentId&, Context&) override; void removeIfFoundAsync(const Bucket&, Timestamp, const DocumentId&, Context&, OperationComplete::UP) override; - - /** - * Default impl empty. - */ Result setClusterState(BucketSpace, const ClusterState&) override { return Result(); } - - /** - * Default impl empty. - */ - Result setActiveState(const Bucket&, BucketInfo::ActiveState) override { return Result(); } - /** - * Default impl empty. - */ BucketIdListResult getModifiedBuckets(BucketSpace bucketSpace) const override; }; diff --git a/persistence/src/vespa/persistence/spi/persistenceprovider.cpp b/persistence/src/vespa/persistence/spi/persistenceprovider.cpp index 575a95269c5..3ea476c33fc 100644 --- a/persistence/src/vespa/persistence/spi/persistenceprovider.cpp +++ b/persistence/src/vespa/persistence/spi/persistenceprovider.cpp @@ -9,19 +9,27 @@ namespace storage::spi { PersistenceProvider::~PersistenceProvider() = default; Result -PersistenceProvider::put(const Bucket& bucket, Timestamp timestamp, DocumentSP doc, Context& context) { +PersistenceProvider::setActiveState(const Bucket& bucket, BucketInfo::ActiveState activeState) { auto catcher = std::make_unique<CatchResult>(); auto future = catcher->future_result(); - putAsync(bucket, timestamp, std::move(doc), context, std::move(catcher)); + setActiveStateAsync(bucket, activeState, std::move(catcher)); + return *future.get(); +} + +Result +PersistenceProvider::deleteBucket(const Bucket& bucket, Context& context) { + auto catcher = std::make_unique<CatchResult>(); + auto future = catcher->future_result(); + deleteBucketAsync(bucket, context, std::move(catcher)); return *future.get(); } -void -PersistenceProvider::putAsync(const Bucket &bucket, Timestamp timestamp, DocumentSP doc, Context &context, - OperationComplete::UP onComplete) -{ - Result result = put(bucket, timestamp, std::move(doc), context); - onComplete->onComplete(std::make_unique<Result>(result)); +Result +PersistenceProvider::put(const Bucket& bucket, Timestamp timestamp, DocumentSP doc, Context& context) { + auto catcher = std::make_unique<CatchResult>(); + auto future = catcher->future_result(); + putAsync(bucket, timestamp, std::move(doc), context, std::move(catcher)); + return *future.get(); } RemoveResult @@ -32,14 +40,6 @@ PersistenceProvider::remove(const Bucket& bucket, Timestamp timestamp, const Doc return dynamic_cast<const RemoveResult &>(*future.get()); } -void -PersistenceProvider::removeAsync(const Bucket &bucket, Timestamp timestamp, const DocumentId & docId, Context &context, - OperationComplete::UP onComplete) -{ - RemoveResult result = remove(bucket, timestamp, docId, context); - onComplete->onComplete(std::make_unique<RemoveResult>(result)); -} - RemoveResult PersistenceProvider::removeIfFound(const Bucket& bucket, Timestamp timestamp, const DocumentId & docId, Context& context) { auto catcher = std::make_unique<CatchResult>(); @@ -48,14 +48,6 @@ PersistenceProvider::removeIfFound(const Bucket& bucket, Timestamp timestamp, co return dynamic_cast<const RemoveResult &>(*future.get()); } -void -PersistenceProvider::removeIfFoundAsync(const Bucket &bucket, Timestamp timestamp, const DocumentId & docId, Context &context, - OperationComplete::UP onComplete) -{ - RemoveResult result = removeIfFound(bucket, timestamp, docId, context); - onComplete->onComplete(std::make_unique<RemoveResult>(result)); -} - UpdateResult PersistenceProvider::update(const Bucket& bucket, Timestamp timestamp, DocumentUpdateSP upd, Context& context) { auto catcher = std::make_unique<CatchResult>(); @@ -64,12 +56,4 @@ PersistenceProvider::update(const Bucket& bucket, Timestamp timestamp, DocumentU return dynamic_cast<const UpdateResult &>(*future.get()); } -void -PersistenceProvider::updateAsync(const Bucket &bucket, Timestamp timestamp, DocumentUpdateSP upd, Context &context, - OperationComplete::UP onComplete) -{ - UpdateResult result = update(bucket, timestamp, std::move(upd), context); - onComplete->onComplete(std::make_unique<UpdateResult>(result)); -} - } diff --git a/persistence/src/vespa/persistence/spi/persistenceprovider.h b/persistence/src/vespa/persistence/spi/persistenceprovider.h index 56ef21b5c77..83eb042d855 100644 --- a/persistence/src/vespa/persistence/spi/persistenceprovider.h +++ b/persistence/src/vespa/persistence/spi/persistenceprovider.h @@ -57,6 +57,14 @@ struct PersistenceProvider virtual ~PersistenceProvider(); + // TODO Move to utility class for use in tests only + Result deleteBucket(const Bucket&, Context&); + Result put(const Bucket&, Timestamp, DocumentSP, Context&); + Result setActiveState(const Bucket&, BucketInfo::ActiveState); + RemoveResult remove(const Bucket&, Timestamp timestamp, const DocumentId& id, Context&); + RemoveResult removeIfFound(const Bucket&, Timestamp timestamp, const DocumentId& id, Context&); + UpdateResult update(const Bucket&, Timestamp timestamp, DocumentUpdateSP update, Context&); + /** * Initializes the persistence provider. This function is called exactly * once when the persistence provider starts. If any error is returned @@ -86,7 +94,7 @@ struct PersistenceProvider * other buckets may be deactivated, so the node must be able to serve * the data from its secondary index or get reduced coverage. */ - virtual Result setActiveState(const Bucket&, BucketInfo::ActiveState) = 0; + virtual void setActiveStateAsync(const Bucket &, BucketInfo::ActiveState, OperationComplete::UP ) = 0; /** * Retrieve metadata for a bucket, previously returned in listBuckets(), @@ -97,11 +105,8 @@ struct PersistenceProvider /** * Store the given document at the given microsecond time. - * An implementation must always implement atleast put or putAsync. - * If not an eternal recursion will occur. */ - virtual Result put(const Bucket&, Timestamp, DocumentSP, Context&); - virtual void putAsync(const Bucket &, Timestamp , DocumentSP , Context &, OperationComplete::UP ); + virtual void putAsync(const Bucket &, Timestamp , DocumentSP , Context &, OperationComplete::UP ) = 0; /** * This remove function assumes that there exist something to be removed. @@ -162,8 +167,7 @@ struct PersistenceProvider * @param timestamp The timestamp for the new bucket entry. * @param id The ID to remove */ - virtual RemoveResult remove(const Bucket&, Timestamp timestamp, const DocumentId& id, Context&); - virtual void removeAsync(const Bucket&, Timestamp timestamp, const DocumentId& id, Context&, OperationComplete::UP); + virtual void removeAsync(const Bucket&, Timestamp timestamp, const DocumentId& id, Context&, OperationComplete::UP) = 0; /** * @see remove() @@ -181,8 +185,7 @@ struct PersistenceProvider * @param timestamp The timestamp for the new bucket entry. * @param id The ID to remove */ - virtual RemoveResult removeIfFound(const Bucket&, Timestamp timestamp, const DocumentId& id, Context&); - virtual void removeIfFoundAsync(const Bucket&, Timestamp timestamp, const DocumentId& id, Context&, OperationComplete::UP); + virtual void removeIfFoundAsync(const Bucket&, Timestamp timestamp, const DocumentId& id, Context&, OperationComplete::UP) = 0; /** * Remove any trace of the entry with the given timestamp. (Be it a document @@ -201,8 +204,7 @@ struct PersistenceProvider * @param timestamp The timestamp to use for the new update entry. * @param update The document update to apply to the stored document. */ - virtual UpdateResult update(const Bucket&, Timestamp timestamp, DocumentUpdateSP update, Context&); - virtual void updateAsync(const Bucket&, Timestamp timestamp, DocumentUpdateSP update, Context&, OperationComplete::UP); + virtual void updateAsync(const Bucket&, Timestamp timestamp, DocumentUpdateSP update, Context&, OperationComplete::UP) = 0; /** * Retrieves the latest version of the document specified by the @@ -341,7 +343,7 @@ struct PersistenceProvider * After this operation has succeeded, a restart of the provider should * not yield the bucket in getBucketList(). */ - virtual Result deleteBucket(const Bucket&, Context&) = 0; + virtual void deleteBucketAsync(const Bucket&, Context&, OperationComplete::UP) = 0; /** * This function is called continuously by the service layer. It allows the diff --git a/sd-plugin/README.md b/sd-plugin/README.md new file mode 100644 index 00000000000..76a0a53d49f --- /dev/null +++ b/sd-plugin/README.md @@ -0,0 +1,19 @@ +This directory holds the code for an IntteliJ plugin for reading SD files. + +NOTE: This is the source code, not the plugin itself. In order to be able to use the plugin you'll need to download it from JetBrains Marketplace or create a zip file and load it to IntelliJ (details later). + +Before cloning, you should download Gradle and create a Gradle project. +You should also download Grammar-Kit plugin from the Marketplace. + +The grammar is defined in 2 files: +- sd.bnf +- sd.flex + +After cloning, you should: +1. Right-click the sd.bnf file and press "Generate Parser Code" +2. Right-click the sd.flex file and press "Run JFlex Generator" + +Now you should have a "gen" folder next to the "java" folder, and it contains all the parser and lexer code. + +Improtant note! After any change in one of this 2 files (bnf, flex) you'll need to generate again. The proper way is to delete the "gen" folder and then do 1-2 again. + diff --git a/sd-plugin/build.gradle b/sd-plugin/build.gradle new file mode 100644 index 00000000000..cb696cb6fd1 --- /dev/null +++ b/sd-plugin/build.gradle @@ -0,0 +1,45 @@ +plugins { + id 'org.jetbrains.intellij' version '1.1.4' + id 'java' + +// id 'java-gradle-plugin' // I added that +// id 'maven-publish' // to deploy the plugin into a Maven repo +} + +group 'org.vz.native' +version '1.0.5-SNAPSHOT' + +sourceCompatibility = 11 // I added that from Simple Plugin + +repositories { + mavenCentral() +} + +dependencies { + testImplementation 'org.junit.jupiter:junit-jupiter-api:5.7.2' +// testRuntimeOnly 'org.junit.jupiter:junit-jupiter-engine:5.7.2' +} + +sourceSets.main.java.srcDirs 'src/main/gen' + +// See https://github.com/JetBrains/gradle-intellij-plugin/ +intellij { + version = '2021.2' + plugins = ['com.intellij.java'] +} + +buildSearchableOptions { // I added that from Simple Plugin + enabled = false +} + +patchPluginXml { + version = project.version + sinceBuild = '203' + untilBuild = '212.*' + changeNotes = """ + <em>fixed a casting bug in SdFindUsagesHandler</em>""" +} + +test { + useJUnitPlatform() +}
\ No newline at end of file diff --git a/sd-plugin/settings.gradle b/sd-plugin/settings.gradle new file mode 100644 index 00000000000..5d47bd4197c --- /dev/null +++ b/sd-plugin/settings.gradle @@ -0,0 +1,3 @@ +// Copyright 2000-2020 JetBrains s.r.o. and other contributors. Use of this source code is governed by the Apache 2.0 license that can be found in the LICENSE file. + +rootProject.name = 'sd_language_plugin' diff --git a/sd-plugin/src/main/java/org/intellij/sdk/language/SdBlock.java b/sd-plugin/src/main/java/org/intellij/sdk/language/SdBlock.java new file mode 100644 index 00000000000..d32baf86d7a --- /dev/null +++ b/sd-plugin/src/main/java/org/intellij/sdk/language/SdBlock.java @@ -0,0 +1,60 @@ +package org.intellij.sdk.language; + +import com.intellij.formatting.Alignment; +import com.intellij.formatting.Block; +import com.intellij.formatting.Indent; +import com.intellij.formatting.Spacing; +import com.intellij.formatting.SpacingBuilder; +import com.intellij.formatting.Wrap; +import com.intellij.formatting.WrapType; +import com.intellij.lang.ASTNode; +import com.intellij.psi.TokenType; +import com.intellij.psi.formatter.common.AbstractBlock; +import org.jetbrains.annotations.NotNull; +import org.jetbrains.annotations.Nullable; + +import java.util.ArrayList; +import java.util.List; + +public class SdBlock extends AbstractBlock { + + private final SpacingBuilder spacingBuilder; + + protected SdBlock(@NotNull ASTNode node, @Nullable Wrap wrap, @Nullable Alignment alignment, + SpacingBuilder spacingBuilder) { + super(node, wrap, alignment); + this.spacingBuilder = spacingBuilder; + } + + @Override + protected List<Block> buildChildren() { + List<Block> blocks = new ArrayList<>(); + ASTNode child = myNode.getFirstChildNode(); + while (child != null) { + if (child.getElementType() != TokenType.WHITE_SPACE) { + Block block = new SdBlock(child, Wrap.createWrap(WrapType.NONE, false), Alignment.createAlignment(), + spacingBuilder); + blocks.add(block); + } + child = child.getTreeNext(); + } + return blocks; + } + + @Override + public Indent getIndent() { + return Indent.getNoneIndent(); + } + + @Nullable + @Override + public Spacing getSpacing(@Nullable Block child1, @NotNull Block child2) { + return spacingBuilder.getSpacing(this, child1, child2); + } + + @Override + public boolean isLeaf() { + return myNode.getFirstChildNode() == null; + } + +} diff --git a/sd-plugin/src/main/java/org/intellij/sdk/language/SdChooseByNameContributor.java b/sd-plugin/src/main/java/org/intellij/sdk/language/SdChooseByNameContributor.java new file mode 100644 index 00000000000..45e26f401d4 --- /dev/null +++ b/sd-plugin/src/main/java/org/intellij/sdk/language/SdChooseByNameContributor.java @@ -0,0 +1,61 @@ +package org.intellij.sdk.language; + +import com.intellij.navigation.ChooseByNameContributor; +import com.intellij.navigation.NavigationItem; +import com.intellij.openapi.project.Project; +import com.intellij.openapi.vfs.VirtualFile; +import com.intellij.psi.PsiManager; +import com.intellij.psi.search.FileTypeIndex; +import com.intellij.psi.search.GlobalSearchScope; +import com.intellij.util.indexing.FileBasedIndex; +import org.intellij.sdk.language.psi.SdDeclaration; +import org.intellij.sdk.language.psi.SdFile; +import org.jetbrains.annotations.NotNull; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.List; + +public class SdChooseByNameContributor implements ChooseByNameContributor { + + @Override + public String @NotNull [] getNames(Project project, boolean includeNonProjectItems) { + Collection<VirtualFile> virtualFiles = FileBasedIndex.getInstance().getContainingFiles( + FileTypeIndex.NAME, + SdFileType.INSTANCE, + GlobalSearchScope.allScope(project) + ); + + List<SdDeclaration> declarations = new ArrayList<>(); + + for (VirtualFile file : virtualFiles) { + SdFile sdFile = (SdFile) PsiManager.getInstance(project).findFile(file); + declarations.addAll(SdUtil.findDeclarations(sdFile)); + } + + List<String> names = new ArrayList<>(declarations.size()); + for (SdDeclaration declaration : declarations) { + names.add(declaration.getName()); + } + return names.toArray(new String[names.size()]); + } + + @Override + public NavigationItem @NotNull [] getItemsByName(String name, String pattern, Project project, boolean includeNonProjectItems) { + Collection<VirtualFile> virtualFiles = FileBasedIndex.getInstance().getContainingFiles( + FileTypeIndex.NAME, + SdFileType.INSTANCE, + GlobalSearchScope.allScope(project) + ); + + List<SdDeclaration> declarations = new ArrayList<>(); + + for (VirtualFile file : virtualFiles) { + SdFile sdFile = (SdFile) PsiManager.getInstance(project).findFile(file); + declarations.addAll(SdUtil.findDeclarationsByName(sdFile, name)); + } + + return declarations.toArray(new NavigationItem[declarations.size()]); + } + +} diff --git a/sd-plugin/src/main/java/org/intellij/sdk/language/SdCodeStyleSettings.java b/sd-plugin/src/main/java/org/intellij/sdk/language/SdCodeStyleSettings.java new file mode 100644 index 00000000000..0719ac7f426 --- /dev/null +++ b/sd-plugin/src/main/java/org/intellij/sdk/language/SdCodeStyleSettings.java @@ -0,0 +1,12 @@ +package org.intellij.sdk.language; + +import com.intellij.psi.codeStyle.CodeStyleSettings; +import com.intellij.psi.codeStyle.CustomCodeStyleSettings; + +public class SdCodeStyleSettings extends CustomCodeStyleSettings { + + public SdCodeStyleSettings(CodeStyleSettings settings) { + super("SdCodeStyleSettings", settings); + } + +} diff --git a/sd-plugin/src/main/java/org/intellij/sdk/language/SdCodeStyleSettingsProvider.java b/sd-plugin/src/main/java/org/intellij/sdk/language/SdCodeStyleSettingsProvider.java new file mode 100644 index 00000000000..6ea68655c70 --- /dev/null +++ b/sd-plugin/src/main/java/org/intellij/sdk/language/SdCodeStyleSettingsProvider.java @@ -0,0 +1,44 @@ +package org.intellij.sdk.language; + +import com.intellij.application.options.CodeStyleAbstractConfigurable; +import com.intellij.application.options.CodeStyleAbstractPanel; +import com.intellij.application.options.TabbedLanguageCodeStylePanel; +import com.intellij.psi.codeStyle.CodeStyleConfigurable; +import com.intellij.psi.codeStyle.CodeStyleSettings; +import com.intellij.psi.codeStyle.CodeStyleSettingsProvider; +import com.intellij.psi.codeStyle.CustomCodeStyleSettings; +import org.jetbrains.annotations.NotNull; +import org.jetbrains.annotations.Nullable; + +public class SdCodeStyleSettingsProvider extends CodeStyleSettingsProvider { + + @Override + public CustomCodeStyleSettings createCustomSettings(CodeStyleSettings settings) { + return new SdCodeStyleSettings(settings); + } + + @Nullable + @Override + public String getConfigurableDisplayName() { + return "Sd"; + } + + @NotNull + public CodeStyleConfigurable createConfigurable(@NotNull CodeStyleSettings settings, @NotNull CodeStyleSettings modelSettings) { + return new CodeStyleAbstractConfigurable(settings, modelSettings, this.getConfigurableDisplayName()) { + @Override + protected CodeStyleAbstractPanel createPanel(CodeStyleSettings settings) { + return new SdCodeStyleMainPanel(getCurrentSettings(), settings); + } + }; + } + + private static class SdCodeStyleMainPanel extends TabbedLanguageCodeStylePanel { + + public SdCodeStyleMainPanel(CodeStyleSettings currentSettings, CodeStyleSettings settings) { + super(SdLanguage.INSTANCE, currentSettings, settings); + } + + } + +} diff --git a/sd-plugin/src/main/java/org/intellij/sdk/language/SdCommenter.java b/sd-plugin/src/main/java/org/intellij/sdk/language/SdCommenter.java new file mode 100644 index 00000000000..e0fc30df22b --- /dev/null +++ b/sd-plugin/src/main/java/org/intellij/sdk/language/SdCommenter.java @@ -0,0 +1,41 @@ +package org.intellij.sdk.language; + +import com.intellij.lang.Commenter; +import org.jetbrains.annotations.Nullable; + +/** + * This class enables turning a line into a comment with "Code -> Comment with line comment" + */ +public class SdCommenter implements Commenter { + + @Nullable + @Override + public String getLineCommentPrefix() { + return "#"; + } + + @Nullable + @Override + public String getBlockCommentPrefix() { + return ""; + } + + @Nullable + @Override + public String getBlockCommentSuffix() { + return null; + } + + @Nullable + @Override + public String getCommentedBlockCommentPrefix() { + return null; + } + + @Nullable + @Override + public String getCommentedBlockCommentSuffix() { + return null; + } + +} diff --git a/sd-plugin/src/main/java/org/intellij/sdk/language/SdCompletionContributor.java b/sd-plugin/src/main/java/org/intellij/sdk/language/SdCompletionContributor.java new file mode 100644 index 00000000000..5dd67e5b6e6 --- /dev/null +++ b/sd-plugin/src/main/java/org/intellij/sdk/language/SdCompletionContributor.java @@ -0,0 +1,30 @@ +package org.intellij.sdk.language; + +import com.intellij.codeInsight.completion.CompletionContributor; +import com.intellij.codeInsight.completion.CompletionParameters; +import com.intellij.codeInsight.completion.CompletionProvider; +import com.intellij.codeInsight.completion.CompletionResultSet; +import com.intellij.codeInsight.completion.CompletionType; +import com.intellij.codeInsight.lookup.LookupElementBuilder; +import com.intellij.patterns.PlatformPatterns; +import com.intellij.util.ProcessingContext; +import org.intellij.sdk.language.psi.SdTypes; +import org.jetbrains.annotations.NotNull; + +public class SdCompletionContributor extends CompletionContributor { + + + public SdCompletionContributor() { + extend(CompletionType.BASIC, + PlatformPatterns.psiElement(SdTypes.IDENTIFIER_VAL), + new CompletionProvider<>() { + public void addCompletions(@NotNull CompletionParameters parameters, //completion parameters contain details of the cursor position + @NotNull ProcessingContext context, + @NotNull CompletionResultSet resultSet) { //result set contains completion details to suggest + resultSet.addElement(LookupElementBuilder.create("")); + } + } + ); + } + +} diff --git a/sd-plugin/src/main/java/org/intellij/sdk/language/SdFileType.java b/sd-plugin/src/main/java/org/intellij/sdk/language/SdFileType.java new file mode 100644 index 00000000000..eeed80ad9f1 --- /dev/null +++ b/sd-plugin/src/main/java/org/intellij/sdk/language/SdFileType.java @@ -0,0 +1,41 @@ +package org.intellij.sdk.language; + +import com.intellij.openapi.fileTypes.LanguageFileType; +import org.jetbrains.annotations.NotNull; +import org.jetbrains.annotations.Nullable; + +import javax.swing.Icon; + +public class SdFileType extends LanguageFileType { + + public static final SdFileType INSTANCE = new SdFileType(); + + private SdFileType() { + super(SdLanguage.INSTANCE); + } + + @NotNull + @Override + public String getName() { + return "Sd File"; + } + + @NotNull + @Override + public String getDescription() { + return "Sd language file"; + } + + @NotNull + @Override + public String getDefaultExtension() { + return "sd"; + } + + @Nullable + @Override + public Icon getIcon() { + return SdIcons.FILE; + } + +} diff --git a/sd-plugin/src/main/java/org/intellij/sdk/language/SdFormattingModelBuilder.java b/sd-plugin/src/main/java/org/intellij/sdk/language/SdFormattingModelBuilder.java new file mode 100644 index 00000000000..35f57f65c26 --- /dev/null +++ b/sd-plugin/src/main/java/org/intellij/sdk/language/SdFormattingModelBuilder.java @@ -0,0 +1,37 @@ +package org.intellij.sdk.language; + +import com.intellij.formatting.Alignment; +import com.intellij.formatting.FormattingContext; +import com.intellij.formatting.FormattingModel; +import com.intellij.formatting.FormattingModelBuilder; +import com.intellij.formatting.FormattingModelProvider; +import com.intellij.formatting.SpacingBuilder; +import com.intellij.formatting.Wrap; +import com.intellij.formatting.WrapType; +import com.intellij.psi.codeStyle.CodeStyleSettings; +import org.intellij.sdk.language.psi.SdTypes; +import org.jetbrains.annotations.NotNull; + +public class SdFormattingModelBuilder implements FormattingModelBuilder { + + private static SpacingBuilder createSpaceBuilder(CodeStyleSettings settings) { + return new SpacingBuilder(settings, SdLanguage.INSTANCE) + .around(SdTypes.SYMBOL) + .spaceIf(settings.getCommonSettings(SdLanguage.INSTANCE.getID()).SPACE_AFTER_COLON) + .before(SdTypes.DOCUMENT_FIELD_DEFINITION) + .none(); + } + + @Override + public @NotNull FormattingModel createModel(@NotNull FormattingContext formattingContext) { + final CodeStyleSettings codeStyleSettings = formattingContext.getCodeStyleSettings(); + return FormattingModelProvider + .createFormattingModelForPsiFile(formattingContext.getContainingFile(), + new SdBlock(formattingContext.getNode(), + Wrap.createWrap(WrapType.NONE, false), + Alignment.createAlignment(), + createSpaceBuilder(codeStyleSettings)), + codeStyleSettings); + } + +} diff --git a/sd-plugin/src/main/java/org/intellij/sdk/language/SdIcons.java b/sd-plugin/src/main/java/org/intellij/sdk/language/SdIcons.java new file mode 100644 index 00000000000..8dadda25774 --- /dev/null +++ b/sd-plugin/src/main/java/org/intellij/sdk/language/SdIcons.java @@ -0,0 +1,19 @@ +package org.intellij.sdk.language; + +import com.intellij.openapi.util.IconLoader; + +import javax.swing.Icon; + +public class SdIcons { + public static final Icon FILE = IconLoader.getIcon("icons/sd_icon.png", SdIcons.class); + public static final Icon STRUCT_FIELD = IconLoader.getIcon("icons/struct_field_icon.png", SdIcons.class); + public static final Icon IMPORTED_FIELD = IconLoader.getIcon("icons/imported_field_icon.png", SdIcons.class); + public static final Icon DOCUMENT_SUMMARY = IconLoader.getIcon("icons/document_summary_icon.png", SdIcons.class); + public static final Icon STRUCT = IconLoader.getIcon("icons/struct_icon.png", SdIcons.class); + public static final Icon DOCUMENT = IconLoader.getIcon("icons/document_icon.png", SdIcons.class); + public static final Icon SUMMARY = IconLoader.getIcon("icons/summary_def_icon.png", SdIcons.class); + public static final Icon MACRO = IconLoader.getIcon("icons/macro_icon.png", SdIcons.class); + public static final Icon OVERRIDE_MACRO = IconLoader.getIcon("icons/override_macro_icon.png", SdIcons.class); + public static final Icon FIRST_PHASE = IconLoader.getIcon("icons/first_phase_icon.png", SdIcons.class); +} +
\ No newline at end of file diff --git a/sd-plugin/src/main/java/org/intellij/sdk/language/SdLanguage.java b/sd-plugin/src/main/java/org/intellij/sdk/language/SdLanguage.java new file mode 100644 index 00000000000..7dbf10b68ac --- /dev/null +++ b/sd-plugin/src/main/java/org/intellij/sdk/language/SdLanguage.java @@ -0,0 +1,16 @@ +package org.intellij.sdk.language; + +import com.intellij.lang.Language; + +public class SdLanguage extends Language { + public static final SdLanguage INSTANCE = new SdLanguage(); + + private SdLanguage() { + super("Sd"); + } + + @Override + public boolean isCaseSensitive() { + return true; + } +} diff --git a/sd-plugin/src/main/java/org/intellij/sdk/language/SdLanguageCodeStyleSettingsProvider.java b/sd-plugin/src/main/java/org/intellij/sdk/language/SdLanguageCodeStyleSettingsProvider.java new file mode 100644 index 00000000000..b51ae286201 --- /dev/null +++ b/sd-plugin/src/main/java/org/intellij/sdk/language/SdLanguageCodeStyleSettingsProvider.java @@ -0,0 +1,33 @@ +package org.intellij.sdk.language; + +import com.intellij.lang.Language; +import com.intellij.psi.codeStyle.CodeStyleSettingsCustomizable; +import com.intellij.psi.codeStyle.LanguageCodeStyleSettingsProvider; +import org.jetbrains.annotations.NotNull; + +public class SdLanguageCodeStyleSettingsProvider extends LanguageCodeStyleSettingsProvider { + + @NotNull + @Override + public Language getLanguage() { + return SdLanguage.INSTANCE; + } + + @Override + public void customizeSettings(@NotNull CodeStyleSettingsCustomizable consumer, @NotNull SettingsType settingsType) { + if (settingsType == SettingsType.SPACING_SETTINGS) { + consumer.showStandardOptions("SPACE_AFTER_COLON"); +// consumer.renameStandardOption("SPACE_AROUND_ASSIGNMENT_OPERATORS", "Separator"); + } else if (settingsType == SettingsType.BLANK_LINES_SETTINGS) { + consumer.showStandardOptions("KEEP_BLANK_LINES_IN_CODE"); +// } else if (settingsType == SettingsType.INDENT_SETTINGS) { +// consumer.showStandardOptions("USE_RELATIVE_INDENTS"); + } + } + + @Override + public String getCodeSample(@NotNull SettingsType settingsType) { + return "field myField type int {\n indexing: summary\n}"; + } + +} diff --git a/sd-plugin/src/main/java/org/intellij/sdk/language/SdLexerAdapter.java b/sd-plugin/src/main/java/org/intellij/sdk/language/SdLexerAdapter.java new file mode 100644 index 00000000000..3363ce730b8 --- /dev/null +++ b/sd-plugin/src/main/java/org/intellij/sdk/language/SdLexerAdapter.java @@ -0,0 +1,11 @@ +package org.intellij.sdk.language; + +import com.intellij.lexer.FlexAdapter; +import org.intellij.sdk.language.lexer.SdLexer; + +public class SdLexerAdapter extends FlexAdapter { + + public SdLexerAdapter() { + super(new SdLexer(null)); + } +} diff --git a/sd-plugin/src/main/java/org/intellij/sdk/language/SdRefactoringSupportProvider.java b/sd-plugin/src/main/java/org/intellij/sdk/language/SdRefactoringSupportProvider.java new file mode 100644 index 00000000000..5867729d54d --- /dev/null +++ b/sd-plugin/src/main/java/org/intellij/sdk/language/SdRefactoringSupportProvider.java @@ -0,0 +1,14 @@ +package org.intellij.sdk.language; + +import com.intellij.lang.refactoring.RefactoringSupportProvider; +import com.intellij.psi.PsiElement; +import org.intellij.sdk.language.psi.SdIdentifierVal; +import org.jetbrains.annotations.NotNull; +import org.jetbrains.annotations.Nullable; + +public class SdRefactoringSupportProvider extends RefactoringSupportProvider { + @Override + public boolean isMemberInplaceRenameAvailable(@NotNull PsiElement elementToRename, @Nullable PsiElement context) { + return (elementToRename instanceof SdIdentifierVal); + } +} diff --git a/sd-plugin/src/main/java/org/intellij/sdk/language/SdReference.java b/sd-plugin/src/main/java/org/intellij/sdk/language/SdReference.java new file mode 100644 index 00000000000..891d94e15cb --- /dev/null +++ b/sd-plugin/src/main/java/org/intellij/sdk/language/SdReference.java @@ -0,0 +1,72 @@ +package org.intellij.sdk.language; + +import com.intellij.codeInsight.lookup.LookupElement; +import com.intellij.codeInsight.lookup.LookupElementBuilder; +import com.intellij.openapi.util.TextRange; +import com.intellij.psi.PsiElement; +import com.intellij.psi.PsiElementResolveResult; +import com.intellij.psi.PsiPolyVariantReference; +import com.intellij.psi.PsiReferenceBase; +import com.intellij.psi.ResolveResult; +import com.intellij.util.IncorrectOperationException; +import org.intellij.sdk.language.psi.SdDeclaration; +import org.intellij.sdk.language.psi.SdIdentifierVal; +import org.jetbrains.annotations.NotNull; +import org.jetbrains.annotations.Nullable; + +import java.util.ArrayList; +import java.util.List; + +public class SdReference extends PsiReferenceBase<PsiElement> implements PsiPolyVariantReference { + + private final String elementName; + + public SdReference(@NotNull PsiElement element, TextRange textRange) { + super(element, textRange); + elementName = element.getText().substring(textRange.getStartOffset(), textRange.getEndOffset()); + } + + @NotNull + @Override + public ResolveResult @NotNull [] multiResolve(boolean incompleteCode) { + + PsiElement file = myElement.getContainingFile(); + + final List<SdDeclaration> declarations = SdUtil.findDeclarationsByScope(file, myElement, elementName); + + List<ResolveResult> results = new ArrayList<>(); + + for (SdDeclaration declaration : declarations) { + results.add(new PsiElementResolveResult(declaration)); + } + return results.toArray(new ResolveResult[results.size()]); + } + + @Nullable + @Override + public PsiElement resolve() { + ResolveResult[] resolveResults = multiResolve(false); + return resolveResults.length == 1 ? resolveResults[0].getElement() : null; + } + + @Override + public Object @NotNull [] getVariants() { + List<SdDeclaration> declarations = SdUtil.findDeclarations(myElement.getContainingFile()); + List<LookupElement> variants = new ArrayList<>(); + for (final SdDeclaration element : declarations) { + if (element.getName() != null && element.getName().length() > 0) { + variants.add(LookupElementBuilder + .create(element).withIcon(SdIcons.FILE) + .withTypeText(element.getContainingFile().getName()) + ); + } + } + return variants.toArray(); + } + + @Override + public PsiElement handleElementRename(@NotNull String newElementName) throws IncorrectOperationException { + return ((SdIdentifierVal) myElement).setName(newElementName); + } + +} diff --git a/sd-plugin/src/main/java/org/intellij/sdk/language/SdSyntaxHighlighter.java b/sd-plugin/src/main/java/org/intellij/sdk/language/SdSyntaxHighlighter.java new file mode 100644 index 00000000000..ea2e6331c38 --- /dev/null +++ b/sd-plugin/src/main/java/org/intellij/sdk/language/SdSyntaxHighlighter.java @@ -0,0 +1,137 @@ +package org.intellij.sdk.language; + +import static com.intellij.openapi.editor.colors.TextAttributesKey.createTextAttributesKey; + +import com.intellij.lexer.Lexer; +import com.intellij.openapi.editor.DefaultLanguageHighlighterColors; +import com.intellij.openapi.editor.HighlighterColors; +import com.intellij.openapi.editor.colors.TextAttributesKey; +import com.intellij.openapi.fileTypes.SyntaxHighlighterBase; +import com.intellij.psi.TokenType; +import com.intellij.psi.tree.IElementType; +import org.intellij.sdk.language.psi.SdTypes; +import org.jetbrains.annotations.NotNull; + +import java.util.HashSet; + +public class SdSyntaxHighlighter extends SyntaxHighlighterBase { + + private static final HashSet<IElementType> keyWordsSet = initKeyWordsSet(); + private static final HashSet<IElementType> constantsSet = initConstantsSet(); +// private static final HashSet<IElementType> symbols = initSymbolsSet(); + + + public static final TextAttributesKey IDENTIFIER = + createTextAttributesKey("SD_IDENTIFIER", DefaultLanguageHighlighterColors.INSTANCE_FIELD); + public static final TextAttributesKey CONSTANT = + createTextAttributesKey("SD_CONSTANT", DefaultLanguageHighlighterColors.CONSTANT); + public static final TextAttributesKey KEY = + createTextAttributesKey("SD_KEY", DefaultLanguageHighlighterColors.KEYWORD); + public static final TextAttributesKey SYMBOL = + createTextAttributesKey("SD_SYMBOL", DefaultLanguageHighlighterColors.BRACKETS); + public static final TextAttributesKey STRING = + createTextAttributesKey("SD_STRING", DefaultLanguageHighlighterColors.STRING); + public static final TextAttributesKey COMMENT = + createTextAttributesKey("SD_COMMENT", DefaultLanguageHighlighterColors.LINE_COMMENT); + public static final TextAttributesKey BAD_CHARACTER = + createTextAttributesKey("Sd_BAD_CHARACTER", HighlighterColors.BAD_CHARACTER); + + + private static final TextAttributesKey[] BAD_CHAR_KEYS = new TextAttributesKey[]{BAD_CHARACTER}; + private static final TextAttributesKey[] IDENTIFIER_KEYS = new TextAttributesKey[]{IDENTIFIER}; + private static final TextAttributesKey[] CONSTANT_KEYS = new TextAttributesKey[]{CONSTANT}; + private static final TextAttributesKey[] KEY_KEYS = new TextAttributesKey[]{KEY}; + private static final TextAttributesKey[] SYMBOL_KEYS = new TextAttributesKey[]{SYMBOL}; + private static final TextAttributesKey[] STRING_KEYS = new TextAttributesKey[]{STRING}; + private static final TextAttributesKey[] COMMENT_KEYS = new TextAttributesKey[]{COMMENT}; + private static final TextAttributesKey[] EMPTY_KEYS = new TextAttributesKey[0]; + + @NotNull + @Override + public Lexer getHighlightingLexer() { + return new SdLexerAdapter(); + } + + @NotNull + @Override + public TextAttributesKey[] getTokenHighlights(IElementType tokenType) { + if (tokenType.equals(SdTypes.IDENTIFIER_VAL)) { + return IDENTIFIER_KEYS; +// } else if (tokenType.equals(SdTypes.KEY)) { +// return KEY_KEYS; +// } else if (tokenType.equals(SdTypes.VALUE)) { +// return VALUE_KEYS; + } else if (keyWordsSet.contains(tokenType)) { + return KEY_KEYS; + } else if (tokenType.equals(SdTypes.SYMBOL)) { + return SYMBOL_KEYS; + } else if (tokenType.equals(SdTypes.STRING)) { + return STRING_KEYS; + } else if (tokenType.equals(SdTypes.COMMENT)) { + return COMMENT_KEYS; + } else if (constantsSet.contains(tokenType)) { + return CONSTANT_KEYS; + } else if (tokenType.equals(TokenType.BAD_CHARACTER)) { + return BAD_CHAR_KEYS; + } else { + return EMPTY_KEYS; + } + } + +// private static HashSet<IElementType> initSymbolsSet() { +// HashSet<IElementType> symbols = new HashSet<>(); +// symbols.add('{'); +// return symbols; +// } + + private static HashSet<IElementType> initKeyWordsSet() { + HashSet<IElementType> keyWords = new HashSet<>(); + keyWords.add(SdTypes.MACRO); + keyWords.add(SdTypes.FIELD); + keyWords.add(SdTypes.TYPE); + keyWords.add(SdTypes.SEARCH); + keyWords.add(SdTypes.DOCUMENT); + keyWords.add(SdTypes.INHERITS); + keyWords.add(SdTypes.STRUCT); + keyWords.add(SdTypes.STRUCT_FIELD); + keyWords.add(SdTypes.MATCH); + keyWords.add(SdTypes.INDEXING); + keyWords.add(SdTypes.RANK); + keyWords.add(SdTypes.INDEXING_REWRITE); + keyWords.add(SdTypes.QUERY_COMMAND); + return keyWords; + } + + private static HashSet<IElementType> initConstantsSet() { + HashSet<IElementType> constants = new HashSet<>(); + constants.add(SdTypes.SUMMARY); + constants.add(SdTypes.ATTRIBUTE); + constants.add(SdTypes.TEXT); + constants.add(SdTypes.EXACT); + constants.add(SdTypes.EXACT_TERMINATOR); + constants.add(SdTypes.WORD); + constants.add(SdTypes.PREFIX); + constants.add(SdTypes.CASED); + constants.add(SdTypes.UNCASED); + constants.add(SdTypes.SUBSTRING); + constants.add(SdTypes.SUFFIX); + constants.add(SdTypes.MAX_LENGTH); + constants.add(SdTypes.GRAM); + constants.add(SdTypes.GRAM_SIZE); + constants.add(SdTypes.INDEX); + constants.add(SdTypes.FAST_SEARCH); + constants.add(SdTypes.FAST_ACCESS); + constants.add(SdTypes.ALIAS); + constants.add(SdTypes.SORTING); + constants.add(SdTypes.DISTANCE_METRIC); + constants.add(SdTypes.FILTER); + constants.add(SdTypes.NORMAL); + constants.add(SdTypes.NONE); + constants.add(SdTypes.FULL); + constants.add(SdTypes.DYNAMIC); + constants.add(SdTypes.MATCHED_ELEMENTS_ONLY); + + return constants; + } +} + diff --git a/sd-plugin/src/main/java/org/intellij/sdk/language/SdSyntaxHighlighterFactory.java b/sd-plugin/src/main/java/org/intellij/sdk/language/SdSyntaxHighlighterFactory.java new file mode 100644 index 00000000000..c4bc4491bf1 --- /dev/null +++ b/sd-plugin/src/main/java/org/intellij/sdk/language/SdSyntaxHighlighterFactory.java @@ -0,0 +1,17 @@ +package org.intellij.sdk.language; + +import com.intellij.openapi.fileTypes.SyntaxHighlighter; +import com.intellij.openapi.fileTypes.SyntaxHighlighterFactory; +import com.intellij.openapi.project.Project; +import com.intellij.openapi.vfs.VirtualFile; +import org.jetbrains.annotations.NotNull; + +public class SdSyntaxHighlighterFactory extends SyntaxHighlighterFactory { + + @NotNull + @Override + public SyntaxHighlighter getSyntaxHighlighter(Project project, VirtualFile virtualFile) { + return new SdSyntaxHighlighter(); + } + +} diff --git a/sd-plugin/src/main/java/org/intellij/sdk/language/SdUtil.java b/sd-plugin/src/main/java/org/intellij/sdk/language/SdUtil.java new file mode 100644 index 00000000000..4dd2fe8cac1 --- /dev/null +++ b/sd-plugin/src/main/java/org/intellij/sdk/language/SdUtil.java @@ -0,0 +1,213 @@ +package org.intellij.sdk.language; + +import com.intellij.lang.ASTNode; +import com.intellij.openapi.project.Project; +import com.intellij.openapi.vfs.VirtualFile; +import com.intellij.psi.PsiElement; +import com.intellij.psi.PsiManager; +import com.intellij.psi.PsiReference; +import com.intellij.psi.search.FileTypeIndex; +import com.intellij.psi.search.GlobalSearchScope; +import com.intellij.psi.util.PsiTreeUtil; +import com.intellij.util.indexing.FileBasedIndex; +import org.intellij.sdk.language.psi.SdArgumentDefinition; +import org.intellij.sdk.language.psi.SdDeclaration; +import org.intellij.sdk.language.psi.SdDocumentDefinition; +import org.intellij.sdk.language.psi.SdDocumentFieldDefinition; +import org.intellij.sdk.language.psi.SdDocumentStructDefinition; +import org.intellij.sdk.language.psi.SdDocumentStructFieldDefinition; +import org.intellij.sdk.language.psi.SdDocumentSummaryDefinition; +import org.intellij.sdk.language.psi.SdFieldTypeName; +import org.intellij.sdk.language.psi.SdFile; +import org.intellij.sdk.language.psi.SdFunctionDefinition; +import org.intellij.sdk.language.psi.SdIdentifier; +import org.intellij.sdk.language.psi.SdImportFieldDefinition; +import org.intellij.sdk.language.psi.SdRankProfileDefinition; +import org.intellij.sdk.language.psi.SdSchemaFieldDefinition; +import org.intellij.sdk.language.psi.SdSummaryDefinition; +import org.intellij.sdk.language.psi.SdTypes; +import org.jetbrains.annotations.NotNull; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.HashMap; +import java.util.List; + +public class SdUtil { + + public static @NotNull HashMap<String, List<PsiElement>> createMacrosMap(SdFile file) { + HashMap<String, List<PsiElement>> macrosMap = new HashMap<>(); + for (SdRankProfileDefinition rankProfile : PsiTreeUtil + .findChildrenOfType(file, SdRankProfileDefinition.class)) { + for (SdFunctionDefinition macro : PsiTreeUtil.findChildrenOfType(rankProfile, SdFunctionDefinition.class)) { + macrosMap.computeIfAbsent(macro.getName(), k -> new ArrayList<>()).add(macro); + } + } + return macrosMap; + } + + /** + * @param baseRankProfile the rank-profile node to find its parent + * @return the rank-profile that the baseRankProfile inherits from, or null if it doesn't exist + */ + public static PsiElement getRankProfileParent(SdRankProfileDefinition baseRankProfile) { + if (baseRankProfile == null) { + return null; + } + ASTNode inheritsNode = baseRankProfile.getNode().findChildByType(SdTypes.INHERITS); + if (inheritsNode == null) { + return null; + } + ASTNode ancestorAST = baseRankProfile.getNode().findChildByType(SdTypes.IDENTIFIER_VAL, inheritsNode); + if (ancestorAST == null) { + ancestorAST = baseRankProfile.getNode().findChildByType(SdTypes.IDENTIFIER_WITH_DASH_VAL, inheritsNode); + if (ancestorAST == null) { + return null; + } + } + SdIdentifier ancestorIdentifier = (SdIdentifier) ancestorAST.getPsi(); + return ancestorIdentifier.getReference().resolve(); + + } + + public static String createFunctionDescription(SdFunctionDefinition macro) { + SdRankProfileDefinition rankProfile = PsiTreeUtil.getParentOfType(macro, SdRankProfileDefinition.class); + String rankProfileName; + if (rankProfile != null) { + rankProfileName = rankProfile.getName(); + List<SdArgumentDefinition> args = macro.getArgumentDefinitionList(); + StringBuilder text = new StringBuilder(rankProfileName + "." + macro.getName() + "("); + for (int i = 0; i < args.size(); i++) { + text.append(args.get(i).getName()); + if (i < args.size() - 1) { + text.append(", "); + } + } + text.append(")"); + return text.toString(); + } else { + return macro.getName(); + } + } + + public static List<SdDeclaration> findDeclarationsByName(PsiElement file, String name) { + List<SdDeclaration> result = new ArrayList<>(); + + for (SdDeclaration declaration : PsiTreeUtil.collectElementsOfType(file, SdDeclaration.class)) { + if (name.equals(declaration.getName())) { + result.add(declaration); + } + } + return result; + } + + + public static List<SdDeclaration> findDeclarationsByScope(PsiElement file, PsiElement element, String name) { + List<SdDeclaration> result = new ArrayList<>(); + + // If element is a field declared in another file (to be imported), return the declaration from the other file + // if found, else return an empty result list + if (element.getParent() instanceof SdImportFieldDefinition && + element.getNextSibling().getNextSibling().getText().equals("as")) { + Project project = file.getProject(); + + PsiReference docFieldRef = element.getPrevSibling().getPrevSibling().getReference(); + PsiElement docField = docFieldRef != null ? docFieldRef.resolve() : null; + SdFieldTypeName fieldType = docField != null ? PsiTreeUtil.findChildOfType(docField, SdFieldTypeName.class) : null; + SdIdentifier docIdentifier = fieldType != null ? PsiTreeUtil.findChildOfType(fieldType, SdIdentifier.class) : null; + String docName = docIdentifier != null ? docIdentifier.getName() : null; + if (docName == null) { + return result; + } + + Collection<VirtualFile> virtualFiles = FileBasedIndex.getInstance().getContainingFiles( + FileTypeIndex.NAME, + SdFileType.INSTANCE, + GlobalSearchScope.allScope(project) + ); + for (VirtualFile vfile : virtualFiles) { + SdFile sdFile = (SdFile) PsiManager.getInstance(project).findFile(vfile); + if (sdFile != null && !sdFile.getName().equals(docName + ".sd")) { + continue; + } + result.addAll(SdUtil.findDeclarationsByName(sdFile, name)); + } + return result; + } + + // If element is the macro's name in the macro definition, return the macro definition + if (element.getParent() instanceof SdFunctionDefinition) { + result.add((SdDeclaration) element.getParent()); + return result; + } + + // Check if element is inside a macro body + SdFunctionDefinition macroParent = PsiTreeUtil.getParentOfType(element, SdFunctionDefinition.class); + if (macroParent != null) { + for (SdArgumentDefinition arg : PsiTreeUtil.findChildrenOfType(macroParent, SdArgumentDefinition.class)) { + if (name.equals(arg.getName())) { // if the element was declared as an argument of the macro + result.add(arg); + return result; + } + } + } + + // If element is a macro's name, return the most specific declaration of the macro + if (((SdIdentifier) element).isFunctionName(file)) { + PsiElement curRankProfile = PsiTreeUtil.getParentOfType(element, SdRankProfileDefinition.class); + while (curRankProfile != null) { + for (SdFunctionDefinition macro : PsiTreeUtil.collectElementsOfType(curRankProfile, SdFunctionDefinition.class)) { + if (macro.getName().equals(name)) { + result.add(macro); + return result; + } + } + curRankProfile = getRankProfileParent((SdRankProfileDefinition) curRankProfile); + } + } + + for (PsiElement declaration : PsiTreeUtil.collectElements(file, psiElement -> + psiElement instanceof SdDeclaration && !(psiElement instanceof SdArgumentDefinition))) { + if (name.equals(((SdDeclaration) declaration).getName())) { + result.add((SdDeclaration) declaration); + break; + } + } + + return result; + } + + public static List<SdDeclaration> findDeclarations(PsiElement element) { + return new ArrayList<>(PsiTreeUtil.collectElementsOfType(element, SdDeclaration.class)); + } + + public static List<PsiElement> findSchemaChildren(PsiElement element) { + return new ArrayList<>(PsiTreeUtil.collectElementsOfType(element, new Class[]{SdDocumentDefinition.class, + SdSchemaFieldDefinition.class, + SdImportFieldDefinition.class, + SdDocumentSummaryDefinition.class, + SdRankProfileDefinition.class})); + } + + public static List<PsiElement> findDocumentChildren(PsiElement element) { + return new ArrayList<>(PsiTreeUtil.collectElementsOfType(element, new Class[]{SdDocumentStructDefinition.class, + SdDocumentFieldDefinition.class})); + } + + public static List<PsiElement> findDocumentStructChildren(PsiElement element) { + return new ArrayList<>(PsiTreeUtil.collectElementsOfType(element, + SdDocumentStructFieldDefinition.class)); + } + + public static List<PsiElement> findRankProfileChildren(PsiElement element) { + return new ArrayList<>(PsiTreeUtil.collectElementsOfType(element, + SdFunctionDefinition.class)); + } + + public static List<PsiElement> findDocumentSummaryChildren(PsiElement element) { + return new ArrayList<>(PsiTreeUtil.collectElementsOfType(element, + SdSummaryDefinition.class)); + } + + +} diff --git a/sd-plugin/src/main/java/org/intellij/sdk/language/findUsages/SdDocumentSummaryGroupingRule.java b/sd-plugin/src/main/java/org/intellij/sdk/language/findUsages/SdDocumentSummaryGroupingRule.java new file mode 100644 index 00000000000..d4d38c8c38a --- /dev/null +++ b/sd-plugin/src/main/java/org/intellij/sdk/language/findUsages/SdDocumentSummaryGroupingRule.java @@ -0,0 +1,32 @@ +package org.intellij.sdk.language.findUsages; + +import com.intellij.openapi.project.DumbAware; +import com.intellij.psi.PsiElement; +import com.intellij.usages.Usage; +import com.intellij.usages.UsageGroup; +import com.intellij.usages.UsageTarget; +import com.intellij.usages.rules.PsiElementUsage; +import com.intellij.usages.rules.SingleParentUsageGroupingRule; +import org.intellij.sdk.language.SdLanguage; +import org.intellij.sdk.language.psi.SdDocumentSummaryDefinition; +import org.jetbrains.annotations.NotNull; +import org.jetbrains.annotations.Nullable; + +public class SdDocumentSummaryGroupingRule extends SingleParentUsageGroupingRule implements DumbAware { + + @Override + protected @Nullable UsageGroup getParentGroupFor(@NotNull Usage usage, UsageTarget @NotNull [] targets) { + PsiElement psiElement = usage instanceof PsiElementUsage ? ((PsiElementUsage)usage).getElement() : null; + if (psiElement == null || psiElement.getLanguage() != SdLanguage.INSTANCE) return null; + + while (psiElement != null) { + if (psiElement instanceof SdDocumentSummaryDefinition) { + final SdDocumentSummaryDefinition componentElement = (SdDocumentSummaryDefinition) psiElement; + return new SdUsageGroup(componentElement); + } + psiElement = psiElement.getParent(); + } + + return null; + } +} diff --git a/sd-plugin/src/main/java/org/intellij/sdk/language/findUsages/SdDocumentSummaryGroupingRuleProvider.java b/sd-plugin/src/main/java/org/intellij/sdk/language/findUsages/SdDocumentSummaryGroupingRuleProvider.java new file mode 100644 index 00000000000..ffd620c822e --- /dev/null +++ b/sd-plugin/src/main/java/org/intellij/sdk/language/findUsages/SdDocumentSummaryGroupingRuleProvider.java @@ -0,0 +1,15 @@ +package org.intellij.sdk.language.findUsages; + +import com.intellij.openapi.project.Project; +import com.intellij.usages.impl.FileStructureGroupRuleProvider; +import com.intellij.usages.rules.UsageGroupingRule; +import org.jetbrains.annotations.NotNull; +import org.jetbrains.annotations.Nullable; + +public class SdDocumentSummaryGroupingRuleProvider implements FileStructureGroupRuleProvider { + + @Override + public @Nullable UsageGroupingRule getUsageGroupingRule(@NotNull Project project) { + return new SdDocumentSummaryGroupingRule(); + } +} diff --git a/sd-plugin/src/main/java/org/intellij/sdk/language/findUsages/SdFindUsagesHandler.java b/sd-plugin/src/main/java/org/intellij/sdk/language/findUsages/SdFindUsagesHandler.java new file mode 100644 index 00000000000..f00ea940613 --- /dev/null +++ b/sd-plugin/src/main/java/org/intellij/sdk/language/findUsages/SdFindUsagesHandler.java @@ -0,0 +1,75 @@ +package org.intellij.sdk.language.findUsages; + +import com.intellij.find.findUsages.FindUsagesHandler; +import com.intellij.find.findUsages.FindUsagesOptions; +import com.intellij.openapi.application.ReadAction; +import com.intellij.openapi.application.ReadActionProcessor; +import com.intellij.psi.PsiElement; +import com.intellij.psi.PsiFile; +import com.intellij.psi.PsiReference; +import com.intellij.psi.search.GlobalSearchScope; +import com.intellij.psi.search.SearchScope; +import com.intellij.psi.search.searches.ReferencesSearch; +import com.intellij.usageView.UsageInfo; +import com.intellij.util.Processor; +import org.intellij.sdk.language.SdUtil; +import org.intellij.sdk.language.psi.SdFile; +import org.intellij.sdk.language.psi.SdFunctionDefinition; +import org.jetbrains.annotations.NotNull; + +import java.util.HashMap; +import java.util.List; + +public class SdFindUsagesHandler extends FindUsagesHandler { + + protected HashMap<String, List<PsiElement>> macrosMap; + + protected SdFindUsagesHandler(@NotNull PsiElement psiElement) { + super(psiElement); + PsiFile file = psiElement.getContainingFile(); + macrosMap = file instanceof SdFile ? SdUtil.createMacrosMap((SdFile) psiElement.getContainingFile()) : new HashMap<>(); + } + + @Override + public boolean processElementUsages(@NotNull final PsiElement elementToSearch, + @NotNull final Processor<? super UsageInfo> processor, + @NotNull final FindUsagesOptions options) { + final ReadActionProcessor<PsiReference> refProcessor = new ReadActionProcessor<>() { + @Override + public boolean processInReadAction(final PsiReference ref) { + return processor.process(new UsageInfo(ref)); + } + }; + + final SearchScope scope = options.searchScope; + + final boolean searchText = options.isSearchForTextOccurrences && scope instanceof GlobalSearchScope; + + if (options.isUsages) { + if (!(elementToSearch instanceof SdFunctionDefinition)) { + boolean success = + ReferencesSearch.search(createSearchParameters(elementToSearch, scope, options)).forEach(refProcessor); + if (!success) return false; + } else { + String macroName = ReadAction.compute( ((SdFunctionDefinition) elementToSearch)::getName); + + for (PsiElement macroImpl : macrosMap.get(macroName)) { + boolean success = + ReferencesSearch.search(createSearchParameters(macroImpl, scope, options)).forEach(refProcessor); + if (!success) return false; + } + } + } + if (searchText) { + if (options.fastTrack != null) { + options.fastTrack.searchCustom(consumer -> processUsagesInText(elementToSearch, processor, (GlobalSearchScope)scope)); + } + else { + return processUsagesInText(elementToSearch, processor, (GlobalSearchScope)scope); + } + } + + return true; + } + +} diff --git a/sd-plugin/src/main/java/org/intellij/sdk/language/findUsages/SdFindUsagesHandlerFactory.java b/sd-plugin/src/main/java/org/intellij/sdk/language/findUsages/SdFindUsagesHandlerFactory.java new file mode 100644 index 00000000000..e5a6580c8c2 --- /dev/null +++ b/sd-plugin/src/main/java/org/intellij/sdk/language/findUsages/SdFindUsagesHandlerFactory.java @@ -0,0 +1,22 @@ +package org.intellij.sdk.language.findUsages; + +import com.intellij.find.findUsages.FindUsagesHandler; +import com.intellij.find.findUsages.FindUsagesHandlerFactory; +import com.intellij.psi.PsiElement; +import com.intellij.psi.PsiNamedElement; +import org.jetbrains.annotations.NotNull; +import org.jetbrains.annotations.Nullable; + +public class SdFindUsagesHandlerFactory extends FindUsagesHandlerFactory { + + @Override + public boolean canFindUsages(@NotNull PsiElement element) { + return element instanceof PsiNamedElement; + } + + @Override + public @Nullable FindUsagesHandler createFindUsagesHandler(@NotNull PsiElement element, + boolean forHighlightUsages) { + return new SdFindUsagesHandler(element); + } +} diff --git a/sd-plugin/src/main/java/org/intellij/sdk/language/findUsages/SdFindUsagesProvider.java b/sd-plugin/src/main/java/org/intellij/sdk/language/findUsages/SdFindUsagesProvider.java new file mode 100644 index 00000000000..eaa5b43641d --- /dev/null +++ b/sd-plugin/src/main/java/org/intellij/sdk/language/findUsages/SdFindUsagesProvider.java @@ -0,0 +1,68 @@ +package org.intellij.sdk.language.findUsages; + +import com.intellij.lang.cacheBuilder.DefaultWordsScanner; +import com.intellij.lang.cacheBuilder.WordsScanner; +import com.intellij.lang.findUsages.FindUsagesProvider; +import com.intellij.psi.PsiElement; +import com.intellij.psi.PsiNamedElement; +import com.intellij.psi.tree.TokenSet; +import org.intellij.sdk.language.SdLexerAdapter; +import org.intellij.sdk.language.psi.SdDeclaration; +import org.intellij.sdk.language.psi.SdIdentifier; +import org.intellij.sdk.language.psi.SdIdentifierVal; +import org.intellij.sdk.language.psi.SdIdentifierWithDashVal; +import org.intellij.sdk.language.psi.SdTypes; +import org.jetbrains.annotations.NotNull; +import org.jetbrains.annotations.Nullable; + +public class SdFindUsagesProvider implements FindUsagesProvider { + @Nullable + @Override + public WordsScanner getWordsScanner() { + return new DefaultWordsScanner(new SdLexerAdapter(), + TokenSet.create(SdTypes.ID_REG, SdTypes.ID_WITH_DASH_REG, SdTypes.IDENTIFIER_VAL, + SdTypes.IDENTIFIER_WITH_DASH_VAL), + TokenSet.create(SdTypes.COMMENT), + TokenSet.create(SdTypes.STRING, SdTypes.INTEGER_REG, SdTypes.FLOAT_REG)); + } + + @Override + public boolean canFindUsagesFor(@NotNull PsiElement psiElement) { + return psiElement instanceof PsiNamedElement; + } + + @Nullable + @Override + public String getHelpId(@NotNull PsiElement psiElement) { + return null; + } + + @NotNull + @Override + public String getType(@NotNull PsiElement element) { + if (element instanceof SdDeclaration) { + return ((SdDeclaration) element).getTypeName(); + } else { + return ""; + } + } + + @NotNull + @Override + public String getDescriptiveName(@NotNull PsiElement element) { + return ""; + } + + @NotNull + @Override + public String getNodeText(@NotNull PsiElement element, boolean useFullName) { + if (element instanceof SdIdentifierVal || element instanceof SdIdentifierWithDashVal) { + return ((SdIdentifier) element).getName(); + } else if (element instanceof SdDeclaration) { + String fullText = element.getNode().getText(); + return fullText.substring(0, fullText.indexOf('{')); + } else { + return ""; + } + } +} diff --git a/sd-plugin/src/main/java/org/intellij/sdk/language/findUsages/SdRankProfileGroupingRule.java b/sd-plugin/src/main/java/org/intellij/sdk/language/findUsages/SdRankProfileGroupingRule.java new file mode 100644 index 00000000000..379f70903df --- /dev/null +++ b/sd-plugin/src/main/java/org/intellij/sdk/language/findUsages/SdRankProfileGroupingRule.java @@ -0,0 +1,32 @@ +package org.intellij.sdk.language.findUsages; + +import com.intellij.openapi.project.DumbAware; +import com.intellij.psi.PsiElement; +import com.intellij.usages.Usage; +import com.intellij.usages.UsageGroup; +import com.intellij.usages.UsageTarget; +import com.intellij.usages.rules.PsiElementUsage; +import com.intellij.usages.rules.SingleParentUsageGroupingRule; +import org.intellij.sdk.language.SdLanguage; +import org.intellij.sdk.language.psi.SdRankProfileDefinition; +import org.jetbrains.annotations.NotNull; +import org.jetbrains.annotations.Nullable; + +public class SdRankProfileGroupingRule extends SingleParentUsageGroupingRule implements DumbAware { + + @Override + protected @Nullable UsageGroup getParentGroupFor(@NotNull Usage usage, UsageTarget @NotNull [] targets) { + PsiElement psiElement = usage instanceof PsiElementUsage ? ((PsiElementUsage)usage).getElement() : null; + if (psiElement == null || psiElement.getLanguage() != SdLanguage.INSTANCE) return null; + + while (psiElement != null) { + if (psiElement instanceof SdRankProfileDefinition) { + final SdRankProfileDefinition componentElement = (SdRankProfileDefinition) psiElement; + return new SdUsageGroup(componentElement); + } + psiElement = psiElement.getParent(); + } + + return null; + } +} diff --git a/sd-plugin/src/main/java/org/intellij/sdk/language/findUsages/SdRankProfileGroupingRuleProvider.java b/sd-plugin/src/main/java/org/intellij/sdk/language/findUsages/SdRankProfileGroupingRuleProvider.java new file mode 100644 index 00000000000..9ed8628dc44 --- /dev/null +++ b/sd-plugin/src/main/java/org/intellij/sdk/language/findUsages/SdRankProfileGroupingRuleProvider.java @@ -0,0 +1,15 @@ +package org.intellij.sdk.language.findUsages; + +import com.intellij.openapi.project.Project; +import com.intellij.usages.impl.FileStructureGroupRuleProvider; +import com.intellij.usages.rules.UsageGroupingRule; +import org.jetbrains.annotations.NotNull; +import org.jetbrains.annotations.Nullable; + +public class SdRankProfileGroupingRuleProvider implements FileStructureGroupRuleProvider { + + @Override + public @Nullable UsageGroupingRule getUsageGroupingRule(@NotNull Project project) { + return new SdRankProfileGroupingRule(); + } +} diff --git a/sd-plugin/src/main/java/org/intellij/sdk/language/findUsages/SdUsageGroup.java b/sd-plugin/src/main/java/org/intellij/sdk/language/findUsages/SdUsageGroup.java new file mode 100644 index 00000000000..21f54f86472 --- /dev/null +++ b/sd-plugin/src/main/java/org/intellij/sdk/language/findUsages/SdUsageGroup.java @@ -0,0 +1,89 @@ +package org.intellij.sdk.language.findUsages; + +import com.intellij.navigation.ItemPresentation; +import com.intellij.navigation.NavigationItemFileStatus; +import com.intellij.openapi.util.NlsSafe; +import com.intellij.openapi.util.text.StringUtil; +import com.intellij.openapi.vcs.FileStatus; +import com.intellij.openapi.vfs.VirtualFile; +import com.intellij.psi.SmartPointerManager; +import com.intellij.psi.SmartPsiElementPointer; +import com.intellij.usages.UsageGroup; +import org.intellij.sdk.language.psi.SdDeclaration; +import org.jetbrains.annotations.NotNull; +import org.jetbrains.annotations.Nullable; + +import javax.swing.*; + +public class SdUsageGroup implements UsageGroup { + private final VirtualFile myFile; + private final SmartPsiElementPointer<SdDeclaration> myElementPointer; + private final String myText; + private final Icon myIcon; + + public SdUsageGroup(SdDeclaration element) { + myFile = element.getContainingFile().getVirtualFile(); + myText = StringUtil.notNullize(element.getName()); + myElementPointer = SmartPointerManager.getInstance(element.getProject()).createSmartPsiElementPointer(element); + ItemPresentation presentation = element.getPresentation(); + myIcon = presentation != null ? presentation.getIcon(true) : null; + } + + @Override + public boolean isValid() { + SdDeclaration element = myElementPointer.getElement(); + return element != null && element.isValid(); + } + + @Override + public void navigate(boolean requestFocus) { + final SdDeclaration nameElement = myElementPointer.getElement(); + if (nameElement != null) { + nameElement.navigate(requestFocus); + } + } + + @Override + public boolean canNavigate() { + return isValid(); + } + + @Override + public boolean canNavigateToSource() { + return canNavigate(); + } + + @Override + public int compareTo(@NotNull UsageGroup usageGroup) { + return getPresentableGroupText().compareToIgnoreCase(usageGroup.getPresentableGroupText()); + } + + @Override + public boolean equals(Object object) { + if (object instanceof SdUsageGroup) { + final SdUsageGroup other = (SdUsageGroup) object; + return myFile.equals(other.myFile) && myText.equals(other.myText); + } + return false; + } + + @Override + public FileStatus getFileStatus() { + return isValid() ? NavigationItemFileStatus.get(myElementPointer.getElement()) : null; + } + + @Override + public int hashCode() { + return myText.hashCode(); + } + + @Override + public @NotNull String getPresentableGroupText() { + return myText; + } + + @Override + public @Nullable Icon getIcon() { + return myIcon; + } +} diff --git a/sd-plugin/src/main/java/org/intellij/sdk/language/hierarchy/SdCallHierarchyBrowser.java b/sd-plugin/src/main/java/org/intellij/sdk/language/hierarchy/SdCallHierarchyBrowser.java new file mode 100644 index 00000000000..5960f1241c3 --- /dev/null +++ b/sd-plugin/src/main/java/org/intellij/sdk/language/hierarchy/SdCallHierarchyBrowser.java @@ -0,0 +1,76 @@ +package org.intellij.sdk.language.hierarchy; + +import com.intellij.ide.hierarchy.CallHierarchyBrowserBase; +import com.intellij.ide.hierarchy.HierarchyNodeDescriptor; +import com.intellij.ide.hierarchy.HierarchyTreeStructure; +import com.intellij.ide.util.treeView.NodeDescriptor; +import com.intellij.openapi.actionSystem.ActionGroup; +import com.intellij.openapi.actionSystem.ActionManager; +import com.intellij.openapi.actionSystem.ActionPlaces; +import com.intellij.openapi.actionSystem.IdeActions; +import com.intellij.openapi.project.Project; +import com.intellij.psi.PsiElement; +import com.intellij.ui.PopupHandler; +import com.intellij.util.ObjectUtils; +import org.intellij.sdk.language.psi.SdFunctionDefinition; +import org.jetbrains.annotations.NotNull; +import org.jetbrains.annotations.Nullable; + +import java.util.Comparator; +import java.util.Map; + +import javax.swing.JTree; + +public class SdCallHierarchyBrowser extends CallHierarchyBrowserBase { + + public SdCallHierarchyBrowser(@NotNull Project project, + @NotNull PsiElement macro) { + super(project, macro); + } + + @Nullable + @Override + protected PsiElement getElementFromDescriptor(@NotNull HierarchyNodeDescriptor descriptor) { + return ObjectUtils.tryCast(descriptor.getPsiElement(), PsiElement.class); + } + + @Override + protected void createTrees(@NotNull Map<? super String, ? super JTree> type2TreeMap) { + ActionGroup group = (ActionGroup) ActionManager.getInstance().getAction(IdeActions.GROUP_CALL_HIERARCHY_POPUP); + type2TreeMap.put(getCallerType(), createHierarchyTree(group)); + type2TreeMap.put(getCalleeType(), createHierarchyTree(group)); + } + + private JTree createHierarchyTree(ActionGroup group) { + final JTree tree = createTree(false); + PopupHandler.installPopupMenu(tree, group, ActionPlaces.CALL_HIERARCHY_VIEW_POPUP); + return tree; + } + + @Override + protected boolean isApplicableElement(@NotNull PsiElement element) { + return element instanceof SdFunctionDefinition; + } + + + @Nullable + @Override + protected HierarchyTreeStructure createHierarchyTreeStructure(@NotNull String typeName, @NotNull PsiElement psiElement) { + if (getCallerType().equals(typeName)) { + return new SdCallerTreeStructure(myProject, psiElement, getCurrentScopeType()); + } + else if (getCalleeType().equals(typeName)) { + return new SdCalleeTreeStructure(myProject, psiElement, getCurrentScopeType()); + } + else { + return null; + } + } + + @Nullable + @Override + protected Comparator<NodeDescriptor<?>> getComparator() { + return SdHierarchyUtil.getComparator(myProject); + } + +} diff --git a/sd-plugin/src/main/java/org/intellij/sdk/language/hierarchy/SdCallHierarchyNodeDescriptor.java b/sd-plugin/src/main/java/org/intellij/sdk/language/hierarchy/SdCallHierarchyNodeDescriptor.java new file mode 100644 index 00000000000..0cf612b0058 --- /dev/null +++ b/sd-plugin/src/main/java/org/intellij/sdk/language/hierarchy/SdCallHierarchyNodeDescriptor.java @@ -0,0 +1,71 @@ +package org.intellij.sdk.language.hierarchy; + +import com.intellij.ide.hierarchy.HierarchyNodeDescriptor; +import com.intellij.ide.util.treeView.NodeDescriptor; +import com.intellij.navigation.ItemPresentation; +import com.intellij.openapi.roots.ui.util.CompositeAppearance; +import com.intellij.openapi.util.Comparing; +import com.intellij.psi.NavigatablePsiElement; +import com.intellij.psi.PsiElement; +import com.intellij.psi.PsiFile; + +import org.intellij.sdk.language.SdIcons; +import org.intellij.sdk.language.SdUtil; +import org.intellij.sdk.language.psi.SdFirstPhaseDefinition; +import org.intellij.sdk.language.psi.SdFunctionDefinition; + +import org.jetbrains.annotations.NotNull; + +import javax.swing.Icon; + +public class SdCallHierarchyNodeDescriptor extends HierarchyNodeDescriptor { + + public SdCallHierarchyNodeDescriptor(final NodeDescriptor parentDescriptor, @NotNull final PsiElement element, final boolean isBase) { + super(element.getProject(), parentDescriptor, element, isBase); + CompositeAppearance.DequeEnd beginning = myHighlightedText.getBeginning(); + if (element instanceof SdFunctionDefinition) { + beginning.addText(SdUtil.createFunctionDescription((SdFunctionDefinition) element)); + } else if (element instanceof SdFirstPhaseDefinition) { + beginning.addText(((SdFirstPhaseDefinition) element).getName()); + } else { + beginning.addText(element.getText()); + } + } + + @Override + public boolean update() { + boolean changes = super.update(); + final CompositeAppearance oldText = myHighlightedText; + myHighlightedText = new CompositeAppearance(); + NavigatablePsiElement element = (NavigatablePsiElement)getPsiElement(); + if (element == null) { + return invalidElement(); + } + + final ItemPresentation presentation = element.getPresentation(); + if (presentation != null) { + myHighlightedText.getEnding().addText(presentation.getPresentableText()); + PsiFile file = element.getContainingFile(); + if (file != null) { // adds the file's name + myHighlightedText.getEnding().addText(" (" + file.getName() + ")", HierarchyNodeDescriptor.getPackageNameAttributes()); + } + Icon icon = SdIcons.FILE; + if (element instanceof SdFunctionDefinition) { + icon = ((SdFunctionDefinition) element).isOverride() ? SdIcons.OVERRIDE_MACRO : SdIcons.MACRO; + } else if (element instanceof SdFirstPhaseDefinition) { + icon = SdIcons.FIRST_PHASE; + } + installIcon(icon, changes); + } + + myName = myHighlightedText.getText(); + if (!Comparing.equal(myHighlightedText, oldText)) { + changes = true; + } + + + return changes; + + } + +}
\ No newline at end of file diff --git a/sd-plugin/src/main/java/org/intellij/sdk/language/hierarchy/SdCallHierarchyProvider.java b/sd-plugin/src/main/java/org/intellij/sdk/language/hierarchy/SdCallHierarchyProvider.java new file mode 100644 index 00000000000..44bfdca134f --- /dev/null +++ b/sd-plugin/src/main/java/org/intellij/sdk/language/hierarchy/SdCallHierarchyProvider.java @@ -0,0 +1,66 @@ +package org.intellij.sdk.language.hierarchy; + + +import com.intellij.ide.hierarchy.CallHierarchyBrowserBase; +import com.intellij.ide.hierarchy.HierarchyBrowser; +import com.intellij.ide.hierarchy.HierarchyProvider; +import com.intellij.openapi.actionSystem.DataContext; +import com.intellij.openapi.editor.Editor; +import com.intellij.psi.PsiDocumentManager; +import com.intellij.psi.PsiElement; +import com.intellij.psi.PsiFile; +import com.intellij.psi.PsiReference; +import org.intellij.sdk.language.SdReference; +import org.intellij.sdk.language.SdUtil; +import org.intellij.sdk.language.psi.SdDeclaration; +import org.intellij.sdk.language.psi.SdFunctionDefinition; +import org.intellij.sdk.language.psi.SdIdentifierVal; +import org.intellij.sdk.language.psi.impl.SdPsiImplUtil; +import org.jetbrains.annotations.NotNull; +import org.jetbrains.annotations.Nullable; +import com.intellij.openapi.actionSystem.CommonDataKeys; +import com.intellij.openapi.project.Project; + +import com.intellij.psi.util.PsiTreeUtil; + + +public class SdCallHierarchyProvider implements HierarchyProvider { + + @Override + public @Nullable PsiElement getTarget(@NotNull DataContext dataContext) { + final Project project = CommonDataKeys.PROJECT.getData(dataContext); + final Editor editor = CommonDataKeys.EDITOR.getData(dataContext); + if (project == null || editor == null) return null; + + final PsiFile file = PsiDocumentManager.getInstance(project).getPsiFile(editor.getDocument()); + if (file == null) { + return null; + } + final PsiElement element = file.findElementAt(editor.getCaretModel().getOffset()); + if (element == null) { + return null; + } + + if (element instanceof SdIdentifierVal || element.getParent() instanceof SdIdentifierVal) { + PsiReference ref = element instanceof SdIdentifierVal ? element.getReference() : element.getParent().getReference(); + if (ref == null) { + return null; + } + PsiElement resolvedRef = ref.resolve(); + if (resolvedRef instanceof SdFunctionDefinition) { + return resolvedRef; + } + } + return null; + } + + @Override + public @NotNull HierarchyBrowser createHierarchyBrowser(@NotNull PsiElement target) { + return new SdCallHierarchyBrowser(target.getProject(), target); + } + + @Override + public void browserActivated(@NotNull HierarchyBrowser hierarchyBrowser) { + ((SdCallHierarchyBrowser) hierarchyBrowser).changeView(CallHierarchyBrowserBase.getCallerType()); + } +} diff --git a/sd-plugin/src/main/java/org/intellij/sdk/language/hierarchy/SdCallTreeStructure.java b/sd-plugin/src/main/java/org/intellij/sdk/language/hierarchy/SdCallTreeStructure.java new file mode 100644 index 00000000000..fb612ed851a --- /dev/null +++ b/sd-plugin/src/main/java/org/intellij/sdk/language/hierarchy/SdCallTreeStructure.java @@ -0,0 +1,73 @@ +package org.intellij.sdk.language.hierarchy; + +import com.intellij.ide.hierarchy.HierarchyNodeDescriptor; +import com.intellij.ide.hierarchy.HierarchyTreeStructure; +import com.intellij.openapi.project.Project; +import com.intellij.psi.PsiElement; +import com.intellij.psi.util.PsiTreeUtil; +import com.intellij.util.ArrayUtil; +import com.intellij.util.ArrayUtilRt; +import org.intellij.sdk.language.SdUtil; +import org.intellij.sdk.language.psi.SdFile; +import org.intellij.sdk.language.psi.SdFunctionDefinition; +import org.intellij.sdk.language.psi.SdIdentifierVal; +import org.intellij.sdk.language.psi.SdRankProfileDefinition; +import org.intellij.sdk.language.psi.impl.SdPsiImplUtil; +import org.jetbrains.annotations.NotNull; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; + +public abstract class SdCallTreeStructure extends HierarchyTreeStructure { + protected final String myScopeType; + protected final SdFile myFile; + protected HashMap<String, List<PsiElement>> macrosMap; + protected HashMap<String, HashSet<SdRankProfileDefinition>> ranksHeritageMap; + + public SdCallTreeStructure(Project project, PsiElement element, String currentScopeType) { + super(project, new SdCallHierarchyNodeDescriptor(null, element, true)); + myScopeType = currentScopeType; + myFile = (SdFile) element.getContainingFile(); + macrosMap = SdUtil.createMacrosMap(myFile); + ranksHeritageMap = new HashMap<>(); + } + + @NotNull + protected abstract HashSet<PsiElement> getChildren(@NotNull SdFunctionDefinition element); + + @Override + protected Object @NotNull [] buildChildren(@NotNull HierarchyNodeDescriptor descriptor) { + final List<SdCallHierarchyNodeDescriptor> descriptors = new ArrayList<>(); + + if (descriptor instanceof SdCallHierarchyNodeDescriptor) { + PsiElement element = descriptor.getPsiElement(); + if (element == null) { + return ArrayUtilRt.EMPTY_OBJECT_ARRAY; + } + boolean isCallable = SdHierarchyUtil.isExecutable(element); + HierarchyNodeDescriptor nodeDescriptor = getBaseDescriptor(); + if (!isCallable || nodeDescriptor == null) { + return ArrayUtilRt.EMPTY_OBJECT_ARRAY; + } + + HashSet<PsiElement> children = getChildren((SdFunctionDefinition) element); + + final HashMap<PsiElement, SdCallHierarchyNodeDescriptor> callerToDescriptorMap = new HashMap<>(); + PsiElement baseClass = PsiTreeUtil.getParentOfType(element, SdRankProfileDefinition.class); + + for (PsiElement caller : children) { + if (isInScope(baseClass, caller, myScopeType)) { + SdCallHierarchyNodeDescriptor callerDescriptor = callerToDescriptorMap.get(caller); + if (callerDescriptor == null) { + callerDescriptor = new SdCallHierarchyNodeDescriptor(descriptor, caller, false); + callerToDescriptorMap.put(caller, callerDescriptor); + descriptors.add(callerDescriptor); + } + } + } + } + return ArrayUtil.toObjectArray(descriptors); + } +} diff --git a/sd-plugin/src/main/java/org/intellij/sdk/language/hierarchy/SdCalleeTreeStructure.java b/sd-plugin/src/main/java/org/intellij/sdk/language/hierarchy/SdCalleeTreeStructure.java new file mode 100644 index 00000000000..977d1948b81 --- /dev/null +++ b/sd-plugin/src/main/java/org/intellij/sdk/language/hierarchy/SdCalleeTreeStructure.java @@ -0,0 +1,64 @@ +package org.intellij.sdk.language.hierarchy; + +import com.intellij.openapi.project.Project; +import com.intellij.psi.PsiElement; +import com.intellij.psi.util.PsiTreeUtil; +import org.intellij.sdk.language.SdReference; +import org.intellij.sdk.language.psi.SdExpressionDefinition; +import org.intellij.sdk.language.psi.SdFunctionDefinition; +import org.intellij.sdk.language.psi.SdIdentifier; +import org.intellij.sdk.language.psi.SdIdentifierVal; +import org.intellij.sdk.language.psi.SdRankProfileDefinition; +import org.jetbrains.annotations.NotNull; + +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; + +public class SdCalleeTreeStructure extends SdCallTreeStructure { + + public SdCalleeTreeStructure(Project project, PsiElement element, String currentScopeType) { + super(project, element, currentScopeType); + } + + @NotNull + @Override + protected HashSet<PsiElement> getChildren(@NotNull SdFunctionDefinition element) { + return getCallees(element, macrosMap); + } + + private HashSet<PsiElement> getCallees(@NotNull SdFunctionDefinition macro, HashMap<String, List<PsiElement>> macrosMap) { + final HashSet<PsiElement> results = new HashSet<>(); + SdExpressionDefinition expression = PsiTreeUtil.findChildOfType(macro, SdExpressionDefinition.class); + if (expression == null) { + return results; + } + for (SdIdentifier identifier : PsiTreeUtil.collectElementsOfType(expression, SdIdentifier.class)) { + if (macrosMap.containsKey(identifier.getName())) { + results.add(identifier.getReference().resolve()); + } + } + + SdRankProfileDefinition rankProfile = PsiTreeUtil.getParentOfType(macro, SdRankProfileDefinition.class); + if (rankProfile == null) { + return results; + } + String rankProfileName = rankProfile.getName(); + if (!ranksHeritageMap.containsKey(rankProfileName)) { + ranksHeritageMap.put(rankProfileName, SdHierarchyUtil.getRankProfileChildren(myFile, rankProfile)); + } + + HashSet<SdRankProfileDefinition> inheritedRanks = ranksHeritageMap.get(rankProfileName); + + for (PsiElement macroImpl : macrosMap.get(macro.getName())) { + if (inheritedRanks.contains(PsiTreeUtil.getParentOfType(macroImpl, SdRankProfileDefinition.class))) { + results.add(macroImpl); + } + + } + + return results; + } + + +}
\ No newline at end of file diff --git a/sd-plugin/src/main/java/org/intellij/sdk/language/hierarchy/SdCallerTreeStructure.java b/sd-plugin/src/main/java/org/intellij/sdk/language/hierarchy/SdCallerTreeStructure.java new file mode 100644 index 00000000000..f6b21fc9ead --- /dev/null +++ b/sd-plugin/src/main/java/org/intellij/sdk/language/hierarchy/SdCallerTreeStructure.java @@ -0,0 +1,63 @@ +package org.intellij.sdk.language.hierarchy; + +import com.intellij.openapi.progress.ProgressManager; +import com.intellij.openapi.project.Project; +import com.intellij.psi.PsiElement; +import com.intellij.psi.PsiReference; +import com.intellij.psi.search.SearchScope; +import com.intellij.psi.search.searches.ReferencesSearch; +import com.intellij.psi.util.PsiTreeUtil; +import com.intellij.util.containers.ContainerUtil; +import org.intellij.sdk.language.psi.SdFirstPhaseDefinition; +import org.intellij.sdk.language.psi.SdFunctionDefinition; +import org.jetbrains.annotations.NotNull; + +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.function.Consumer; + +public class SdCallerTreeStructure extends SdCallTreeStructure { + + private HashMap<String, HashSet<PsiElement>> macroTreeChildren; + + public SdCallerTreeStructure(Project project, PsiElement element, String currentScopeType) { + super(project, element, currentScopeType); + macroTreeChildren = new HashMap<>(); + } + + @NotNull + @Override + protected HashSet<PsiElement> getChildren(@NotNull SdFunctionDefinition element) { + return getCallers(element, macrosMap); + } + + private HashSet<PsiElement> getCallers(@NotNull SdFunctionDefinition macro, @NotNull HashMap<String, List<PsiElement>> macrosMap) { + String macroName = macro.getName(); + + if (macroTreeChildren.containsKey(macroName)) { + return macroTreeChildren.get(macroName); + } + + HashSet<PsiElement> results = new HashSet<>(); + + for (PsiElement macroImpl : macrosMap.get(macroName)) { + SearchScope searchScope = getSearchScope(myScopeType, macroImpl); + ReferencesSearch.search(macroImpl, searchScope).forEach((Consumer<? super PsiReference>) r -> { + ProgressManager.checkCanceled(); + PsiElement psiElement = r.getElement(); + SdFunctionDefinition f = PsiTreeUtil.getParentOfType(psiElement, SdFunctionDefinition.class, false); + if (f != null && !f.getName().equals(macroName)) { + ContainerUtil.addIfNotNull(results, f); + } else { + SdFirstPhaseDefinition fp = PsiTreeUtil.getParentOfType(psiElement, SdFirstPhaseDefinition.class, false); + ContainerUtil.addIfNotNull(results, fp); + } + }); + } + + macroTreeChildren.put(macroName, results); + return results; + } + +}
\ No newline at end of file diff --git a/sd-plugin/src/main/java/org/intellij/sdk/language/hierarchy/SdHierarchyUtil.java b/sd-plugin/src/main/java/org/intellij/sdk/language/hierarchy/SdHierarchyUtil.java new file mode 100644 index 00000000000..b3376a85524 --- /dev/null +++ b/sd-plugin/src/main/java/org/intellij/sdk/language/hierarchy/SdHierarchyUtil.java @@ -0,0 +1,67 @@ +package org.intellij.sdk.language.hierarchy; + +import com.intellij.ide.hierarchy.HierarchyBrowserManager; +import com.intellij.ide.util.treeView.AlphaComparator; +import com.intellij.ide.util.treeView.NodeDescriptor; +import com.intellij.openapi.project.Project; +import com.intellij.psi.*; +import com.intellij.psi.util.PsiTreeUtil; + +import org.intellij.sdk.language.SdUtil; +import org.intellij.sdk.language.psi.SdFile; +import org.intellij.sdk.language.psi.SdFunctionDefinition; +import org.intellij.sdk.language.psi.SdRankProfileDefinition; + +import java.util.Comparator; +import java.util.HashSet; + + +public class SdHierarchyUtil { + + private static final Comparator<NodeDescriptor<?>> NODE_DESCRIPTOR_COMPARATOR = Comparator.comparingInt(NodeDescriptor::getIndex); + + private SdHierarchyUtil() { + } + + public static boolean isExecutable(PsiElement component) { + return component instanceof SdFunctionDefinition; + } + + public static HashSet<SdRankProfileDefinition> getRankProfileChildren(SdFile file, SdRankProfileDefinition rankProfileTarget) { + HashSet<SdRankProfileDefinition> result = new HashSet<>(); + HashSet<SdRankProfileDefinition> notResult = new HashSet<>(); + + for (SdRankProfileDefinition rank : PsiTreeUtil.collectElementsOfType(file, SdRankProfileDefinition.class)) { + if (notResult.contains(rank)) { + continue; + } + HashSet<SdRankProfileDefinition> tempRanks = new HashSet<>(); + SdRankProfileDefinition curRank = rank; + while (curRank != null) { + if (curRank.getName().equals(rankProfileTarget.getName())) { + result.addAll(tempRanks); + break; + } + tempRanks.add(curRank); + PsiElement temp = SdUtil.getRankProfileParent(curRank); + curRank = temp != null ? (SdRankProfileDefinition) temp : null; + } + if (curRank == null) { + notResult.addAll(tempRanks); + } + } + return result; + } + + + public static Comparator<NodeDescriptor<?>> getComparator(Project project) { + final HierarchyBrowserManager.State state = HierarchyBrowserManager.getInstance(project).getState(); + if (state != null && state.SORT_ALPHABETICALLY) { + return AlphaComparator.INSTANCE; + } + else { + return NODE_DESCRIPTOR_COMPARATOR; + } + } + +} diff --git a/sd-plugin/src/main/java/org/intellij/sdk/language/parser/SdParserDefinition.java b/sd-plugin/src/main/java/org/intellij/sdk/language/parser/SdParserDefinition.java new file mode 100644 index 00000000000..9d888a89cbf --- /dev/null +++ b/sd-plugin/src/main/java/org/intellij/sdk/language/parser/SdParserDefinition.java @@ -0,0 +1,87 @@ +package org.intellij.sdk.language.parser; + +import com.intellij.lang.ASTNode; +import com.intellij.lang.ParserDefinition; +import com.intellij.lang.PsiParser; +import com.intellij.lexer.Lexer; +import com.intellij.openapi.project.Project; +import com.intellij.psi.FileViewProvider; +import com.intellij.psi.PsiElement; +import com.intellij.psi.PsiFile; +import com.intellij.psi.TokenType; +import com.intellij.psi.tree.IFileElementType; +import com.intellij.psi.tree.TokenSet; +import org.intellij.sdk.language.SdLanguage; +import org.intellij.sdk.language.SdLexerAdapter; +import org.intellij.sdk.language.parser.SdParser; +import org.intellij.sdk.language.psi.SdFile; +import org.intellij.sdk.language.psi.SdTypes; +import org.jetbrains.annotations.NotNull; + +public class SdParserDefinition implements ParserDefinition { + public static final TokenSet WHITE_SPACES = TokenSet.create(TokenType.WHITE_SPACE); + public static final TokenSet COMMENTS = TokenSet.create(SdTypes.COMMENT); + public static final TokenSet STRINGS = TokenSet.create(SdTypes.STRING); + + public static final IFileElementType FILE = new IFileElementType(SdLanguage.INSTANCE); + + @NotNull + @Override + public Lexer createLexer(Project project) { + return new SdLexerAdapter(); + } + + @NotNull + @Override + public PsiParser createParser(final Project project) { + return new SdParser(); + } + + @NotNull + @Override + public TokenSet getWhitespaceTokens() { + return WHITE_SPACES; + } + + @NotNull + @Override + public TokenSet getCommentTokens() { + return COMMENTS; + } + + @NotNull + @Override + public TokenSet getStringLiteralElements() { + return STRINGS; + } + + @NotNull + @Override + public IFileElementType getFileNodeType() { + return FILE; + } + + @NotNull + @Override + public PsiFile createFile(@NotNull FileViewProvider viewProvider) { + return new SdFile(viewProvider); + } + + @NotNull + @Override + public SpaceRequirements spaceExistenceTypeBetweenTokens(ASTNode left, ASTNode right) { + return SpaceRequirements.MAY; + } + + @NotNull + @Override + public PsiElement createElement(ASTNode node) { + return SdTypes.Factory.createElement(node); + } + +} + + + + +
\ No newline at end of file diff --git a/sd-plugin/src/main/java/org/intellij/sdk/language/parser/sd.bnf b/sd-plugin/src/main/java/org/intellij/sdk/language/parser/sd.bnf new file mode 100644 index 00000000000..a1bc4c548a0 --- /dev/null +++ b/sd-plugin/src/main/java/org/intellij/sdk/language/parser/sd.bnf @@ -0,0 +1,355 @@ +{ + parserClass="org.intellij.sdk.language.parser.SdParser" // Name and the location of the parser which will be generated. + + extends="com.intellij.extapi.psi.ASTWrapperPsiElement" // All nodes will extend this class. Wraps AST node to a PSI node. + + // Prefix and suffix for all generated classes + psiClassPrefix="Sd" + psiImplClassSuffix="Impl" + + psiPackage="org.intellij.sdk.language.psi" // Location to be used when generating PSI classes. + psiImplPackage="org.intellij.sdk.language.psi.impl" // Location to be used when generating PSI implementation classes. + + elementTypeHolderClass="org.intellij.sdk.language.psi.SdTypes" // Element type holder class name. + + elementTypeClass="org.intellij.sdk.language.psi.SdElementType" // Class which will be used to create internal nodes. + tokenTypeClass="org.intellij.sdk.language.psi.SdTokenType" // Class which will be used to create leaf nodes. + + psiImplUtilClass="org.intellij.sdk.language.psi.impl.SdPsiImplUtil" + + extends(".*Expr")=RankingExpression // Here to deal with left-recursion that happens in expressions + + tokens = [ + ID_REG = 'regexp:[a-zA-Z_][a-zA-Z0-9_]*' + ID_WITH_DASH_REG = 'regexp:[a-zA-Z_][a-zA-Z0-9_-]*' + WHITE_SPACE = 'regexp:\s+' + COMMENT = 'regexp:#.*' + SYMBOL = 'regexp:[|:{}(),.\[\]]' + COMPARISON_OPERATOR = 'regexp:[<>]|(==)|(<=)|(>=)|(~=)' + ARITHMETIC_OPERATOR = 'regexp:[\-+*/]' + INTEGER_REG = 'regexp:[0-9]+' + FLOAT_REG = 'regexp:[0-9]+[.][0-9]+[e]?' + STRING = 'regexp:[\"][^\"]*[\"]' + WORD_REG = 'regexp:\w+' + ] +} + +// IMPORTANT NOTE: This grammar does not enforce zero-or-one occurrences of elements (treats it like zero-to-many) + +SdFile ::= Schema +Schema ::= search IdentifierVal? '{' SchemaBody '}' +SchemaBody ::= SchemaBodyOptions* DocumentDefinition SchemaBodyOptions* // Does not support zero-or-one occurrences +private SchemaBodyOptions ::= SchemaFieldDefinition | ImportFieldDefinition | DocumentSummaryDefinition | + RankProfileDefinition | + FieldSetDefinition | ConstantDefinition | OnnxModelDefinition | StemmingDefinition | + raw-as-base64-in-summary | AnnotationDefinition + + + + +SchemaFieldDefinition ::= field IdentifierVal type FieldTypeName '{' SchemaFieldBody '}' + { mixin="org.intellij.sdk.language.psi.impl.SdNamedElementImpl" + implements=["org.intellij.sdk.language.psi.SdDeclaration" "org.intellij.sdk.language.psi.SdNamedElement"] + methods=[getName setName getType getTypeName getNameIdentifier getPresentation] } + +FieldTypeName ::= "array" '<' (FieldTypeName | IdentifierVal) '>' | "weightedset" '<' SingleValueFieldTypeName '>'| + "map" '<' (FieldTypeName | IdentifierVal) ',' (FieldTypeName | IdentifierVal) '>' | TensorType | + SingleValueFieldTypeName +private SingleValueFieldTypeName ::= "string" | "int" | "long" | "bool" | "byte" | "float" | "double" | "position" | "predicate" | "raw" | "uri" | + "reference" '<' IdentifierVal '>' | "annotationreference" '<' IdentifierVal '>' | IdentifierVal +private TensorType ::= "tensor" '<' ("float" | "double" | "int8" | "bfloat16") '>' '(' TensorDimension (',' TensorDimension)* ')' +private TensorDimension ::= WORD_REG ('{' '}') | ('[' INTEGER_REG ']') + +SchemaFieldBody ::= SchemaFieldBodyOptions* // Does not support zero-or-one occurrences +SchemaFieldBodyOptions ::= SchemaFieldIndexingDefinition | AttributeDefinition | RankDefinition | IndexingRewriteState | + MatchDefinition | StructFieldDefinition | QueryCommandDefinition + +SchemaFieldIndexingDefinition ::= indexing (':' SchemaFieldIndexingStatement) | ('{' SchemaFieldIndexingStatement+ '}') +SchemaFieldIndexingStatement ::= (input IdentifierVal) | IndexingStatement + +DocumentSummaryDefinition ::= document-summary IdentifierWithDashVal (inherits IdentifierWithDashVal)? '{' DocumentSummaryBody '}' + { mixin="org.intellij.sdk.language.psi.impl.SdNamedElementImpl" + implements=["org.intellij.sdk.language.psi.SdDeclaration" "org.intellij.sdk.language.psi.SdNamedElement"] + methods=[getName setName getType getTypeName getNameIdentifier getPresentation] } +DocumentSummaryBody ::= DocumentSummaryBodyOptions* // Does not support zero-or-one occurrences +private DocumentSummaryBodyOptions ::= SummaryDefinition | omit-summary-features | from-disk + +ImportFieldDefinition ::= import field IdentifierVal '.' IdentifierVal as IdentifierVal '{''}' + { mixin="org.intellij.sdk.language.psi.impl.SdNamedElementImpl" + implements=["org.intellij.sdk.language.psi.SdDeclaration" "org.intellij.sdk.language.psi.SdNamedElement"] + methods=[getName setName getType getTypeName getNameIdentifier getPresentation] } + +FieldSetDefinition ::= fieldset IdentifierVal '{' FieldSetBody '}' +FieldSetBody ::= FieldSetBodyOptions* +private FieldSetBodyOptions ::= (fields ':' IdentifierVal (',' IdentifierVal)*) | QueryCommandDefinition | MatchDefinition + +ConstantDefinition ::= constant IdentifierVal '{' ConstantBody '}' +ConstantBody ::= ConstantBodyOptions* +private ConstantBodyOptions ::= (file ':' FilePath) | (uri ':' UriPath) | (type ':' TensorType) +private FilePath ::= (IdentifierVal | WORD_REG) ('.' | '/' | IdentifierWithDashVal | WORD_REG)+ +private UriPath ::= ('H'|'h') ('T'|'t') ('T'|'t') ('P'|'p') ('S'|'s')? ':' ('//')? (IdentifierWithDashVal | '.' | '/' | ':')+ + + +OnnxModelDefinition ::= onnx-model IdentifierVal '{' OnnxModelBody '}' +OnnxModelBody ::= OnnxModelBodyOptions* +private OnnxModelBodyOptions ::= (file ':' FilePath) | (uri ':' UriPath) | + ((input | output) (IdentifierVal | STRING) ':' ('.' | '/' | '(' | ')' | IdentifierWithDashVal | WORD_REG)) + +AnnotationDefinition ::= annotation IdentifierVal '{' '}' // todo ask Vespa for syntax + +//------------------------- +//--- Expressions rules --- +//------------------------- +RankingExpression ::= ParenthesisedExpr | BooleanExpr |ArithmeticExpr | IfFunctionExpr | + QueryDefinitionExpr | FunctionCallExpr | PrimitiveExpr //ReduceExpr | + +IfFunctionExpr ::= "if" '(' (InListRankingExpression | RankingExpression) ',' RankingExpression ',' RankingExpression ')' +InListRankingExpression ::= RankingExpression "in" '[' RankingExpression (',' RankingExpression)* ']' + +//ReduceExpr ::= reduce '(' RankingExpression ',' (avg|count|max|median|min|prod|sum) (',' RankingExpression)* ')' + +BooleanExpr ::= RankingExpression COMPARISON_OPERATOR RankingExpression + +ArithmeticExpr ::= RankingExpression ARITHMETIC_OPERATOR RankingExpression + +QueryDefinitionExpr ::= QueryDefinition | ItemRawScoreDefinition + +FunctionCallExpr ::= IdentifierWithDashVal '(' RankingExpression (',' RankingExpression)* ')' ('.' IdentifierWithDashVal)? + +ParenthesisedExpr ::= '(' RankingExpression ')' + +PrimitiveExpr ::= (('-')? INTEGER_REG) | (('-')? FLOAT_REG) | IdentifierVal | RankFeature + +//------------------------- +//-- Rank Profile rules --- +//------------------------- +RankProfileDefinition ::= rank-profile IdentifierWithDashVal (inherits IdentifierWithDashVal)? '{' RankProfileBody '}' + { mixin="org.intellij.sdk.language.psi.impl.SdNamedElementImpl" + implements=["org.intellij.sdk.language.psi.SdDeclaration" "org.intellij.sdk.language.psi.SdNamedElement"] + methods=[getName setName getType getTypeName getNameIdentifier getPresentation] } +private RankProfileBody ::= RankProfileBodyOptions* // Does not support zero-or-one occurrences +private RankProfileBodyOptions ::= MatchPhaseDefinition | NumThreadsDefinition | FunctionDefinition | TermwiseLimitDefinition | + ignore-default-rank-features | RankPropertiesDefinition | FirstPhaseDefinition | + SummaryFeaturesDefinition | RankFeaturesDefinition | SecondPhaseDefinition | ConstantsDefinition | + RankDefinition | RankTypeDefinition | MinHitsDefinition | NumSearchPartitionDefinition + // | FieldWeightDefinition | ExecuteDefinition | RankDegradationDefinition // todo check with Vespa if need to add these + +MatchPhaseDefinition ::= match-phase '{' MatchPhaseBody '}' +MatchPhaseBody ::= MatchPhaseBodyOptions+ // todo check with Vespa- are there more options? +MatchPhaseBodyOptions ::= (attribute ':' IdentifierVal (order ':' (ascending | descending))?) | (max-hits ':' INTEGER_REG) + | DiversityDefinition // Does not support zero-or-one occurrences +DiversityDefinition ::= diversity '{' (attribute ':' IdentifierVal min-groups ':' INTEGER_REG) | + (min-groups ':' INTEGER_REG attribute ':' IdentifierVal) '}' + +private NumThreadsDefinition ::= num-threads-per-search ':' INTEGER_REG +private TermwiseLimitDefinition ::= termwise-limit ':' (FLOAT_REG | INTEGER_REG) +private MinHitsDefinition ::= min-hits-per-thread ':' INTEGER_REG +private NumSearchPartitionDefinition ::= num-search-partition ':' INTEGER_REG +FirstPhaseDefinition ::= first-phase '{' FirstPhaseBody '}' { methods=[getName getPresentation] } +FirstPhaseBody ::= FirstPhaseBodyOptions* // Does not support zero-or-one occurrences +private FirstPhaseBodyOptions ::= (keep-rank-count ':' INTEGER_REG) | (rank-score-drop-limit ':' (FLOAT_REG | INTEGER_REG)) | ExpressionDefinition + +ExpressionDefinition ::= expression ((':' RankingExpression) | ('{' RankingExpression* '}') | + (':' file ':' FilePath)) + +SecondPhaseDefinition ::= second-phase '{' SecondPhaseBody '}' +SecondPhaseBody ::= SecondPhaseBodyOptions* +private SecondPhaseBodyOptions ::= (rerank-count ':' INTEGER_REG) | ExpressionDefinition + +RankPropertiesDefinition ::= rank-properties '{' RankPropertiesBody '}' +RankPropertiesBody ::= (RankPropertiesKey ':' RankPropertiesValue)+ +RankPropertiesKey ::= (IdentifierWithDashVal | STRING | '(' | ')' | '.' | ',')+ +RankPropertiesValue ::= (('-')? INTEGER_REG) | (('-')? FLOAT_REG) | WORD_REG | IdentifierVal | STRING + +FunctionDefinition ::= (function | macro) inline? IdentifierVal '(' (ArgumentDefinition (',' ArgumentDefinition)*)? ')' + '{' ExpressionDefinition '}' + { mixin="org.intellij.sdk.language.psi.impl.SdNamedElementImpl" + implements=["org.intellij.sdk.language.psi.SdDeclaration" "org.intellij.sdk.language.psi.SdNamedElement"] + methods=[getName setName getType getTypeName getNameIdentifier getPresentation isOverride] } +ArgumentDefinition ::= IdentifierVal + { mixin="org.intellij.sdk.language.psi.impl.SdNamedElementImpl" + implements=["org.intellij.sdk.language.psi.SdDeclaration" "org.intellij.sdk.language.psi.SdNamedElement"] + methods=[getName setName getType getTypeName getNameIdentifier getPresentation] } + +SummaryFeaturesDefinition ::= summary-features ((':' RankFeature+) | ((inherits IdentifierVal)? '{' RankFeature* '}')) + +RankFeaturesDefinition ::= rank-features (':' RankFeature+) | ('{' RankFeature* '}') + +ConstantsDefinition ::= constants '{' (IdentifierVal ':' RankPropertiesValue)* '}' + +//******** Rank features ********* +RankFeature ::= QueryDefinition | ItemRawScoreDefinition | FunctionCallExpr | (IdentifierWithDashVal ('.' IdentifierWithDashVal)* ) + +QueryDefinition ::= "query" '(' IdentifierWithDashVal ')' + { mixin="org.intellij.sdk.language.psi.impl.SdNamedElementImpl" + implements=["org.intellij.sdk.language.psi.SdDeclaration" "org.intellij.sdk.language.psi.SdNamedElement"] + methods=[getName setName getType getTypeName getNameIdentifier getPresentation] } +ItemRawScoreDefinition ::= "itemRawScore" '(' IdentifierVal ')' + { mixin="org.intellij.sdk.language.psi.impl.SdNamedElementImpl" + implements=["org.intellij.sdk.language.psi.SdDeclaration" "org.intellij.sdk.language.psi.SdNamedElement"] + methods=[getName setName getType getTypeName getNameIdentifier getPresentation] } + +// QueryFeature | DocumentFeature | QueryTermFieldMatchFeature | AttributeMatchFeature | RankScoreFeature | +// GlobalFeature | MatchOperatorScoreFeature | UtilityFeature | RankingExpressionMethod + //todo add the rest (| FieldMatchFeature | QueryFieldSimilarityFeature | IdxMultiStrFieldFeature | MultiFieldsAttributesFeature) + +// todo maybe not specify all of the rank features here? Maybe it would be better to write it more generally? like ID '(' ID ')' '.' ID.. so it could handle new features in the future +//QueryFeature ::= QueryDefinition | (term '(' (FLOAT_REG | INTEGER_REG) ')' '.' (significant | weight | connectedness)) // todo add queryTermCount +//DocumentFeature ::= (fieldLength '(' IdentifierVal ')') | (attribute '(' IdentifierVal ')' ('.' count)?) | +// (attribute '(' IdentifierVal ',' (INTEGER_REG | IdentifierVal) ')' ('.' (weight | contains))?) | +// (tensorFromWeightedSet '(' ((attribute'('IdentifierVal')') | QueryDefinition | IdentifierVal) ',' IdentifierVal ')') | +// (tensorFromLabels '(' ((attribute'('IdentifierVal')') | QueryDefinition | IdentifierVal) ',' IdentifierVal ')') +//QueryTermFieldMatchFeature ::= (matchCount'('IdentifierVal')') | (matches'('IdentifierVal (',' INTEGER_REG)? ')') // todo add the rest +//AttributeMatchFeature ::= (attributeMatch'('IdentifierVal')' ('.' (completeness | queryCompleteness | fieldCompleteness | +// normalizedWeight | normalizedWeightedWeight | matches | totalWeight | averageWeight | maxWeight))?) | +// (distance'('IdentifierVal')' ('.' (index | latitude | longitude))?) | +// (distanceToPath'('IdentifierVal')' ('.' (distance | traveled | product))?) | (age'('IdentifierVal')') // todo add closeness and freshness +//RankScoreFeature ::= (nativeDotProduct'('IdentifierVal')') // todo add the rest +//GlobalFeature ::= (random'('IdentifierVal')' '.' match) | (random) // todo add the rest +//MatchOperatorScoreFeature ::= (rawScore'('IdentifierVal')') | ItemRawScoreDefinition +//UtilityFeature ::= (dotProduct'('IdentifierVal ',' IdentifierVal ')') // todo add the rest +//RankingExpressionMethod ::= "rankingExpression" '(' (RankFeature | IdentifierVal) ')' +//***** End of Rank features ***** + + +//------------------------- +//---- Document rules ----- +//------------------------- +DocumentDefinition ::= document (IdentifierVal (inherits IdentifierVal (',' IdentifierVal)*)?)? '{' DocumentBody '}' + { mixin="org.intellij.sdk.language.psi.impl.SdNamedElementImpl" + implements=["org.intellij.sdk.language.psi.SdDeclaration" "org.intellij.sdk.language.psi.SdNamedElement"] + methods=[getName setName getType getTypeName getNameIdentifier getPresentation] } + +DocumentBody ::= DocumentBodyOptions* +DocumentBodyOptions ::= DocumentStructDefinition | DocumentFieldDefinition + +DocumentStructDefinition ::= struct IdentifierVal '{' DocumentStructBody '}' + { mixin="org.intellij.sdk.language.psi.impl.SdNamedElementImpl" + implements=["org.intellij.sdk.language.psi.SdDeclaration" "org.intellij.sdk.language.psi.SdNamedElement"] + methods=[getName setName getType getTypeName getNameIdentifier getPresentation] } + +DocumentStructBody ::= DocumentStructFieldDefinition* +DocumentStructFieldDefinition ::= field IdentifierVal type FieldTypeName '{' DocumentStructFieldBody '}' + { mixin="org.intellij.sdk.language.psi.impl.SdNamedElementImpl" + implements=["org.intellij.sdk.language.psi.SdDeclaration" "org.intellij.sdk.language.psi.SdNamedElement"] + methods=[getName setName getType getTypeName getNameIdentifier getPresentation] } + +DocumentStructFieldBody ::= MatchDefinition? + +DocumentFieldDefinition ::= field IdentifierVal type FieldTypeName '{' DocumentFieldBody '}' + { mixin="org.intellij.sdk.language.psi.impl.SdNamedElementImpl" + implements=["org.intellij.sdk.language.psi.SdDeclaration" "org.intellij.sdk.language.psi.SdNamedElement"] + methods=[getName setName getType getTypeName getNameIdentifier getPresentation] } + +DocumentFieldBody ::= DocumentFieldBodyOptions* // Does not support zero-or-one occurrences +private DocumentFieldBodyOptions ::= StructFieldDefinition | MatchDefinition | IndexingDefinition | AttributeDefinition | + AliasDef | RankDefinition | IndexingRewriteState | QueryCommandDefinition | SummaryDefinition | + BoldingDefinition | (id ':' INTEGER_REG) | IndexDefinition | (normalizing ':' IdentifierWithDashVal) | + SortingDefinition | StemmingDefinition | (weight ': INTEGER_REG') | WeightedSetDefinition | + RankTypeDefinition | DictionaryDefinition // todo check with Vespa- SummaryToDefinition is needed here? it's deprecated + +//***** Field's body elements ******// +// Struct +StructFieldDefinition ::= struct-field IdentifierVal '{' StructFieldBody '}' + { mixin="org.intellij.sdk.language.psi.impl.SdNamedElementImpl" + implements=["org.intellij.sdk.language.psi.SdDeclaration" "org.intellij.sdk.language.psi.SdNamedElement"] + methods=[getName setName getType getTypeName getNameIdentifier getPresentation] } + +StructFieldBody ::= StructFieldBodyOptions* // Does not support zero-or-one occurrences +StructFieldBodyOptions ::= IndexingDefinition | AttributeDefinition | MatchDefinition | QueryCommandDefinition | + StructFieldDefinition | SummaryDefinition +// Match +MatchDefinition ::= match ((':' MatchProperty) | ('{' MatchProperty+ '}')) +MatchProperty ::= text | exact | exact-terminator | word | prefix | cased | uncased | substring | suffix | max-length | + gram | gram-size +// Indexing +IndexingDefinition ::= indexing (':' IndexingStatement) | ('{' IndexingStatement+ '}') +IndexingStatement ::= IndexingStatementOptions (('|' IndexingStatementOptions)*) | ((';' IndexingStatementOptions)*) + // Does not support zero-or-one occurrences // todo check with Vespa- Can "input" be here? +IndexingStatementOptions ::= summary | attribute | index | "set_language" +// Attribute +AttributeDefinition ::= attribute ((':' SimpleAttributeProperty) | ('{' (SimpleAttributeProperty | ComplexAttributeProperty)+ '}')) +SimpleAttributeProperty ::= fast-search | fast-access | paged | mutable // Does not support zero-or-one occurrences +ComplexAttributeProperty ::= AliasDef | SortingDefinition | DistanceMetricDef // Does not support zero-or-one occurrences +DistanceMetricDef ::= distance-metric ':' IdentifierWithDashVal +// Alias +AliasDef ::= alias IdentifierVal? ':' IdentifierWithDashVal +// Stemming +StemmingDefinition ::= stemming ':' IdentifierWithDashVal +// Rank +RankDefinition ::= rank ((IdentifierVal? ':' RankingSetting) | ('{' RankingSetting '}')) +RankingSetting ::= filter | normal | literal // todo check with Vespa- is "literal" good here? +// Indexing Rewrite +IndexingRewriteState ::= indexing-rewrite ':' none +// Query Command +QueryCommandDefinition ::= query-command ':' IdentifierVal | STRING +// Summary +SummaryDefinition ::= summary ((':' SummaryBodyOptions) | (IdentifierWithDashVal? (type FieldTypeName)? '{' SummaryBody '}')) + { methods=[getName getPresentation] } +SummaryBody ::= SummaryBodyOptions* // Does not support zero-or-one occurrences +SummaryBodyOptions ::= full | dynamic | (source ':' IdentifierVal (',' IdentifierVal)*) | + (to ':' IdentifierVal (',' IdentifierVal)*) | matched-elements-only +// Bolding +BoldingDefinition ::= bolding ':' (on | off | true | false) +// Index +IndexDefinition ::= index IdentifierVal (':' IndexProperty) | ('{' IndexProperty* '}') +IndexProperty ::= IndexPropertyOptions* +private IndexPropertyOptions ::= (alias ':' IdentifierWithDashVal) | StemmingDefinition | (arity ':' INTEGER_REG) | + (lower-bound ':' INTEGER_REG ('L')?) | (upper-bound ':' INTEGER_REG ('L')?) | + (dense-posting-list-threshold ':' FLOAT_REG) | enable-bm25 | HnswDefinition +HnswDefinition ::= hnsw '{' HnswBody '}' +HnswBody ::= HnswBodyOptions* +private HnswBodyOptions ::= (max-links-per-node ':' INTEGER_REG) | (neighbors-to-explore-at-insert ':' INTEGER_REG) | + (multi-threaded-indexing ':' (on | off | true | false)) +// Sorting +SortingDefinition ::= sorting (':' SortingProperty) | ('{' SortingProperty* '}') +SortingProperty ::= ascending | descending | (function ':' SortingFunction) | (strength ':' SortingStrength) | + (locale ':' IdentifierWithDashVal) +SortingFunction ::= uca | raw | lowercase +SortingStrength ::= primary | secondary | tertiary | quaternary | identical +// Rank Type +RankTypeDefinition ::= rank-type IdentifierVal ':' IdentifierVal +// Weighted Set +WeightedSetDefinition ::= weightedset (':' WeightedSetProperty) | ('{' WeightedSetProperty* '}') // Does not support + // zero-or-one occurrences +WeightedSetProperty ::= create-if-nonexistent | remove-if-zero +// Dictionary +DictionaryDefinition ::= dictionary (':' DictionarySetting) | ('{' DictionarySetting* '}') +DictionarySetting ::= hash | btree | cased | uncased +//***** End of Field's body elements ******// + +//--------------------- +//---- Util rules ----- +//--------------------- + +IdentifierVal ::= KeywordOrIdentifier | ID_REG { implements=["org.intellij.sdk.language.psi.SdIdentifier"] + methods=[getName setName getNameIdentifier getReference isFunctionName] } + +IdentifierWithDashVal ::= ID_WITH_DASH_REG | IdentifierVal { implements=["org.intellij.sdk.language.psi.SdIdentifier"] + methods=[getName setName getNameIdentifier getReference isFunctionName] } + +// Those lists of keywords (KeywordOrIdentifier and KeywordNotIdentifier) have to be synchronized with sd.flex file. +// If you add a keyword here, you should add it to the sd.flex file as well. +KeywordOrIdentifier ::= search | document | struct | field | type | indexing | input | output | inherits | import | as | + raw | uri | file | annotationreference | array | weightedset | map | + order | ascending | descending | diversity | constants | literal | expression | weight | match | + function | macro | inline | text | exact | word | prefix | cased | uncased | substring | suffix | + gram | paged | mutable | alias | sorting | strength | locale | uca | lowercase | + primary | secondary | tertiary | quaternary | identical | rank | filter | normal | none | full | dynamic | + source | to | + bolding | on | off | true | false | id | normalizing | stemming | arity | hnsw | dictionary | hash | btree | + fieldset | fields | constant | annotation + // KeywordNotIdentifier ::= (not enforced in this version) + | attribute | body | header | index | + reference | summary + +KeywordNotIdentifier ::= struct-field | document-summary | omit-summary-features | from-disk | rank-profile | rank-type | + num-threads-per-search | termwise-limit | ignore-default-rank-features | min-hits-per-thread | + num-search-partition | match-phase | max-hits | second-phase | rerank-count | min-groups | + first-phase | keep-rank-count | rank-score-drop-limit | rank-properties | summary-features | + exact-terminator | max-length | gram-size | fast-search | fast-access | distance-metric | + indexing-rewrite | query-command | matched-elements-only | lower-bound | upper-bound | + dense-posting-list-threshold | enable-bm25 | max-links-per-node | neighbors-to-explore-at-insert | + multi-threaded-indexing | create-if-nonexistent | remove-if-zero | raw-as-base64-in-summary | + onnx-model +
\ No newline at end of file diff --git a/sd-plugin/src/main/java/org/intellij/sdk/language/psi/SdDeclaration.java b/sd-plugin/src/main/java/org/intellij/sdk/language/psi/SdDeclaration.java new file mode 100644 index 00000000000..c46466ddb84 --- /dev/null +++ b/sd-plugin/src/main/java/org/intellij/sdk/language/psi/SdDeclaration.java @@ -0,0 +1,14 @@ +package org.intellij.sdk.language.psi; + +import com.intellij.navigation.NavigationItem; +import com.intellij.psi.PsiElement; +import com.intellij.psi.PsiNamedElement; + +public interface SdDeclaration extends PsiElement, PsiNamedElement, NavigationItem { + String getName(); + + String getTypeName(); + + SdDeclarationType getType(); + +} diff --git a/sd-plugin/src/main/java/org/intellij/sdk/language/psi/SdDeclarationType.java b/sd-plugin/src/main/java/org/intellij/sdk/language/psi/SdDeclarationType.java new file mode 100644 index 00000000000..f832a0d01e2 --- /dev/null +++ b/sd-plugin/src/main/java/org/intellij/sdk/language/psi/SdDeclarationType.java @@ -0,0 +1,27 @@ +package org.intellij.sdk.language.psi; + +public enum SdDeclarationType { + DOCUMENT("Document"), + STRUCT("Struct"), + SCHEMA_FIELD("Field (in Schema)"), + DOCUMENT_FIELD("Field (in Document)"), + STRUCT_FIELD("Struct-Field"), + DOCUMENT_STRUCT_FIELD("Field (in Struct)"), + IMPORTED_FIELD("Imported Field"), + DOCUMENT_SUMMARY("Document-Summary"), + RANK_PROFILE("Rank Profile"), + MACRO("Macro"), + MACRO_ARGUMENT("Macro's Argument"), + QUERY("Query (first use in file)"), + ITEM_RAW_SCORE("ItemRawScore (first use in file)"); + + private final String typeName; + SdDeclarationType(String name) { + this.typeName = name; + } + + @Override + public String toString() { + return typeName; + } +} diff --git a/sd-plugin/src/main/java/org/intellij/sdk/language/psi/SdElementDescriptionProvider.java b/sd-plugin/src/main/java/org/intellij/sdk/language/psi/SdElementDescriptionProvider.java new file mode 100644 index 00000000000..ea588f965e5 --- /dev/null +++ b/sd-plugin/src/main/java/org/intellij/sdk/language/psi/SdElementDescriptionProvider.java @@ -0,0 +1,29 @@ +package org.intellij.sdk.language.psi; + +import com.intellij.psi.ElementDescriptionLocation; +import com.intellij.psi.ElementDescriptionProvider; +import com.intellij.psi.PsiElement; +import com.intellij.psi.PsiNamedElement; +import org.intellij.sdk.language.psi.SdDeclaration; +import org.jetbrains.annotations.NotNull; +import org.jetbrains.annotations.Nullable; + + +public class SdElementDescriptionProvider implements ElementDescriptionProvider { + + /** + * Controls the headline of the element in the "Find Usages" window + * @param psiElement the element to describe + * @param elementDescriptionLocation + * @return a string with the description to write in the headline + */ + @Nullable + @Override + public String getElementDescription(@NotNull PsiElement psiElement, @NotNull ElementDescriptionLocation elementDescriptionLocation) { + if (psiElement instanceof SdDeclaration) { + return ((SdDeclaration) psiElement).getTypeName(); + } else { + return ""; + } + } +} diff --git a/sd-plugin/src/main/java/org/intellij/sdk/language/psi/SdElementFactory.java b/sd-plugin/src/main/java/org/intellij/sdk/language/psi/SdElementFactory.java new file mode 100644 index 00000000000..0961704443c --- /dev/null +++ b/sd-plugin/src/main/java/org/intellij/sdk/language/psi/SdElementFactory.java @@ -0,0 +1,30 @@ +package org.intellij.sdk.language.psi; + +import com.intellij.openapi.project.Project; +import com.intellij.psi.PsiFileFactory; + +import com.intellij.psi.util.PsiTreeUtil; +import org.intellij.sdk.language.SdFileType; + +public class SdElementFactory { + + private static final String GENERAL_FILE_TEXT = "search {document %s {} rank-profile %s {}}"; + + public static SdIdentifierVal createIdentifierVal(Project project, String name) { + String fileText = String.format(GENERAL_FILE_TEXT, name, name); + final SdFile file = createFile(project, fileText); + return PsiTreeUtil.findChildOfType(file, SdIdentifierVal.class); + } + + public static SdIdentifierWithDashVal createIdentifierWithDashVal(Project project, String name) { + String fileText = String.format(GENERAL_FILE_TEXT, name, name); + final SdFile file = createFile(project, fileText); + return PsiTreeUtil.findChildOfType(file, SdIdentifierWithDashVal.class); + } + + public static SdFile createFile(Project project, String text) { + String name = "dummy.sd"; + return (SdFile) PsiFileFactory.getInstance(project). + createFileFromText(name, SdFileType.INSTANCE, text); + } +} diff --git a/sd-plugin/src/main/java/org/intellij/sdk/language/psi/SdElementType.java b/sd-plugin/src/main/java/org/intellij/sdk/language/psi/SdElementType.java new file mode 100644 index 00000000000..a17ae3c1c07 --- /dev/null +++ b/sd-plugin/src/main/java/org/intellij/sdk/language/psi/SdElementType.java @@ -0,0 +1,13 @@ +package org.intellij.sdk.language.psi; + +import com.intellij.psi.tree.IElementType; +import org.intellij.sdk.language.SdLanguage; +import org.jetbrains.annotations.NonNls; +import org.jetbrains.annotations.NotNull; + +public class SdElementType extends IElementType { + + public SdElementType(@NotNull @NonNls String debugName) { + super(debugName, SdLanguage.INSTANCE); + } +} diff --git a/sd-plugin/src/main/java/org/intellij/sdk/language/psi/SdFile.java b/sd-plugin/src/main/java/org/intellij/sdk/language/psi/SdFile.java new file mode 100644 index 00000000000..0e0a8b0661c --- /dev/null +++ b/sd-plugin/src/main/java/org/intellij/sdk/language/psi/SdFile.java @@ -0,0 +1,26 @@ +package org.intellij.sdk.language.psi; + +import com.intellij.extapi.psi.PsiFileBase; +import com.intellij.openapi.fileTypes.FileType; +import com.intellij.psi.FileViewProvider; +import org.intellij.sdk.language.SdFileType; +import org.intellij.sdk.language.SdLanguage; +import org.jetbrains.annotations.NotNull; + +public class SdFile extends PsiFileBase { + + public SdFile(@NotNull FileViewProvider viewProvider) { + super(viewProvider, SdLanguage.INSTANCE); + } + + @NotNull + @Override + public FileType getFileType() { + return SdFileType.INSTANCE; + } + + @Override + public String toString() { + return "Sd File"; + } +} diff --git a/sd-plugin/src/main/java/org/intellij/sdk/language/psi/SdIdentifier.java b/sd-plugin/src/main/java/org/intellij/sdk/language/psi/SdIdentifier.java new file mode 100644 index 00000000000..27009e9369c --- /dev/null +++ b/sd-plugin/src/main/java/org/intellij/sdk/language/psi/SdIdentifier.java @@ -0,0 +1,15 @@ +package org.intellij.sdk.language.psi; + +import com.intellij.psi.PsiElement; +import com.intellij.psi.PsiReference; +import org.jetbrains.annotations.NotNull; + +public interface SdIdentifier extends PsiElement { + + String getName(); + + boolean isFunctionName(PsiElement file); + + @NotNull PsiReference getReference(); + +} diff --git a/sd-plugin/src/main/java/org/intellij/sdk/language/psi/SdNamedElement.java b/sd-plugin/src/main/java/org/intellij/sdk/language/psi/SdNamedElement.java new file mode 100644 index 00000000000..7f4133246a0 --- /dev/null +++ b/sd-plugin/src/main/java/org/intellij/sdk/language/psi/SdNamedElement.java @@ -0,0 +1,7 @@ +package org.intellij.sdk.language.psi; + +import com.intellij.psi.PsiNameIdentifierOwner; + +public interface SdNamedElement extends PsiNameIdentifierOwner { + +} diff --git a/sd-plugin/src/main/java/org/intellij/sdk/language/psi/SdTokenType.java b/sd-plugin/src/main/java/org/intellij/sdk/language/psi/SdTokenType.java new file mode 100644 index 00000000000..ec587ceeb86 --- /dev/null +++ b/sd-plugin/src/main/java/org/intellij/sdk/language/psi/SdTokenType.java @@ -0,0 +1,18 @@ +package org.intellij.sdk.language.psi; + +import com.intellij.psi.tree.IElementType; +import org.intellij.sdk.language.SdLanguage; +import org.jetbrains.annotations.NonNls; +import org.jetbrains.annotations.NotNull; + +public class SdTokenType extends IElementType { + + public SdTokenType(@NotNull @NonNls String debugName) { + super(debugName, SdLanguage.INSTANCE); + } + + @Override + public String toString() { + return "SdTokenType." + super.toString(); + } +} diff --git a/sd-plugin/src/main/java/org/intellij/sdk/language/psi/impl/SdNamedElementImpl.java b/sd-plugin/src/main/java/org/intellij/sdk/language/psi/impl/SdNamedElementImpl.java new file mode 100644 index 00000000000..999fc4bd290 --- /dev/null +++ b/sd-plugin/src/main/java/org/intellij/sdk/language/psi/impl/SdNamedElementImpl.java @@ -0,0 +1,14 @@ +package org.intellij.sdk.language.psi.impl; + +import com.intellij.extapi.psi.ASTWrapperPsiElement; +import com.intellij.lang.ASTNode; +import org.intellij.sdk.language.psi.SdNamedElement; +import org.jetbrains.annotations.NotNull; + +public abstract class SdNamedElementImpl extends ASTWrapperPsiElement implements SdNamedElement { + + public SdNamedElementImpl(@NotNull ASTNode node) { + super(node); + } + +} diff --git a/sd-plugin/src/main/java/org/intellij/sdk/language/psi/impl/SdPsiImplUtil.java b/sd-plugin/src/main/java/org/intellij/sdk/language/psi/impl/SdPsiImplUtil.java new file mode 100644 index 00000000000..d22caa5adaa --- /dev/null +++ b/sd-plugin/src/main/java/org/intellij/sdk/language/psi/impl/SdPsiImplUtil.java @@ -0,0 +1,352 @@ +package org.intellij.sdk.language.psi.impl; + +import com.intellij.icons.AllIcons; +import com.intellij.lang.ASTNode; +import com.intellij.navigation.ItemPresentation; +import com.intellij.openapi.util.TextRange; +import com.intellij.psi.PsiElement; +import com.intellij.psi.PsiFile; +import com.intellij.psi.PsiReference; +import com.intellij.psi.util.PsiTreeUtil; +import org.intellij.sdk.language.SdIcons; +import org.intellij.sdk.language.SdReference; +import org.intellij.sdk.language.SdUtil; +import org.intellij.sdk.language.hierarchy.SdHierarchyUtil; +import org.intellij.sdk.language.psi.SdArgumentDefinition; +import org.intellij.sdk.language.psi.SdDeclaration; +import org.intellij.sdk.language.psi.SdDeclarationType; +import org.intellij.sdk.language.psi.SdDocumentDefinition; +import org.intellij.sdk.language.psi.SdDocumentFieldDefinition; +import org.intellij.sdk.language.psi.SdDocumentStructDefinition; +import org.intellij.sdk.language.psi.SdDocumentStructFieldDefinition; +import org.intellij.sdk.language.psi.SdDocumentSummaryDefinition; +import org.intellij.sdk.language.psi.SdElementFactory; +import org.intellij.sdk.language.psi.SdFile; +import org.intellij.sdk.language.psi.SdFirstPhaseDefinition; +import org.intellij.sdk.language.psi.SdIdentifier; +import org.intellij.sdk.language.psi.SdIdentifierVal; +import org.intellij.sdk.language.psi.SdFunctionDefinition; +import org.intellij.sdk.language.psi.SdIdentifierWithDashVal; +import org.intellij.sdk.language.psi.SdImportFieldDefinition; +import org.intellij.sdk.language.psi.SdItemRawScoreDefinition; +import org.intellij.sdk.language.psi.SdQueryDefinition; +import org.intellij.sdk.language.psi.SdRankProfileDefinition; +import org.intellij.sdk.language.psi.SdSchemaFieldDefinition; +import org.intellij.sdk.language.psi.SdStructFieldDefinition; +import org.intellij.sdk.language.psi.SdSummaryDefinition; +import org.intellij.sdk.language.psi.SdTypes; +import org.jetbrains.annotations.NotNull; +import org.jetbrains.annotations.Nullable; + +import javax.swing.Icon; + +/** + * I this class there are implementations of methods from rules in the .bnf file. While generating the psi files + * (classes and interfaces) the implementations would be taken from here. + */ +public class SdPsiImplUtil { + + @NotNull + public static PsiReference getReference(SdIdentifier element) { + return new SdReference(element, new TextRange(0, element.getName().length())); + } + + public static SdDeclarationType getType(SdDeclaration declaration) { + if (declaration instanceof SdSchemaFieldDefinition) { + return SdDeclarationType.SCHEMA_FIELD; + } else if (declaration instanceof SdDocumentSummaryDefinition) { + return SdDeclarationType.DOCUMENT_SUMMARY; + } else if (declaration instanceof SdImportFieldDefinition) { + return SdDeclarationType.IMPORTED_FIELD; + } else if (declaration instanceof SdRankProfileDefinition) { + return SdDeclarationType.RANK_PROFILE; + } else if (declaration instanceof SdFunctionDefinition) { + return SdDeclarationType.MACRO; + } else if (declaration instanceof SdArgumentDefinition) { + return SdDeclarationType.MACRO_ARGUMENT; + } else if (declaration instanceof SdDocumentDefinition) { + return SdDeclarationType.DOCUMENT; + } else if (declaration instanceof SdDocumentStructDefinition) { + return SdDeclarationType.STRUCT; + } else if (declaration instanceof SdDocumentStructFieldDefinition) { + return SdDeclarationType.DOCUMENT_STRUCT_FIELD; + } else if (declaration instanceof SdDocumentFieldDefinition) { + return SdDeclarationType.DOCUMENT_FIELD; + } else if (declaration instanceof SdStructFieldDefinition) { + return SdDeclarationType.STRUCT_FIELD; + } else if (declaration instanceof SdQueryDefinition) { + return SdDeclarationType.QUERY; + } else if (declaration instanceof SdItemRawScoreDefinition) { + return SdDeclarationType.ITEM_RAW_SCORE; + } else { + return null; + } + } + + public static String getTypeName(SdDeclaration declaration) { + return declaration.getType().toString(); + } + + public static boolean isFunctionName(SdIdentifier identifier, PsiElement file) { + String name = identifier.getName(); + for (SdFunctionDefinition macro : PsiTreeUtil.collectElementsOfType(file, SdFunctionDefinition.class)) { + if (name.equals(macro.getName())) { + return true; + } + } + return false; + } + + public static boolean isOverride(SdFunctionDefinition macroDeclaration) { + String macroName = macroDeclaration.getName(); + + SdRankProfileDefinition curRankProfile = PsiTreeUtil.getParentOfType(macroDeclaration, SdRankProfileDefinition.class); + if (curRankProfile != null) { + curRankProfile = (SdRankProfileDefinition) SdUtil.getRankProfileParent(curRankProfile); + } + while (curRankProfile != null) { + for (SdFunctionDefinition macro : PsiTreeUtil.collectElementsOfType(curRankProfile, SdFunctionDefinition.class)) { + if (macro.getName().equals(macroName)) { + return true; + } + } + curRankProfile = (SdRankProfileDefinition) SdUtil.getRankProfileParent(curRankProfile); + } + return false; + + } + + // ################################### // + // ##### getName implementations ##### // + // ################################### // + + @NotNull + public static String getName(SdIdentifier element) { + if (element != null) { + // IMPORTANT: Convert embedded escaped spaces to simple spaces + return element.getText().replaceAll("\\\\ ", " "); + } else { + return ""; + } + } + + @NotNull + public static String getName(SdDeclaration declaration) { + ASTNode node; + if (declaration instanceof SdImportFieldDefinition) { + ASTNode asNode = declaration.getNode().findChildByType(SdTypes.AS); + node = declaration.getNode().findChildByType(SdTypes.IDENTIFIER_VAL, asNode); + } else if (declaration instanceof SdRankProfileDefinition || declaration instanceof SdDocumentSummaryDefinition + || declaration instanceof SdQueryDefinition) { + node = declaration.getNode().findChildByType(SdTypes.IDENTIFIER_WITH_DASH_VAL); + } else { + node = declaration.getNode().findChildByType(SdTypes.IDENTIFIER_VAL); + } + if (node != null) { + return node.getText(); + } else { + return ""; + } + } + + @NotNull + public static String getName(SdSummaryDefinition summary) { + ASTNode node; + node = summary.getNode().findChildByType(SdTypes.IDENTIFIER_WITH_DASH_VAL); + if (node != null) { + return node.getText(); + } else { + return ""; + } + } + + @NotNull + public static String getName(SdFirstPhaseDefinition firstPhase) { + SdRankProfileDefinition rankProfile = PsiTreeUtil.getParentOfType(firstPhase, SdRankProfileDefinition.class); + if (rankProfile == null) { + return ""; + } + return "first-phase of " + rankProfile.getName(); + } + + // ################################### // + // ##### setName implementations ##### // + // ################################### // + + @NotNull + public static PsiElement setName(SdIdentifierVal element, String newName) { + ASTNode node = element.getNode().getFirstChildNode(); + if (node != null) { + SdIdentifierVal elementName = SdElementFactory.createIdentifierVal(element.getProject(), newName); + ASTNode newNode = elementName.getFirstChild().getNode(); + element.getNode().replaceChild(node, newNode); + } + return element; + } + + @NotNull + public static PsiElement setName(SdIdentifierWithDashVal element, String newName) { + ASTNode node = element.getNode().getFirstChildNode(); + if (node != null) { + SdIdentifierWithDashVal elementName = SdElementFactory.createIdentifierWithDashVal(element.getProject(), newName); + ASTNode newNode = elementName.getFirstChild().getNode(); + element.getNode().replaceChild(node, newNode); + } + return element; + } + + @NotNull + public static PsiElement setName(SdDeclaration element, String newName) { + ASTNode node; + if (element instanceof SdImportFieldDefinition) { + ASTNode asNode = element.getNode().findChildByType(SdTypes.AS); + node = element.getNode().findChildByType(SdTypes.IDENTIFIER_VAL, asNode); + } else { + node = element.getNode().findChildByType(SdTypes.IDENTIFIER_VAL); + } + SdIdentifier elementName = null; + if (node != null) { + elementName = SdElementFactory.createIdentifierVal(element.getProject(), newName); + } else { + node = element.getNode().findChildByType(SdTypes.IDENTIFIER_WITH_DASH_VAL); + } + if (node != null) { + elementName = SdElementFactory.createIdentifierWithDashVal(element.getProject(), newName); + } + if (elementName != null) { + ASTNode newNode = elementName.getFirstChild().getNode(); + element.getNode().replaceChild(node, newNode); + } + return element; + } + + // ##################################### // + // # getNameIdentifier implementations # // + // ##################################### // + + public static PsiElement getNameIdentifier(SdIdentifierVal element) { + ASTNode keyNode = element.getNode().findChildByType(SdTypes.ID); + if (keyNode != null) { + return keyNode.getPsi(); + } else { + return null; + } + } + + public static PsiElement getNameIdentifier(SdDeclaration element) { + ASTNode keyNode = element.getNode().findChildByType(SdTypes.ID); + if (keyNode != null) { + return keyNode.getPsi(); + } else { + return null; + } + } + + // ################################### // + // # getPresentation implementations # // + // ################################### // + + public static ItemPresentation getPresentation(final SdDeclaration element) { + return new ItemPresentation() { + @Nullable + @Override + public String getPresentableText() { + if (element instanceof SdFunctionDefinition) { + return SdUtil.createFunctionDescription((SdFunctionDefinition) element); + } + SdRankProfileDefinition rankProfileParent = PsiTreeUtil.getParentOfType(element, SdRankProfileDefinition.class); + if (rankProfileParent != null) { + if (element instanceof SdQueryDefinition || element instanceof SdItemRawScoreDefinition) { + return element.getName() + " in " + rankProfileParent.getName(); + } + return rankProfileParent.getName() + "." + element.getName(); + } + return element.getName(); + } + + @Nullable + @Override + public String getLocationString() { + return element.getContainingFile() != null ? element.getContainingFile().getName() : null; + } + + @Nullable + @Override + public Icon getIcon(boolean unused) { + if (element instanceof SdFile) { + return SdIcons.FILE; + } else if (element instanceof SdSchemaFieldDefinition || element instanceof SdDocumentFieldDefinition || + element instanceof SdQueryDefinition || element instanceof SdItemRawScoreDefinition) { + return AllIcons.Nodes.Field; + } else if (element instanceof SdStructFieldDefinition || + element instanceof SdDocumentStructFieldDefinition) { + return SdIcons.STRUCT_FIELD; + } else if (element instanceof SdImportFieldDefinition) { + return SdIcons.IMPORTED_FIELD; + } else if (element instanceof SdFunctionDefinition) { + return AllIcons.Nodes.Method; + // Didn't use isOverride() here because it causes the Structure View to load too slow + // return ((SdFunctionDefinition) element).isOverride() ? SdIcons.OVERRIDE_MACRO : AllIcons.Nodes.Method; + } else if (element instanceof SdDocumentStructDefinition) { + return SdIcons.STRUCT; + } else if (element instanceof SdRankProfileDefinition) { + return AllIcons.Nodes.Record; + } else if (element instanceof SdDocumentSummaryDefinition) { + return SdIcons.DOCUMENT_SUMMARY; + } else if (element instanceof SdDocumentDefinition) { + return SdIcons.DOCUMENT; + } + else { + return null; + } + } + }; + } + + public static ItemPresentation getPresentation(final SdSummaryDefinition element) { + return new ItemPresentation() { + + @Override + public String getPresentableText() { + return element.getName(); + } + + @Nullable + @Override + public String getLocationString() { + return element.getContainingFile() != null ? element.getContainingFile().getName() : null; + } + + @Override + public Icon getIcon(boolean unused) { + return SdIcons.SUMMARY; + } + }; + } + + public static ItemPresentation getPresentation(final SdFirstPhaseDefinition element) { + return new ItemPresentation() { + + @Override + public String getPresentableText() { + SdRankProfileDefinition rankProfile = PsiTreeUtil.getParentOfType(element, SdRankProfileDefinition.class); + if (rankProfile == null) { + return ""; + } + return "first-phase of " + rankProfile.getName(); + } + + @Nullable + @Override + public String getLocationString() { + return element.getContainingFile() != null ? element.getContainingFile().getName() : null; + } + + @Override + public Icon getIcon(boolean unused) { + return SdIcons.FIRST_PHASE; + } + }; + } + +} diff --git a/sd-plugin/src/main/java/org/intellij/sdk/language/sd.flex b/sd-plugin/src/main/java/org/intellij/sdk/language/sd.flex new file mode 100644 index 00000000000..c657af4f658 --- /dev/null +++ b/sd-plugin/src/main/java/org/intellij/sdk/language/sd.flex @@ -0,0 +1,202 @@ +package org.intellij.sdk.language.lexer; + +import com.intellij.lexer.FlexLexer; +import com.intellij.psi.tree.IElementType;import com.intellij.ui.components.MultiColumnList;import org.intellij.sdk.language.psi.SdTokenType; + +import static com.intellij.psi.TokenType.BAD_CHARACTER; // Pre-defined bad character token. +import static com.intellij.psi.TokenType.WHITE_SPACE; // Pre-defined whitespace character token. +import static org.intellij.sdk.language.psi.SdTypes.*; // That is the class which is specified as `elementTypeHolderClass` in bnf + // grammar file. This will contain all other tokens which we will use. + +%% + +%public +%class SdLexer +%implements FlexLexer +%function advance +%type IElementType +%unicode + +//**--------- REGEXES ---------**// +// If some character sequence is matched to this regex, it will be treated as an IDENTIFIER. +ID=[a-zA-Z_][a-zA-Z0-9_]* +ID_WITH_DASH = [a-zA-Z_][a-zA-Z0-9_-]* +// If some character sequence is matched to this regex, it will be treated as a WHITE_SPACE. +WHITE_SPACE=[ \t\n\x0B\f\r]+ + +COMMENT=#.* +SYMBOL= [|:{}(),.\[\]] +INTEGER = [0-9]+ +FLOAT = {INTEGER}[.][0-9]+[e]? +COMPARISON_OPERATOR = [<>]|(==)|(<=)|(>=)|(\~=) +ARITHMETIC_OPERATOR = [\-+*/] +STRING = [\"][^\"\n]*[\"] +WORD = \w+ + + +%% + +<YYINITIAL> { + // In here, we match keywords. So if a keyword is found, this returns a token which corresponds to that keyword. + // These tokens are generated using the 'sd.bnf' file and located in the SdTypes class. + // These tokens are Parsed uses these return values to match token squence to a parser rule. + + "search" { return SEARCH; } + "document" { return DOCUMENT; } + "inherits" { return INHERITS; } + "struct" { return STRUCT; } + "field" { return FIELD; } + "type" { return TYPE; } + "struct-field" { return STRUCT_FIELD; } + "match" { return MATCH; } + + "indexing" { return INDEXING; } + "summary" { return SUMMARY; } + "attribute" { return ATTRIBUTE; } + + "array" { return ARRAY; } + "raw" { return RAW; } + "uri" { return URI; } + "reference" { return REFERENCE; } + "annotationreference" { return ANNOTATIONREFERENCE; } + "weightedset" { return WEIGHTEDSET; } + "map" { return MAP; } + + "text" { return TEXT; } + "exact" { return EXACT; } + "exact-terminator" { return EXACT_TERMINATOR; } + "word" { return WORD; } + "prefix" { return PREFIX; } + "cased" { return CASED; } + "uncased" { return UNCASED; } + "substring" { return SUBSTRING; } + "suffix" { return SUFFIX; } + "max-length" { return MAX_LENGTH; } + "gram" { return GRAM; } + "gram-size" { return GRAM_SIZE; } + + "fast-search" { return FAST_SEARCH; } + "fast-access" { return FAST_ACCESS; } + "alias" { return ALIAS; } + "sorting" { return SORTING; } + "uca" { return UCA; } + "lowercase" { return LOWERCASE; } + "paged" { return PAGED; } + "strength" { return STRENGTH; } + "primary" { return PRIMARY; } + "secondary" { return SECONDARY; } + "tertiary" { return TERTIARY; } + "quaternary" { return QUATERNARY; } + "identical" { return IDENTICAL; } + "distance-metric" { return DISTANCE_METRIC; } + + "rank" { return RANK; } + "filter" { return FILTER; } + "normal" { return NORMAL; } + "indexing-rewrite" { return INDEXING_REWRITE; } + "none" { return NONE; } + "query-command" { return QUERY_COMMAND; } + "full" { return FULL; } + "dinamic" { return DYNAMIC; } + "source" { return SOURCE; } + "to" { return TO; } + "matched-elements-only" { return MATCHED_ELEMENTS_ONLY; } + + "input" { return INPUT; } + "mutable" { return MUTABLE; } + "document-summary" { return DOCUMENT_SUMMARY; } + "from-disk" { return FROM_DISK; } + "omit-summary-features" { return OMIT_SUMMARY_FEATURES; } + "import" { return IMPORT; } + "as" { return AS; } + + "rank-profile" { return RANK_PROFILE; } + "match-phase" { return MATCH_PHASE; } + "order" { return ORDER; } + "ascending" { return ASCENDING; } + "descending" { return DESCENDING; } + "max-hits" { return MAX_HITS; } + "diversity" { return DIVERSITY; } + "min-groups" { return MIN_GROUPS; } + "rank-properties" { return RANK_PROPERTIES; } + + "first-phase" { return FIRST_PHASE; } + "keep-rank-count" { return KEEP_RANK_COUNT; } + "rank-score-drop-limit" { return RANK_SCORE_DROP_LIMIT; } + "expression" { return EXPRESSION; } + "file" { return FILE; } + "expression" { return EXPRESSION; } + "num-threads-per-search" { return NUM_THREADS_PER_SEARCH; } + "termwise-limit" { return TERMWISE_LIMIT; } + "ignore-default-rank-features" { return IGNORE_DEFAULT_RANK_FEATURES; } + "min-hits-per-thread" { return MIN_HITS_PER_THREAD; } + "num-search-partition" { return NUM_SEARCH_PARTITION; } + "constants" { return CONSTANTS; } + "literal" { return LITERAL; } + "second-phase" { return SECOND_PHASE; } + "rerank-count" { return RERANK_COUNT; } + + "weight" { return WEIGHT; } + "index" { return INDEX; } + "bolding" { return BOLDING; } + "on" { return ON; } + "off" { return OFF; } + "true" { return TRUE; } + "false" { return FALSE; } + "id" { return ID; } + "normalizing" { return NORMALIZING; } + "stemming" { return STEMMING; } + "arity" { return ARITY; } + "lower-bound" { return LOWER_BOUND; } + "upper-bound" { return UPPER_BOUND; } + "dense-posting-list-threshold" {return DENSE_POSTING_LIST_THRESHOLD; } + "enable-bm25" { return ENABLE_BM25; } + "hnsw" { return HNSW; } + "max-links-per-node" { return MAX_LINKS_PER_NODE; } + "neighbors-to-explore-at-insert" { return NEIGHBORS_TO_EXPLORE_AT_INSERT; } + "multi-threaded-indexing" { return MULTI_THREADED_INDEXING; } + "create-if-nonexistent" { return CREATE_IF_NONEXISTENT; } + "remove-if-zero" { return REMOVE_IF_ZERO; } + "dictionary" { return DICTIONARY; } + "hash" { return HASH; } + "btree" { return BTREE; } + + "fieldset" { return FIELDSET; } + "fields" { return FIELDS; } + "constant" { return CONSTANT; } + "output" { return OUTPUT; } + + "annotation" { return ANNOTATION; } + "rank-type" { return RANK_TYPE; } + "onnx-model" { return ONNX_MODEL; } + "raw-as-base64-in-summary" { return RAW_AS_BASE64_IN_SUMMARY; } + + "function" { return FUNCTION; } + "macro" { return MACRO; } + "inline" { return INLINE; } + + "summary-features" { return SUMMARY_FEATURES; } + + "body" { return BODY; } + "header" { return HEADER; } + + // In here, we check for character sequences which matches regular expressions defined above. + {ID} { return ID_REG; } + {ID_WITH_DASH} { return ID_WITH_DASH_REG; } + + {WHITE_SPACE} { return WHITE_SPACE; } + + {COMMENT} { return COMMENT; } + {SYMBOL} { return SYMBOL; } + {INTEGER} { return INTEGER_REG; } + {FLOAT} { return FLOAT_REG; } + {ARITHMETIC_OPERATOR} { return ARITHMETIC_OPERATOR; } + {COMPARISON_OPERATOR} { return COMPARISON_OPERATOR; } + {WORD} { return WORD_REG; } + {STRING} { return STRING; } + +} + +// If the character sequence does not match any of the above rules, we return BAD_CHARACTER which indicates that +// there is an error in the character sequence. This is used to highlight errors. +[^] { return BAD_CHARACTER; }
\ No newline at end of file diff --git a/sd-plugin/src/main/java/org/intellij/sdk/language/structure/SdStructureViewElement.java b/sd-plugin/src/main/java/org/intellij/sdk/language/structure/SdStructureViewElement.java new file mode 100644 index 00000000000..a5f45e6b26b --- /dev/null +++ b/sd-plugin/src/main/java/org/intellij/sdk/language/structure/SdStructureViewElement.java @@ -0,0 +1,84 @@ +package org.intellij.sdk.language.structure; + +import com.intellij.ide.projectView.PresentationData; +import com.intellij.ide.structureView.StructureViewTreeElement; +import com.intellij.ide.util.treeView.smartTree.SortableTreeElement; +import com.intellij.ide.util.treeView.smartTree.TreeElement; +import com.intellij.navigation.ItemPresentation; +import com.intellij.psi.NavigatablePsiElement; +import com.intellij.psi.PsiElement; +import org.intellij.sdk.language.SdUtil; +import org.intellij.sdk.language.psi.SdDocumentDefinition; +import org.intellij.sdk.language.psi.SdDocumentStructDefinition; +import org.intellij.sdk.language.psi.SdDocumentSummaryDefinition; +import org.intellij.sdk.language.psi.SdFile; +import org.intellij.sdk.language.psi.SdRankProfileDefinition; +import org.jetbrains.annotations.NotNull; + +import java.util.ArrayList; +import java.util.List; + +public class SdStructureViewElement implements StructureViewTreeElement, SortableTreeElement { + + private final NavigatablePsiElement myElement; + + public SdStructureViewElement(NavigatablePsiElement element) { + this.myElement = element; + } + + @Override + public Object getValue() { + return myElement; + } + + @Override + public void navigate(boolean requestFocus) { + myElement.navigate(requestFocus); + } + + @Override + public boolean canNavigate() { + return myElement.canNavigate(); + } + + @Override + public boolean canNavigateToSource() { + return myElement.canNavigateToSource(); + } + + @NotNull + @Override + public String getAlphaSortKey() { + String name = myElement.getName(); + return name != null ? name : ""; + } + + @NotNull + @Override + public ItemPresentation getPresentation() { + ItemPresentation presentation = myElement.getPresentation(); + return presentation != null ? presentation : new PresentationData(); + } + + @Override + public TreeElement @NotNull [] getChildren() { + List<PsiElement> children = new ArrayList<>();; + if (myElement instanceof SdFile) { + children = SdUtil.findSchemaChildren(myElement); + } else if (myElement instanceof SdDocumentDefinition) { + children = SdUtil.findDocumentChildren(myElement); + } else if (myElement instanceof SdDocumentStructDefinition) { + children = SdUtil.findDocumentStructChildren(myElement); + } else if (myElement instanceof SdRankProfileDefinition) { + children = SdUtil.findRankProfileChildren(myElement); + } else if (myElement instanceof SdDocumentSummaryDefinition) { + children = SdUtil.findDocumentSummaryChildren(myElement); + } + + List<TreeElement> treeElements = new ArrayList<>(children.size()); + for (PsiElement child : children) { + treeElements.add(new SdStructureViewElement((NavigatablePsiElement) child)); + } + return treeElements.toArray(new TreeElement[0]); + } +} diff --git a/sd-plugin/src/main/java/org/intellij/sdk/language/structure/SdStructureViewFactory.java b/sd-plugin/src/main/java/org/intellij/sdk/language/structure/SdStructureViewFactory.java new file mode 100644 index 00000000000..05590362fcf --- /dev/null +++ b/sd-plugin/src/main/java/org/intellij/sdk/language/structure/SdStructureViewFactory.java @@ -0,0 +1,26 @@ +package org.intellij.sdk.language.structure; + +import com.intellij.ide.structureView.StructureViewBuilder; +import com.intellij.ide.structureView.StructureViewModel; +import com.intellij.ide.structureView.TreeBasedStructureViewBuilder; +import com.intellij.lang.PsiStructureViewFactory; +import com.intellij.openapi.editor.Editor; +import com.intellij.psi.PsiFile; +import org.jetbrains.annotations.NotNull; +import org.jetbrains.annotations.Nullable; + +public class SdStructureViewFactory implements PsiStructureViewFactory { + + @Nullable + @Override + public StructureViewBuilder getStructureViewBuilder(@NotNull final PsiFile psiFile) { + return new TreeBasedStructureViewBuilder() { + @NotNull + @Override + public StructureViewModel createStructureViewModel(@Nullable Editor editor) { + return new SdStructureViewModel(psiFile); + } + }; + } + +} diff --git a/sd-plugin/src/main/java/org/intellij/sdk/language/structure/SdStructureViewModel.java b/sd-plugin/src/main/java/org/intellij/sdk/language/structure/SdStructureViewModel.java new file mode 100644 index 00000000000..549bd811dcf --- /dev/null +++ b/sd-plugin/src/main/java/org/intellij/sdk/language/structure/SdStructureViewModel.java @@ -0,0 +1,31 @@ +package org.intellij.sdk.language.structure; + +import com.intellij.ide.structureView.StructureViewModel; +import com.intellij.ide.structureView.StructureViewModelBase; +import com.intellij.ide.structureView.StructureViewTreeElement; +import com.intellij.ide.util.treeView.smartTree.Sorter; +import com.intellij.psi.PsiFile; +import org.jetbrains.annotations.NotNull; + +public class SdStructureViewModel extends StructureViewModelBase implements StructureViewModel.ElementInfoProvider { + public SdStructureViewModel(PsiFile psiFile) { + super(psiFile, new SdStructureViewElement(psiFile)); + } + + + public Sorter @NotNull [] getSorters() { + return new Sorter[]{Sorter.ALPHA_SORTER}; + } + + + @Override + public boolean isAlwaysShowsPlus(StructureViewTreeElement element) { + return false; + } + + @Override + public boolean isAlwaysLeaf(StructureViewTreeElement element) { + return false; + } + +} diff --git a/sd-plugin/src/main/resources/META-INF/plugin.xml b/sd-plugin/src/main/resources/META-INF/plugin.xml new file mode 100644 index 00000000000..206b33144e2 --- /dev/null +++ b/sd-plugin/src/main/resources/META-INF/plugin.xml @@ -0,0 +1,48 @@ +<idea-plugin> + <id>org.intellij.sdk.language</id> + <name>SdPlugin</name> + + <!-- Text to display as company information on Preferences/Settings | Plugin page --> + <vendor email="shahar.ariel@verizonmedia.com">VerizonMedia</vendor> + + <!-- Product and plugin compatibility requirements --> + <depends>com.intellij.java</depends> + + <!-- Text to display as description on Preferences/Settings | Plugin page --> + <description><![CDATA[ + A plugin to recognize sd files and work with them through IntelliJ<br> + <em></em> + ]]></description> + + <!-- please see https://plugins.jetbrains.com/docs/intellij/plugin-compatibility.html + on how to target different products --> + <depends>com.intellij.modules.platform</depends> + <depends>com.intellij.java</depends> + + <extensions defaultExtensionNs="com.intellij"> + <fileType name="Sd File" implementationClass="org.intellij.sdk.language.SdFileType" fieldName="INSTANCE" + language="Sd" extensions="sd"/> + <lang.parserDefinition language="Sd" implementationClass="org.intellij.sdk.language.parser.SdParserDefinition"/> + <lang.syntaxHighlighterFactory language="Sd" implementationClass="org.intellij.sdk.language.SdSyntaxHighlighterFactory"/> + <completion.contributor language="Sd" implementationClass="org.intellij.sdk.language.SdCompletionContributor"/> + + <lang.findUsagesProvider language="Sd" implementationClass="org.intellij.sdk.language.findUsages.SdFindUsagesProvider"/> + <findUsagesHandlerFactory implementation="org.intellij.sdk.language.findUsages.SdFindUsagesHandlerFactory"/> + <fileStructureGroupRuleProvider implementation="org.intellij.sdk.language.findUsages.SdRankProfileGroupingRuleProvider"/> + <fileStructureGroupRuleProvider implementation="org.intellij.sdk.language.findUsages.SdDocumentSummaryGroupingRuleProvider"/> + <elementDescriptionProvider implementation="org.intellij.sdk.language.psi.SdElementDescriptionProvider"/> + + <lang.psiStructureViewFactory language="Sd" implementationClass="org.intellij.sdk.language.structure.SdStructureViewFactory"/> + <lang.formatter language="Sd" implementationClass="org.intellij.sdk.language.SdFormattingModelBuilder"/> + <codeStyleSettingsProvider implementation="org.intellij.sdk.language.SdCodeStyleSettingsProvider"/> + <langCodeStyleSettingsProvider implementation="org.intellij.sdk.language.SdLanguageCodeStyleSettingsProvider"/> + <lang.commenter language="Sd" implementationClass="org.intellij.sdk.language.SdCommenter"/> + <lang.refactoringSupport language="Sd" implementationClass="org.intellij.sdk.language.SdRefactoringSupportProvider"/> + <gotoSymbolContributor implementation="org.intellij.sdk.language.SdChooseByNameContributor"/> + <callHierarchyProvider language="Sd" implementationClass="org.intellij.sdk.language.hierarchy.SdCallHierarchyProvider"/> + </extensions> + + <actions> + <!-- Add your actions here --> + </actions> +</idea-plugin>
\ No newline at end of file diff --git a/sd-plugin/src/main/resources/META-INF/pluginIcon.svg b/sd-plugin/src/main/resources/META-INF/pluginIcon.svg new file mode 100644 index 00000000000..9babe4c4ef4 --- /dev/null +++ b/sd-plugin/src/main/resources/META-INF/pluginIcon.svg @@ -0,0 +1,19 @@ +<svg version="1.2" baseProfile="tiny-ps" xmlns="http://www.w3.org/2000/svg" viewBox="0 0 40 40" width="40" height="40"> + <title>pluginIcon</title> + <defs> + <image width="40" height="40" id="img1" href="data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAACgAAAAoAQMAAAC2MCouAAAAAXNSR0IB2cksfwAAAANQTFRFAAAAp3o92gAAAAF0Uk5TAEDm2GYAAAAMSURBVHicY2AYWQAAAPAAAQaUif0AAAAASUVORK5CYII="/> + <image width="36" height="36" id="img2" href="data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAACQAAAAkAQMAAADbzgrbAAAAAXNSR0IB2cksfwAAAANQTFRFuubnWfllrQAAAAxJREFUeJxjYBieAAAA2AABOXA/oAAAAABJRU5ErkJggg=="/> + <image width="34" height="34" id="img3" href="data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAACIAAAAiAQMAAAAAiZmBAAAAAXNSR0IB2cksfwAAAAZQTFRFAAAAP5aZJJgdHgAAAAJ0Uk5TAP9bkSK1AAAAWUlEQVR4nI3Ouw2AMAxF0YcoXGYERsloyWiMwhgUiIsTKxESDS5O49+T7JRXpgm7tMAhreANg0tK3Q3uNghhVfno+In/lum4kKfjY3plsJ4qEkZaX6uuefMBOOp0ufGf8MYAAAAASUVORK5CYII="/> + </defs> + <style> + tspan { white-space:pre } + .txt0 { font-size: 20px;fill: #efab00;font-weight: 400;font-family: "DejaVuSans", "DejaVu Sans" } + </style> + <use id="Background" href="#img1" x="0" y="0" /> + <use id="Layer 1" style="opacity: 0.42" href="#img2" x="2" y="2" /> + <use id="Layer 2" style="opacity: 0.769" href="#img3" x="3" y="3" /> + <text id="SD" style="transform: matrix(1,0,0,1,7,27);paint-order:stroke fill markers;stroke: #826c34;stroke-width: 2;stroke-linejoin: round;" > + <tspan x="0" y="0" class="txt0">S</tspan><tspan y="0" class="txt0">D +</tspan> + </text> +</svg>
\ No newline at end of file diff --git a/sd-plugin/src/main/resources/META-INF/pluginIcon_dark.svg b/sd-plugin/src/main/resources/META-INF/pluginIcon_dark.svg new file mode 100644 index 00000000000..9babe4c4ef4 --- /dev/null +++ b/sd-plugin/src/main/resources/META-INF/pluginIcon_dark.svg @@ -0,0 +1,19 @@ +<svg version="1.2" baseProfile="tiny-ps" xmlns="http://www.w3.org/2000/svg" viewBox="0 0 40 40" width="40" height="40"> + <title>pluginIcon</title> + <defs> + <image width="40" height="40" id="img1" href="data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAACgAAAAoAQMAAAC2MCouAAAAAXNSR0IB2cksfwAAAANQTFRFAAAAp3o92gAAAAF0Uk5TAEDm2GYAAAAMSURBVHicY2AYWQAAAPAAAQaUif0AAAAASUVORK5CYII="/> + <image width="36" height="36" id="img2" href="data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAACQAAAAkAQMAAADbzgrbAAAAAXNSR0IB2cksfwAAAANQTFRFuubnWfllrQAAAAxJREFUeJxjYBieAAAA2AABOXA/oAAAAABJRU5ErkJggg=="/> + <image width="34" height="34" id="img3" href="data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAACIAAAAiAQMAAAAAiZmBAAAAAXNSR0IB2cksfwAAAAZQTFRFAAAAP5aZJJgdHgAAAAJ0Uk5TAP9bkSK1AAAAWUlEQVR4nI3Ouw2AMAxF0YcoXGYERsloyWiMwhgUiIsTKxESDS5O49+T7JRXpgm7tMAhreANg0tK3Q3uNghhVfno+In/lum4kKfjY3plsJ4qEkZaX6uuefMBOOp0ufGf8MYAAAAASUVORK5CYII="/> + </defs> + <style> + tspan { white-space:pre } + .txt0 { font-size: 20px;fill: #efab00;font-weight: 400;font-family: "DejaVuSans", "DejaVu Sans" } + </style> + <use id="Background" href="#img1" x="0" y="0" /> + <use id="Layer 1" style="opacity: 0.42" href="#img2" x="2" y="2" /> + <use id="Layer 2" style="opacity: 0.769" href="#img3" x="3" y="3" /> + <text id="SD" style="transform: matrix(1,0,0,1,7,27);paint-order:stroke fill markers;stroke: #826c34;stroke-width: 2;stroke-linejoin: round;" > + <tspan x="0" y="0" class="txt0">S</tspan><tspan y="0" class="txt0">D +</tspan> + </text> +</svg>
\ No newline at end of file diff --git a/sd-plugin/src/main/resources/flex/idea-flex.skeleton b/sd-plugin/src/main/resources/flex/idea-flex.skeleton new file mode 100644 index 00000000000..218399976ca --- /dev/null +++ b/sd-plugin/src/main/resources/flex/idea-flex.skeleton @@ -0,0 +1,248 @@ + + /** This character denotes the end of file */ + public static final int YYEOF = -1; + + /** initial size of the lookahead buffer */ +--- private static final int ZZ_BUFFERSIZE = ...; + + /** lexical states */ +--- lexical states, charmap + + /* error codes */ + private static final int ZZ_UNKNOWN_ERROR = 0; + private static final int ZZ_NO_MATCH = 1; + private static final int ZZ_PUSHBACK_2BIG = 2; + + /* error messages for the codes above */ + private static final String[] ZZ_ERROR_MSG = { + "Unknown internal scanner error", + "Error: could not match input", + "Error: pushback value was too large" + }; + +--- isFinal list + /** the input device */ + private java.io.Reader zzReader; + + /** the current state of the DFA */ + private int zzState; + + /** the current lexical state */ + private int zzLexicalState = YYINITIAL; + + /** this buffer contains the current text to be matched and is + the source of the yytext() string */ + private CharSequence zzBuffer = ""; + + /** the textposition at the last accepting state */ + private int zzMarkedPos; + + /** the current text position in the buffer */ + private int zzCurrentPos; + + /** startRead marks the beginning of the yytext() string in the buffer */ + private int zzStartRead; + + /** endRead marks the last character in the buffer, that has been read + from input */ + private int zzEndRead; + + /** + * zzAtBOL == true <=> the scanner is currently at the beginning of a line + */ + private boolean zzAtBOL = true; + + /** zzAtEOF == true <=> the scanner is at the EOF */ + private boolean zzAtEOF; + + /** denotes if the user-EOF-code has already been executed */ + private boolean zzEOFDone; + +--- user class code + +--- constructor declaration + + public final int getTokenStart() { + return zzStartRead; + } + + public final int getTokenEnd() { + return getTokenStart() + yylength(); + } + + public void reset(CharSequence buffer, int start, int end, int initialState) { + zzBuffer = buffer; + zzCurrentPos = zzMarkedPos = zzStartRead = start; + zzAtEOF = false; + zzAtBOL = true; + zzEndRead = end; + yybegin(initialState); + } + + /** + * Refills the input buffer. + * + * @return {@code false}, iff there was new input. + * + * @exception java.io.IOException if any I/O-Error occurs + */ + private boolean zzRefill() throws java.io.IOException { + return true; + } + + + /** + * Returns the current lexical state. + */ + public final int yystate() { + return zzLexicalState; + } + + + /** + * Enters a new lexical state + * + * @param newState the new lexical state + */ + public final void yybegin(int newState) { + zzLexicalState = newState; + } + + + /** + * Returns the text matched by the current regular expression. + */ + public final CharSequence yytext() { + return zzBuffer.subSequence(zzStartRead, zzMarkedPos); + } + + + /** + * Returns the character at position {@code pos} from the + * matched text. + * + * It is equivalent to yytext().charAt(pos), but faster + * + * @param pos the position of the character to fetch. + * A value from 0 to yylength()-1. + * + * @return the character at position pos + */ + public final char yycharat(int pos) { + return zzBuffer.charAt(zzStartRead+pos); + } + + + /** + * Returns the length of the matched text region. + */ + public final int yylength() { + return zzMarkedPos-zzStartRead; + } + + + /** + * Reports an error that occurred while scanning. + * + * In a wellformed scanner (no or only correct usage of + * yypushback(int) and a match-all fallback rule) this method + * will only be called with things that "Can't Possibly Happen". + * If this method is called, something is seriously wrong + * (e.g. a JFlex bug producing a faulty scanner etc.). + * + * Usual syntax/scanner level error handling should be done + * in error fallback rules. + * + * @param errorCode the code of the errormessage to display + */ +--- zzScanError declaration + String message; + try { + message = ZZ_ERROR_MSG[errorCode]; + } + catch (ArrayIndexOutOfBoundsException e) { + message = ZZ_ERROR_MSG[ZZ_UNKNOWN_ERROR]; + } + +--- throws clause + } + + + /** + * Pushes the specified amount of characters back into the input stream. + * + * They will be read again by then next call of the scanning method + * + * @param number the number of characters to be read again. + * This number must not be greater than yylength()! + */ +--- yypushback decl (contains zzScanError exception) + if ( number > yylength() ) + zzScanError(ZZ_PUSHBACK_2BIG); + + zzMarkedPos -= number; + } + + +--- zzDoEOF + /** + * Resumes scanning until the next regular expression is matched, + * the end of input is encountered or an I/O-Error occurs. + * + * @return the next token + * @exception java.io.IOException if any I/O-Error occurs + */ +--- yylex declaration + int zzInput; + int zzAction; + + // cached fields: + int zzCurrentPosL; + int zzMarkedPosL; + int zzEndReadL = zzEndRead; + CharSequence zzBufferL = zzBuffer; + +--- local declarations + + while (true) { + zzMarkedPosL = zzMarkedPos; + +--- start admin (line, char, col count) + zzAction = -1; + + zzCurrentPosL = zzCurrentPos = zzStartRead = zzMarkedPosL; + +--- start admin (lexstate etc) + + zzForAction: { + while (true) { + +--- next input, line, col, char count, next transition, isFinal action + zzAction = zzState; + zzMarkedPosL = zzCurrentPosL; +--- line count update + } + + } + } + + // store back cached position + zzMarkedPos = zzMarkedPosL; +--- char count update + + if (zzInput == YYEOF && zzStartRead == zzCurrentPos) { + zzAtEOF = true; +--- eofvalue + } + else { +--- actions + default: +--- no match + } + } + } + } + +--- main + +} diff --git a/sd-plugin/src/main/resources/flex/jflex-1.7.0-2.jar b/sd-plugin/src/main/resources/flex/jflex-1.7.0-2.jar Binary files differnew file mode 100644 index 00000000000..c152c2c3384 --- /dev/null +++ b/sd-plugin/src/main/resources/flex/jflex-1.7.0-2.jar diff --git a/sd-plugin/src/main/resources/icons/document_icon.png b/sd-plugin/src/main/resources/icons/document_icon.png Binary files differnew file mode 100644 index 00000000000..497c218ab91 --- /dev/null +++ b/sd-plugin/src/main/resources/icons/document_icon.png diff --git a/sd-plugin/src/main/resources/icons/document_summary_icon.png b/sd-plugin/src/main/resources/icons/document_summary_icon.png Binary files differnew file mode 100644 index 00000000000..72f42c68902 --- /dev/null +++ b/sd-plugin/src/main/resources/icons/document_summary_icon.png diff --git a/sd-plugin/src/main/resources/icons/first_phase_icon.png b/sd-plugin/src/main/resources/icons/first_phase_icon.png Binary files differnew file mode 100644 index 00000000000..18f24e9e462 --- /dev/null +++ b/sd-plugin/src/main/resources/icons/first_phase_icon.png diff --git a/sd-plugin/src/main/resources/icons/imported_field_icon.png b/sd-plugin/src/main/resources/icons/imported_field_icon.png Binary files differnew file mode 100644 index 00000000000..1ec9b1d6a96 --- /dev/null +++ b/sd-plugin/src/main/resources/icons/imported_field_icon.png diff --git a/sd-plugin/src/main/resources/icons/macro_icon.png b/sd-plugin/src/main/resources/icons/macro_icon.png Binary files differnew file mode 100644 index 00000000000..736d8168119 --- /dev/null +++ b/sd-plugin/src/main/resources/icons/macro_icon.png diff --git a/sd-plugin/src/main/resources/icons/override_macro_icon.png b/sd-plugin/src/main/resources/icons/override_macro_icon.png Binary files differnew file mode 100644 index 00000000000..ae9e2636798 --- /dev/null +++ b/sd-plugin/src/main/resources/icons/override_macro_icon.png diff --git a/sd-plugin/src/main/resources/icons/sd_icon.png b/sd-plugin/src/main/resources/icons/sd_icon.png Binary files differnew file mode 100644 index 00000000000..1c96aaee633 --- /dev/null +++ b/sd-plugin/src/main/resources/icons/sd_icon.png diff --git a/sd-plugin/src/main/resources/icons/struct_field_icon.png b/sd-plugin/src/main/resources/icons/struct_field_icon.png Binary files differnew file mode 100644 index 00000000000..7f5074630d4 --- /dev/null +++ b/sd-plugin/src/main/resources/icons/struct_field_icon.png diff --git a/sd-plugin/src/main/resources/icons/struct_icon.png b/sd-plugin/src/main/resources/icons/struct_icon.png Binary files differnew file mode 100644 index 00000000000..c2f6fdc8440 --- /dev/null +++ b/sd-plugin/src/main/resources/icons/struct_icon.png diff --git a/sd-plugin/src/main/resources/icons/summary_def_icon.png b/sd-plugin/src/main/resources/icons/summary_def_icon.png Binary files differnew file mode 100644 index 00000000000..a8faf12601f --- /dev/null +++ b/sd-plugin/src/main/resources/icons/summary_def_icon.png diff --git a/searchcore/src/tests/proton/documentdb/buckethandler/buckethandler_test.cpp b/searchcore/src/tests/proton/documentdb/buckethandler/buckethandler_test.cpp index 487c8741a65..29748a2010c 100644 --- a/searchcore/src/tests/proton/documentdb/buckethandler/buckethandler_test.cpp +++ b/searchcore/src/tests/proton/documentdb/buckethandler/buckethandler_test.cpp @@ -96,7 +96,7 @@ struct Fixture BucketStateCalculator::SP _calc; test::BucketIdListResultHandler _bucketList; test::BucketInfoResultHandler _bucketInfo; - test::GenericResultHandler _genResult; + std::shared_ptr<test::GenericResultHandler> _genResult; Fixture() : _builder(), _bucketDB(std::make_shared<bucketdb::BucketDBOwner>()), @@ -107,7 +107,8 @@ struct Fixture _handler(_exec), _changedHandler(), _calc(new BucketStateCalculator()), - _bucketList(), _bucketInfo(), _genResult() + _bucketList(), _bucketInfo(), + _genResult(std::make_shared<test::GenericResultHandler>()) { // bucket 2 & 3 & 4 & 7 in ready _ready.insertDocs(_builder.createDocs(2, 1, 3). // 2 docs diff --git a/searchcore/src/tests/proton/documentdb/documentbucketmover/documentbucketmover_test.cpp b/searchcore/src/tests/proton/documentdb/documentbucketmover/documentbucketmover_test.cpp index e2f2f21f596..2c21a30396d 100644 --- a/searchcore/src/tests/proton/documentdb/documentbucketmover/documentbucketmover_test.cpp +++ b/searchcore/src/tests/proton/documentdb/documentbucketmover/documentbucketmover_test.cpp @@ -23,6 +23,8 @@ using storage::spi::BucketInfo; using BlockedReason = IBlockableMaintenanceJob::BlockedReason; using MoveOperationVector = std::vector<MoveOperation>; using storage::spi::dummy::DummyBucketExecutor; +using vespalib::MonitoredRefCount; +using vespalib::RetainGuard; using vespalib::ThreadStackExecutor; struct ControllerFixtureBase : public ::testing::Test diff --git a/searchcore/src/tests/proton/documentdb/lid_space_compaction/lid_space_jobtest.cpp b/searchcore/src/tests/proton/documentdb/lid_space_compaction/lid_space_jobtest.cpp index c9b009e4a17..13955953eb5 100644 --- a/searchcore/src/tests/proton/documentdb/lid_space_compaction/lid_space_jobtest.cpp +++ b/searchcore/src/tests/proton/documentdb/lid_space_compaction/lid_space_jobtest.cpp @@ -6,6 +6,8 @@ #include <vespa/persistence/dummyimpl/dummy_bucket_executor.h> #include <vespa/vespalib/util/threadstackexecutor.h> +using vespalib::RetainGuard; + using BlockedReason = IBlockableMaintenanceJob::BlockedReason; struct MyDirectJobRunner : public IMaintenanceJobRunner { diff --git a/searchcore/src/tests/proton/documentdb/lid_space_compaction/lid_space_jobtest.h b/searchcore/src/tests/proton/documentdb/lid_space_compaction/lid_space_jobtest.h index 774acd0071a..14f2ff42dbe 100644 --- a/searchcore/src/tests/proton/documentdb/lid_space_compaction/lid_space_jobtest.h +++ b/searchcore/src/tests/proton/documentdb/lid_space_compaction/lid_space_jobtest.h @@ -2,14 +2,14 @@ #include "lid_space_common.h" #include <vespa/searchcore/proton/server/blockable_maintenance_job.h> -#include <vespa/searchcore/proton/common/monitored_refcount.h> #include <vespa/persistence/spi/bucketexecutor.h> #include <vespa/searchcorespi/index/i_thread_service.h> +#include <vespa/vespalib/util/monitored_refcount.h> #include <vespa/vespalib/gtest/gtest.h> namespace storage::spi::dummy { class DummyBucketExecutor; } struct JobTestBase : public ::testing::Test { - MonitoredRefCount _refCount; + vespalib::MonitoredRefCount _refCount; test::ClusterStateHandler _clusterStateHandler; test::DiskMemUsageNotifier _diskMemUsageNotifier; std::unique_ptr<storage::spi::dummy::DummyBucketExecutor> _bucketExecutor; diff --git a/searchcore/src/tests/proton/documentdb/maintenancecontroller/maintenancecontroller_test.cpp b/searchcore/src/tests/proton/documentdb/maintenancecontroller/maintenancecontroller_test.cpp index 1463be8cdbd..09c782cef07 100644 --- a/searchcore/src/tests/proton/documentdb/maintenancecontroller/maintenancecontroller_test.cpp +++ b/searchcore/src/tests/proton/documentdb/maintenancecontroller/maintenancecontroller_test.cpp @@ -6,7 +6,6 @@ #include <vespa/searchcore/proton/attribute/i_attribute_manager.h> #include <vespa/searchcore/proton/bucketdb/bucket_create_notifier.h> #include <vespa/searchcore/proton/common/doctypename.h> -#include <vespa/searchcore/proton/common/monitored_refcount.h> #include <vespa/searchcore/proton/common/transient_resource_usage_provider.h> #include <vespa/searchcore/proton/documentmetastore/operation_listener.h> #include <vespa/searchcore/proton/documentmetastore/documentmetastore.h> @@ -16,7 +15,6 @@ #include <vespa/searchcore/proton/feedoperation/removeoperation.h> #include <vespa/searchcore/proton/server/blockable_maintenance_job.h> #include <vespa/searchcore/proton/server/executor_thread_service.h> -#include <vespa/searchcore/proton/server/i_document_scan_iterator.h> #include <vespa/searchcore/proton/server/i_operation_storer.h> #include <vespa/searchcore/proton/server/ibucketmodifiedhandler.h> #include <vespa/searchcore/proton/server/idocumentmovehandler.h> @@ -42,6 +40,7 @@ #include <vespa/vespalib/util/destructor_callbacks.h> #include <vespa/vespalib/util/gate.h> #include <vespa/vespalib/util/lambdatask.h> +#include <vespa/vespalib/util/monitored_refcount.h> #include <vespa/vespalib/util/size_literals.h> #include <vespa/vespalib/util/threadstackexecutor.h> #include <vespa/fastos/thread.h> @@ -68,6 +67,7 @@ using search::SerialNum; using search::CommitParam; using storage::spi::BucketInfo; using storage::spi::Timestamp; +using vespalib::MonitoredRefCount; using vespalib::Slime; using vespalib::makeLambdaTask; using vespa::config::search::AttributesConfigBuilder; @@ -727,7 +727,7 @@ MyExecutor::isIdle() { (void) getStats(); sync(); - Stats stats(getStats()); + auto stats = getStats(); return stats.acceptedTasks == 0u; } diff --git a/searchcore/src/tests/proton/persistenceengine/persistence_handler_map/persistence_handler_map_test.cpp b/searchcore/src/tests/proton/persistenceengine/persistence_handler_map/persistence_handler_map_test.cpp index 2bb1eb44e25..e8cc1b54235 100644 --- a/searchcore/src/tests/proton/persistenceengine/persistence_handler_map/persistence_handler_map_test.cpp +++ b/searchcore/src/tests/proton/persistenceengine/persistence_handler_map/persistence_handler_map_test.cpp @@ -20,7 +20,7 @@ struct DummyPersistenceHandler : public IPersistenceHandler { void handleRemove(FeedToken, const storage::spi::Bucket &, storage::spi::Timestamp, const document::DocumentId &) override {} void handleListBuckets(IBucketIdListResultHandler &) override {} void handleSetClusterState(const storage::spi::ClusterState &, IGenericResultHandler &) override {} - void handleSetActiveState(const storage::spi::Bucket &, storage::spi::BucketInfo::ActiveState, IGenericResultHandler &) override {} + void handleSetActiveState(const storage::spi::Bucket &, storage::spi::BucketInfo::ActiveState, std::shared_ptr<IGenericResultHandler>) override {} void handleGetBucketInfo(const storage::spi::Bucket &, IBucketInfoResultHandler &) override {} void handleCreateBucket(FeedToken, const storage::spi::Bucket &) override {} void handleDeleteBucket(FeedToken, const storage::spi::Bucket &) override {} @@ -44,8 +44,6 @@ DummyPersistenceHandler::SP handler_b(std::make_shared<DummyPersistenceHandler>( DummyPersistenceHandler::SP handler_c(std::make_shared<DummyPersistenceHandler>()); DummyPersistenceHandler::SP handler_a_new(std::make_shared<DummyPersistenceHandler>()); - - void assertHandler(const IPersistenceHandler::SP & lhs, const IPersistenceHandler * rhs) { diff --git a/searchcore/src/tests/proton/persistenceengine/persistenceengine_test.cpp b/searchcore/src/tests/proton/persistenceengine/persistenceengine_test.cpp index 9613c505f77..c252c89a2f8 100644 --- a/searchcore/src/tests/proton/persistenceengine/persistenceengine_test.cpp +++ b/searchcore/src/tests/proton/persistenceengine/persistenceengine_test.cpp @@ -227,10 +227,10 @@ struct MyHandler : public IPersistenceHandler, IBucketFreezer { } void handleSetActiveState(const Bucket &bucket, storage::spi::BucketInfo::ActiveState newState, - IGenericResultHandler &resultHandler) override { + std::shared_ptr<IGenericResultHandler> resultHandler) override { lastBucket = bucket; lastBucketState = newState; - resultHandler.handle(bucketStateResult); + resultHandler->handle(bucketStateResult); } void handleGetBucketInfo(const Bucket &, IBucketInfoResultHandler &resultHandler) override { diff --git a/searchcore/src/tests/proton/reference/document_db_reference_resolver/document_db_reference_resolver_test.cpp b/searchcore/src/tests/proton/reference/document_db_reference_resolver/document_db_reference_resolver_test.cpp index a76c375bbfc..061a3fd4c32 100644 --- a/searchcore/src/tests/proton/reference/document_db_reference_resolver/document_db_reference_resolver_test.cpp +++ b/searchcore/src/tests/proton/reference/document_db_reference_resolver/document_db_reference_resolver_test.cpp @@ -6,7 +6,6 @@ #include <vespa/document/datatype/referencedatatype.h> #include <vespa/log/log.h> #include <vespa/searchcore/proton/attribute/imported_attributes_repo.h> -#include <vespa/searchcore/proton/common/monitored_refcount.h> #include <vespa/searchcore/proton/reference/document_db_reference_resolver.h> #include <vespa/searchcore/proton/reference/gid_to_lid_change_listener.h> #include <vespa/searchcore/proton/reference/i_document_db_reference.h> @@ -19,6 +18,7 @@ #include <vespa/searchlib/attribute/reference_attribute.h> #include <vespa/searchlib/common/i_gid_to_lid_mapper.h> #include <vespa/searchlib/common/i_gid_to_lid_mapper_factory.h> +#include <vespa/vespalib/util/monitored_refcount.h> #include <vespa/vespalib/util/sequencedtaskexecutor.h> #include <vespa/searchlib/test/mock_attribute_manager.h> #include <vespa/vespalib/test/insertion_operators.h> @@ -33,6 +33,7 @@ using proton::test::MockDocumentDBReference; using search::attribute::test::MockAttributeManager; using vespa::config::search::ImportedFieldsConfig; using vespa::config::search::ImportedFieldsConfigBuilder; +using vespalib::MonitoredRefCount; using vespalib::SequencedTaskExecutor; using vespalib::ISequencedTaskExecutor; diff --git a/searchcore/src/tests/proton/reference/gid_to_lid_change_listener/gid_to_lid_change_listener_test.cpp b/searchcore/src/tests/proton/reference/gid_to_lid_change_listener/gid_to_lid_change_listener_test.cpp index 668be4c2c2d..15281563b93 100644 --- a/searchcore/src/tests/proton/reference/gid_to_lid_change_listener/gid_to_lid_change_listener_test.cpp +++ b/searchcore/src/tests/proton/reference/gid_to_lid_change_listener/gid_to_lid_change_listener_test.cpp @@ -2,12 +2,12 @@ #include <vespa/vespalib/testkit/testapp.h> #include <vespa/vespalib/stllike/string.h> #include <vespa/document/base/documentid.h> -#include <vespa/vespalib/util/sequencedtaskexecutor.h> -#include <vespa/searchcore/proton/common/monitored_refcount.h> #include <vespa/searchcore/proton/reference/gid_to_lid_change_listener.h> #include <vespa/searchlib/common/i_gid_to_lid_mapper_factory.h> #include <vespa/vespalib/util/destructor_callbacks.h> #include <vespa/vespalib/util/gate.h> +#include <vespa/vespalib/util/monitored_refcount.h> +#include <vespa/vespalib/util/sequencedtaskexecutor.h> #include <vespa/searchlib/test/mock_gid_to_lid_mapping.h> #include <map> #include <vespa/log/log.h> @@ -16,12 +16,13 @@ LOG_SETUP("gid_to_lid_change_listener_test"); using document::GlobalId; using document::BucketId; using document::DocumentId; -using vespalib::GenerationHandler; using search::attribute::Config; using search::attribute::BasicType; using search::attribute::Reference; using search::attribute::ReferenceAttribute; using search::attribute::test::MockGidToLidMapperFactory; +using vespalib::MonitoredRefCount; +using vespalib::GenerationHandler; namespace proton { diff --git a/searchcore/src/vespa/searchcore/proton/common/CMakeLists.txt b/searchcore/src/vespa/searchcore/proton/common/CMakeLists.txt index 07d749d8c4f..219a2ea43a4 100644 --- a/searchcore/src/vespa/searchcore/proton/common/CMakeLists.txt +++ b/searchcore/src/vespa/searchcore/proton/common/CMakeLists.txt @@ -16,7 +16,6 @@ vespa_add_library(searchcore_pcommon STATIC hw_info_sampler.cpp indexschema_inspector.cpp ipendinglidtracker.cpp - monitored_refcount.cpp operation_rate_tracker.cpp pendinglidtracker.cpp select_utils.cpp diff --git a/searchcore/src/vespa/searchcore/proton/common/feedtoken.h b/searchcore/src/vespa/searchcore/proton/common/feedtoken.h index e75c16ddef6..8ccb4863878 100644 --- a/searchcore/src/vespa/searchcore/proton/common/feedtoken.h +++ b/searchcore/src/vespa/searchcore/proton/common/feedtoken.h @@ -50,21 +50,22 @@ private: */ class OwningState : public State { public: - OwningState(std::unique_ptr<ITransport> transport) + OwningState(std::shared_ptr<ITransport> transport) : State(*transport), _owned(std::move(transport)) {} ~OwningState() override; private: - std::unique_ptr<ITransport> _owned; + std::shared_ptr<ITransport> _owned; }; inline std::shared_ptr<State> make(ITransport & latch) { return std::make_shared<State>(latch); } + inline std::shared_ptr<State> -make(std::unique_ptr<ITransport> transport) { +make(std::shared_ptr<ITransport> transport) { return std::make_shared<OwningState>(std::move(transport)); } diff --git a/searchcore/src/vespa/searchcore/proton/flushengine/flushengine.h b/searchcore/src/vespa/searchcore/proton/flushengine/flushengine.h index 068d7bd033c..4eaa722e0ba 100644 --- a/searchcore/src/vespa/searchcore/proton/flushengine/flushengine.h +++ b/searchcore/src/vespa/searchcore/proton/flushengine/flushengine.h @@ -114,7 +114,7 @@ public: * * @return executor stats **/ - vespalib::ThreadStackExecutor::Stats getExecutorStats() { return _executor.getStats(); } + vespalib::ExecutorStats getExecutorStats() { return _executor.getStats(); } /** * Returns the underlying executor. Only used for state explorers. diff --git a/searchcore/src/vespa/searchcore/proton/matchengine/matchengine.h b/searchcore/src/vespa/searchcore/proton/matchengine/matchengine.h index 9364fc7b097..74a39a3ec78 100644 --- a/searchcore/src/vespa/searchcore/proton/matchengine/matchengine.h +++ b/searchcore/src/vespa/searchcore/proton/matchengine/matchengine.h @@ -62,7 +62,7 @@ public: * * @return executor stats **/ - vespalib::ThreadStackExecutor::Stats getExecutorStats() { return _executor.getStats(); } + vespalib::ExecutorStats getExecutorStats() { return _executor.getStats(); } /** * Returns the underlying executor. Only used for state explorers. diff --git a/searchcore/src/vespa/searchcore/proton/metrics/executor_metrics.cpp b/searchcore/src/vespa/searchcore/proton/metrics/executor_metrics.cpp index fa825024878..74e0971178c 100644 --- a/searchcore/src/vespa/searchcore/proton/metrics/executor_metrics.cpp +++ b/searchcore/src/vespa/searchcore/proton/metrics/executor_metrics.cpp @@ -5,7 +5,7 @@ namespace proton { void -ExecutorMetrics::update(const vespalib::ThreadStackExecutorBase::Stats &stats) +ExecutorMetrics::update(const vespalib::ExecutorStats &stats) { maxPending.set(stats.queueSize.max()); accepted.inc(stats.acceptedTasks); diff --git a/searchcore/src/vespa/searchcore/proton/metrics/executor_metrics.h b/searchcore/src/vespa/searchcore/proton/metrics/executor_metrics.h index 5f7dfdf45b0..273c4ed8979 100644 --- a/searchcore/src/vespa/searchcore/proton/metrics/executor_metrics.h +++ b/searchcore/src/vespa/searchcore/proton/metrics/executor_metrics.h @@ -5,7 +5,7 @@ #include <vespa/metrics/metricset.h> #include <vespa/metrics/countmetric.h> #include <vespa/metrics/valuemetric.h> -#include <vespa/vespalib/util/threadstackexecutorbase.h> +#include <vespa/vespalib/util/executor_stats.h> namespace proton { @@ -16,7 +16,7 @@ struct ExecutorMetrics : metrics::MetricSet metrics::LongCountMetric rejected; metrics::LongAverageMetric queueSize; - void update(const vespalib::ThreadStackExecutorBase::Stats &stats); + void update(const vespalib::ExecutorStats &stats); ExecutorMetrics(const std::string &name, metrics::MetricSet *parent); ~ExecutorMetrics(); }; diff --git a/searchcore/src/vespa/searchcore/proton/metrics/executor_threading_service_stats.h b/searchcore/src/vespa/searchcore/proton/metrics/executor_threading_service_stats.h index 0a113207f65..e2c53af11b5 100644 --- a/searchcore/src/vespa/searchcore/proton/metrics/executor_threading_service_stats.h +++ b/searchcore/src/vespa/searchcore/proton/metrics/executor_threading_service_stats.h @@ -11,10 +11,8 @@ namespace proton { * document db. */ class ExecutorThreadingServiceStats { -public: - using Stats = vespalib::ExecutorStats; - private: + using Stats = vespalib::ExecutorStats; Stats _masterExecutorStats; Stats _indexExecutorStats; Stats _summaryExecutorStats; diff --git a/searchcore/src/vespa/searchcore/proton/persistenceengine/ipersistencehandler.h b/searchcore/src/vespa/searchcore/proton/persistenceengine/ipersistencehandler.h index b4544868bbe..b393a85f632 100644 --- a/searchcore/src/vespa/searchcore/proton/persistenceengine/ipersistencehandler.h +++ b/searchcore/src/vespa/searchcore/proton/persistenceengine/ipersistencehandler.h @@ -53,7 +53,7 @@ public: virtual void handleSetActiveState(const storage::spi::Bucket &bucket, storage::spi::BucketInfo::ActiveState newState, - IGenericResultHandler &resultHandler) = 0; + std::shared_ptr<IGenericResultHandler> resultHandler) = 0; virtual void handleGetBucketInfo(const storage::spi::Bucket &bucket, IBucketInfoResultHandler &resultHandler) = 0; virtual void handleCreateBucket(FeedToken token, const storage::spi::Bucket &bucket) = 0; diff --git a/searchcore/src/vespa/searchcore/proton/persistenceengine/persistenceengine.cpp b/searchcore/src/vespa/searchcore/proton/persistenceengine/persistenceengine.cpp index 136d95a068b..114292d055d 100644 --- a/searchcore/src/vespa/searchcore/proton/persistenceengine/persistenceengine.cpp +++ b/searchcore/src/vespa/searchcore/proton/persistenceengine/persistenceengine.cpp @@ -306,20 +306,21 @@ PersistenceEngine::setClusterState(BucketSpace bucketSpace, const ClusterState & } -Result -PersistenceEngine::setActiveState(const Bucket& bucket, - storage::spi::BucketInfo::ActiveState newState) +void +PersistenceEngine::setActiveStateAsync(const Bucket & bucket, BucketInfo::ActiveState newState, OperationComplete::UP onComplete) { ReadGuard rguard(_rwMutex); HandlerSnapshot snap = getHandlerSnapshot(rguard, bucket.getBucketSpace()); - auto catchResult = std::make_unique<storage::spi::CatchResult>(); - auto futureResult = catchResult->future_result(); - GenericResultHandler resultHandler(snap.size(), std::move(catchResult)); - for (; snap.handlers().valid(); snap.handlers().next()) { + auto resultHandler = std::make_shared<GenericResultHandler>(snap.size(), std::move(onComplete)); + while (snap.handlers().valid()) { IPersistenceHandler *handler = snap.handlers().get(); - handler->handleSetActiveState(bucket, newState, resultHandler); + snap.handlers().next(); + if (snap.handlers().valid()) { + handler->handleSetActiveState(bucket, newState, resultHandler); + } else { + handler->handleSetActiveState(bucket, newState, std::move(resultHandler)); + } } - return *futureResult.get(); } @@ -363,7 +364,7 @@ PersistenceEngine::putAsync(const Bucket &bucket, Timestamp ts, storage::spi::Do return onComplete->onComplete(std::make_unique<Result>(Result::ErrorType::PERMANENT_ERROR, make_string("No handler for document type '%s'", docType.toString().c_str()))); } - auto transportContext = std::make_unique<AsyncTranportContext>(1, std::move(onComplete)); + auto transportContext = std::make_shared<AsyncTranportContext>(1, std::move(onComplete)); handler->handlePut(feedtoken::make(std::move(transportContext)), bucket, ts, std::move(doc)); } @@ -383,7 +384,7 @@ PersistenceEngine::removeAsync(const Bucket& b, Timestamp t, const DocumentId& d return onComplete->onComplete(std::make_unique<RemoveResult>(Result::ErrorType::PERMANENT_ERROR, make_string("No handler for document type '%s'", docType.toString().c_str()))); } - auto transportContext = std::make_unique<AsyncTranportContext>(1, std::move(onComplete)); + auto transportContext = std::make_shared<AsyncTranportContext>(1, std::move(onComplete)); handler->handleRemove(feedtoken::make(std::move(transportContext)), b, t, did); } @@ -435,7 +436,7 @@ PersistenceEngine::updateAsync(const Bucket& b, Timestamp t, DocumentUpdate::SP if (handler == nullptr) { return onComplete->onComplete(std::make_unique<UpdateResult>(Result::ErrorType::PERMANENT_ERROR, make_string("No handler for document type '%s'", docType.toString().c_str()))); } - auto transportContext = std::make_unique<AsyncTranportContext>(1, std::move(onComplete)); + auto transportContext = std::make_shared<AsyncTranportContext>(1, std::move(onComplete)); handler->handleUpdate(feedtoken::make(std::move(transportContext)), b, t, std::move(upd)); } @@ -563,19 +564,23 @@ PersistenceEngine::createBucket(const Bucket &b, Context &) } -Result -PersistenceEngine::deleteBucket(const Bucket& b, Context&) +void +PersistenceEngine::deleteBucketAsync(const Bucket& b, Context&, OperationComplete::UP onComplete) { ReadGuard rguard(_rwMutex); LOG(spam, "deleteBucket(%s)", b.toString().c_str()); HandlerSnapshot snap = getHandlerSnapshot(rguard, b.getBucketSpace()); - TransportLatch latch(snap.size()); - for (; snap.handlers().valid(); snap.handlers().next()) { + + auto transportContext = std::make_shared<AsyncTranportContext>(snap.size(), std::move(onComplete)); + while (snap.handlers().valid()) { IPersistenceHandler *handler = snap.handlers().get(); - handler->handleDeleteBucket(feedtoken::make(latch), b); + snap.handlers().next(); + if (snap.handlers().valid()) { + handler->handleDeleteBucket(feedtoken::make(transportContext), b); + } else { + handler->handleDeleteBucket(feedtoken::make(std::move(transportContext)), b); + } } - latch.await(); - return latch.getResult(); } diff --git a/searchcore/src/vespa/searchcore/proton/persistenceengine/persistenceengine.h b/searchcore/src/vespa/searchcore/proton/persistenceengine/persistenceengine.h index 0aeb3e16351..94331ac2cd6 100644 --- a/searchcore/src/vespa/searchcore/proton/persistenceengine/persistenceengine.h +++ b/searchcore/src/vespa/searchcore/proton/persistenceengine/persistenceengine.h @@ -103,7 +103,7 @@ public: Result initialize() override; BucketIdListResult listBuckets(BucketSpace bucketSpace) const override; Result setClusterState(BucketSpace bucketSpace, const ClusterState& calc) override; - Result setActiveState(const Bucket& bucket, BucketInfo::ActiveState newState) override; + void setActiveStateAsync(const Bucket&, BucketInfo::ActiveState, OperationComplete::UP) override; BucketInfoResult getBucketInfo(const Bucket&) const override; void putAsync(const Bucket &, Timestamp, storage::spi::DocumentSP, Context &context, OperationComplete::UP) override; void removeAsync(const Bucket&, Timestamp, const document::DocumentId&, Context&, OperationComplete::UP) override; @@ -115,7 +115,7 @@ public: Result destroyIterator(IteratorId, Context&) override; Result createBucket(const Bucket &bucketId, Context &) override ; - Result deleteBucket(const Bucket&, Context&) override; + void deleteBucketAsync(const Bucket&, Context&, OperationComplete::UP) override; BucketIdListResult getModifiedBuckets(BucketSpace bucketSpace) const override; Result split(const Bucket& source, const Bucket& target1, const Bucket& target2, Context&) override; Result join(const Bucket& source1, const Bucket& source2, const Bucket& target, Context&) override; diff --git a/searchcore/src/vespa/searchcore/proton/reference/document_db_reference_resolver.cpp b/searchcore/src/vespa/searchcore/proton/reference/document_db_reference_resolver.cpp index 4cc5fb0b5db..5d1e1318b04 100644 --- a/searchcore/src/vespa/searchcore/proton/reference/document_db_reference_resolver.cpp +++ b/searchcore/src/vespa/searchcore/proton/reference/document_db_reference_resolver.cpp @@ -29,6 +29,8 @@ using search::AttributeVector; using search::IAttributeManager; using search::NotImplementedAttribute; using vespalib::ISequencedTaskExecutor; +using vespalib::MonitoredRefCount; +using vespalib::RetainGuard; using vespa::config::search::ImportedFieldsConfig; namespace proton { diff --git a/searchcore/src/vespa/searchcore/proton/reference/document_db_reference_resolver.h b/searchcore/src/vespa/searchcore/proton/reference/document_db_reference_resolver.h index 761b9407fe5..522cdf83477 100644 --- a/searchcore/src/vespa/searchcore/proton/reference/document_db_reference_resolver.h +++ b/searchcore/src/vespa/searchcore/proton/reference/document_db_reference_resolver.h @@ -21,15 +21,17 @@ namespace search::attribute { namespace vespa::config::search::internal { class InternalImportedFieldsType; } namespace vespalib { - class ISequencedTaskExecutor; + +class ISequencedTaskExecutor; +class MonitoredRefCount; } + namespace proton { class IDocumentDBReference; class IDocumentDBReferenceRegistry; class ImportedAttributesRepo; class GidToLidChangeRegistrator; -class MonitoredRefCount; /** * Class that for a given document db resolves all references to parent document dbs: @@ -42,7 +44,7 @@ private: const document::DocumentType &_thisDocType; const ImportedFieldsConfig &_importedFieldsCfg; const document::DocumentType &_prevThisDocType; - MonitoredRefCount &_refCount; + vespalib::MonitoredRefCount &_refCount; vespalib::ISequencedTaskExecutor &_attributeFieldWriter; bool _useReferences; std::map<vespalib::string, std::unique_ptr<GidToLidChangeRegistrator>> _registrators; @@ -61,7 +63,7 @@ public: const document::DocumentType &thisDocType, const ImportedFieldsConfig &importedFieldsCfg, const document::DocumentType &prevThisDocType, - MonitoredRefCount &refCount, + vespalib::MonitoredRefCount &refCount, vespalib::ISequencedTaskExecutor &attributeFieldWriter, bool useReferences); ~DocumentDBReferenceResolver() override; diff --git a/searchcore/src/vespa/searchcore/proton/reference/gid_to_lid_change_listener.cpp b/searchcore/src/vespa/searchcore/proton/reference/gid_to_lid_change_listener.cpp index c7e716b4173..427cbab2a14 100644 --- a/searchcore/src/vespa/searchcore/proton/reference/gid_to_lid_change_listener.cpp +++ b/searchcore/src/vespa/searchcore/proton/reference/gid_to_lid_change_listener.cpp @@ -3,6 +3,8 @@ #include "gid_to_lid_change_listener.h" #include <future> +using vespalib::RetainGuard; + namespace proton { GidToLidChangeListener::GidToLidChangeListener(vespalib::ISequencedTaskExecutor &attributeFieldWriter, diff --git a/searchcore/src/vespa/searchcore/proton/reference/gid_to_lid_change_listener.h b/searchcore/src/vespa/searchcore/proton/reference/gid_to_lid_change_listener.h index 23bb32af87a..28e9684ed86 100644 --- a/searchcore/src/vespa/searchcore/proton/reference/gid_to_lid_change_listener.h +++ b/searchcore/src/vespa/searchcore/proton/reference/gid_to_lid_change_listener.h @@ -4,8 +4,8 @@ #include "i_gid_to_lid_change_listener.h" #include <vespa/searchlib/attribute/reference_attribute.h> +#include <vespa/vespalib/util/retain_guard.h> #include <vespa/vespalib/util/sequencedtaskexecutor.h> -#include <vespa/searchcore/proton/common/monitored_refcount.h> namespace proton { @@ -18,14 +18,14 @@ class GidToLidChangeListener : public IGidToLidChangeListener vespalib::ISequencedTaskExecutor &_attributeFieldWriter; vespalib::ISequencedTaskExecutor::ExecutorId _executorId; std::shared_ptr<search::attribute::ReferenceAttribute> _attr; - RetainGuard _retainGuard; + vespalib::RetainGuard _retainGuard; vespalib::string _name; vespalib::string _docTypeName; public: GidToLidChangeListener(vespalib::ISequencedTaskExecutor &attributeFieldWriter, std::shared_ptr<search::attribute::ReferenceAttribute> attr, - RetainGuard refCount, + vespalib::RetainGuard refCount, const vespalib::string &name, const vespalib::string &docTypeName); ~GidToLidChangeListener() override; diff --git a/searchcore/src/vespa/searchcore/proton/server/buckethandler.cpp b/searchcore/src/vespa/searchcore/proton/server/buckethandler.cpp index d6602e18c81..c15be9336fe 100644 --- a/searchcore/src/vespa/searchcore/proton/server/buckethandler.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/buckethandler.cpp @@ -98,10 +98,10 @@ BucketHandler::handleListBuckets(IBucketIdListResultHandler &resultHandler) void BucketHandler::handleSetCurrentState(const BucketId &bucketId, storage::spi::BucketInfo::ActiveState newState, - IGenericResultHandler &resultHandler) + std::shared_ptr<IGenericResultHandler> resultHandlerSP) { - _executor.execute(makeLambdaTask([this, bucketId, newState, resultHandlerP = &resultHandler]() { - performSetCurrentState(bucketId, newState, resultHandlerP); + _executor.execute(makeLambdaTask([this, bucketId, newState, resultHandler = std::move(resultHandlerSP)]() { + performSetCurrentState(bucketId, newState, resultHandler.get()); })); } diff --git a/searchcore/src/vespa/searchcore/proton/server/buckethandler.h b/searchcore/src/vespa/searchcore/proton/server/buckethandler.h index 927744e1b8e..7f44d2ebd71 100644 --- a/searchcore/src/vespa/searchcore/proton/server/buckethandler.h +++ b/searchcore/src/vespa/searchcore/proton/server/buckethandler.h @@ -55,7 +55,7 @@ public: void handleListBuckets(IBucketIdListResultHandler &resultHandler); void handleSetCurrentState(const document::BucketId &bucketId, storage::spi::BucketInfo::ActiveState newState, - IGenericResultHandler &resultHandler); + std::shared_ptr<IGenericResultHandler> resultHandler); void handleGetBucketInfo(const storage::spi::Bucket &bucket, IBucketInfoResultHandler &resultHandler); void handleListActiveBuckets(IBucketIdListResultHandler &resultHandler); diff --git a/searchcore/src/vespa/searchcore/proton/server/bucketmovejob.cpp b/searchcore/src/vespa/searchcore/proton/server/bucketmovejob.cpp index c70928cd5e8..386d8367e52 100644 --- a/searchcore/src/vespa/searchcore/proton/server/bucketmovejob.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/bucketmovejob.cpp @@ -25,6 +25,7 @@ using document::BucketId; using storage::spi::BucketInfo; using storage::spi::Bucket; using proton::bucketdb::BucketMover; +using vespalib::RetainGuard; using vespalib::makeLambdaTask; using vespalib::Trinary; diff --git a/searchcore/src/vespa/searchcore/proton/server/bucketmovejob.h b/searchcore/src/vespa/searchcore/proton/server/bucketmovejob.h index 66d24efb466..d4438fcd411 100644 --- a/searchcore/src/vespa/searchcore/proton/server/bucketmovejob.h +++ b/searchcore/src/vespa/searchcore/proton/server/bucketmovejob.h @@ -10,7 +10,7 @@ #include "maintenancedocumentsubdb.h" #include <vespa/searchcore/proton/bucketdb/bucketscaniterator.h> #include <vespa/searchcore/proton/bucketdb/i_bucket_create_listener.h> -#include <vespa/searchcore/proton/common/monitored_refcount.h> +#include <vespa/vespalib/util/retain_guard.h> namespace storage::spi { struct BucketExecutor; } @@ -59,7 +59,7 @@ private: using Movers = std::vector<BucketMoverSP>; using GuardedMoveOps = BucketMover::GuardedMoveOps; std::shared_ptr<IBucketStateCalculator> _calc; - RetainGuard _dbRetainer; + vespalib::RetainGuard _dbRetainer; IDocumentMoveHandler &_moveHandler; IBucketModifiedHandler &_modifiedHandler; IThreadService &_master; @@ -80,7 +80,7 @@ private: IDiskMemUsageNotifier &_diskMemUsageNotifier; BucketMoveJob(const std::shared_ptr<IBucketStateCalculator> &calc, - RetainGuard dbRetainer, + vespalib::RetainGuard dbRetainer, IDocumentMoveHandler &moveHandler, IBucketModifiedHandler &modifiedHandler, IThreadService & master, @@ -115,7 +115,7 @@ private: public: static std::shared_ptr<BucketMoveJob> create(const std::shared_ptr<IBucketStateCalculator> &calc, - RetainGuard dbRetainer, + vespalib::RetainGuard dbRetainer, IDocumentMoveHandler &moveHandler, IBucketModifiedHandler &modifiedHandler, IThreadService & master, diff --git a/searchcore/src/vespa/searchcore/proton/server/documentdb.h b/searchcore/src/vespa/searchcore/proton/server/documentdb.h index 936ba812a44..014bba11f83 100644 --- a/searchcore/src/vespa/searchcore/proton/server/documentdb.h +++ b/searchcore/src/vespa/searchcore/proton/server/documentdb.h @@ -19,13 +19,13 @@ #include "threading_service_config.h" #include <vespa/searchcore/proton/attribute/attribute_usage_filter.h> #include <vespa/searchcore/proton/common/doctypename.h> -#include <vespa/searchcore/proton/common/monitored_refcount.h> #include <vespa/searchcore/proton/metrics/documentdb_job_trackers.h> #include <vespa/searchcore/proton/metrics/documentdb_tagged_metrics.h> #include <vespa/searchcore/proton/persistenceengine/i_resource_write_filter.h> #include <vespa/searchcore/proton/index/indexmanager.h> #include <vespa/searchlib/docstore/cachestats.h> #include <vespa/searchlib/transactionlog/syncproxy.h> +#include <vespa/vespalib/util/retain_guard.h> #include <vespa/vespalib/util/varholder.h> #include <mutex> #include <condition_variable> @@ -111,7 +111,7 @@ private: DocumentDBTaggedMetrics _metrics; std::unique_ptr<metrics::UpdateHook> _metricsHook; vespalib::VarHolder<IFeedView::SP> _feedView; - MonitoredRefCount _refCount; + vespalib::MonitoredRefCount _refCount; bool _syncFeedViewEnabled; IDocumentDBOwner &_owner; storage::spi::BucketExecutor &_bucketExecutor; @@ -381,7 +381,7 @@ public: /** * Reference counting */ - RetainGuard retain() { return RetainGuard(_refCount); } + vespalib::RetainGuard retain() { return vespalib::RetainGuard(_refCount); } bool getDelayedConfig() const { return _state.getDelayedConfig(); } void replayConfig(SerialNum serialNum) override; diff --git a/searchcore/src/vespa/searchcore/proton/server/executor_thread_service.cpp b/searchcore/src/vespa/searchcore/proton/server/executor_thread_service.cpp index 3b4f12b7c85..684132b34e7 100644 --- a/searchcore/src/vespa/searchcore/proton/server/executor_thread_service.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/executor_thread_service.cpp @@ -73,7 +73,7 @@ ExecutorThreadService::isCurrentThread() const return FastOS_Thread::CompareThreadIds(_threadId->_id, currentThreadId); } -vespalib::ThreadExecutor::Stats ExecutorThreadService::getStats() { +vespalib::ExecutorStats ExecutorThreadService::getStats() { return _executor.getStats(); } diff --git a/searchcore/src/vespa/searchcore/proton/server/executor_thread_service.h b/searchcore/src/vespa/searchcore/proton/server/executor_thread_service.h index 8adb80889e7..44a330ca696 100644 --- a/searchcore/src/vespa/searchcore/proton/server/executor_thread_service.h +++ b/searchcore/src/vespa/searchcore/proton/server/executor_thread_service.h @@ -21,7 +21,7 @@ public: ExecutorThreadService(vespalib::SyncableThreadExecutor &executor); ~ExecutorThreadService(); - Stats getStats() override; + vespalib::ExecutorStats getStats() override; vespalib::Executor::Task::UP execute(vespalib::Executor::Task::UP task) override { return _executor.execute(std::move(task)); diff --git a/searchcore/src/vespa/searchcore/proton/server/flushhandlerproxy.h b/searchcore/src/vespa/searchcore/proton/server/flushhandlerproxy.h index fe937a35db7..2098466a2c0 100644 --- a/searchcore/src/vespa/searchcore/proton/server/flushhandlerproxy.h +++ b/searchcore/src/vespa/searchcore/proton/server/flushhandlerproxy.h @@ -3,7 +3,7 @@ #pragma once #include <vespa/searchcore/proton/flushengine/iflushhandler.h> -#include <vespa/searchcore/proton/common/monitored_refcount.h> +#include <vespa/vespalib/util/retain_guard.h> namespace proton { @@ -13,7 +13,7 @@ class FlushHandlerProxy : public IFlushHandler { private: std::shared_ptr<DocumentDB> _documentDB; - RetainGuard _retainGuard; + vespalib::RetainGuard _retainGuard; public: FlushHandlerProxy(const std::shared_ptr<DocumentDB> &documentDB); diff --git a/searchcore/src/vespa/searchcore/proton/server/lid_space_compaction_job.cpp b/searchcore/src/vespa/searchcore/proton/server/lid_space_compaction_job.cpp index efe92a11f02..a5c1d1fc2c9 100644 --- a/searchcore/src/vespa/searchcore/proton/server/lid_space_compaction_job.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/lid_space_compaction_job.cpp @@ -26,6 +26,7 @@ using search::DocumentMetaData; using search::LidUsageStats; using storage::spi::makeBucketTask; using storage::spi::Bucket; +using vespalib::RetainGuard; using vespalib::makeLambdaTask; namespace proton::lidspace { diff --git a/searchcore/src/vespa/searchcore/proton/server/lid_space_compaction_job.h b/searchcore/src/vespa/searchcore/proton/server/lid_space_compaction_job.h index 717c19f093d..917ff12be4a 100644 --- a/searchcore/src/vespa/searchcore/proton/server/lid_space_compaction_job.h +++ b/searchcore/src/vespa/searchcore/proton/server/lid_space_compaction_job.h @@ -6,9 +6,9 @@ #include "document_db_maintenance_config.h" #include "i_disk_mem_usage_listener.h" #include "iclusterstatechangedhandler.h" -#include <vespa/searchlib/common/idocumentmetastore.h> -#include <vespa/searchcore/proton/common/monitored_refcount.h> #include <vespa/document/bucket/bucketspace.h> +#include <vespa/searchlib/common/idocumentmetastore.h> +#include <vespa/vespalib/util/retain_guard.h> #include <atomic> namespace storage::spi { struct BucketExecutor; } @@ -50,7 +50,7 @@ private: bool _shouldCompactLidSpace; IThreadService &_master; BucketExecutor &_bucketExecutor; - RetainGuard _dbRetainer; + vespalib::RetainGuard _dbRetainer; document::BucketSpace _bucketSpace; bool hasTooMuchLidBloat(const search::LidUsageStats &stats) const; @@ -68,7 +68,7 @@ private: class MoveTask; CompactionJob(const DocumentDBLidSpaceCompactionConfig &config, - RetainGuard dbRetainer, + vespalib::RetainGuard dbRetainer, std::shared_ptr<ILidSpaceCompactionHandler> handler, IOperationStorer &opStorer, IThreadService & master, @@ -81,7 +81,7 @@ private: public: static std::shared_ptr<CompactionJob> create(const DocumentDBLidSpaceCompactionConfig &config, - RetainGuard dbRetainer, + vespalib::RetainGuard dbRetainer, std::shared_ptr<ILidSpaceCompactionHandler> handler, IOperationStorer &opStorer, IThreadService & master, diff --git a/searchcore/src/vespa/searchcore/proton/server/maintenancecontroller.cpp b/searchcore/src/vespa/searchcore/proton/server/maintenancecontroller.cpp index 2adcdbab217..c4826bba8ea 100644 --- a/searchcore/src/vespa/searchcore/proton/server/maintenancecontroller.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/maintenancecontroller.cpp @@ -14,6 +14,7 @@ LOG_SETUP(".proton.server.maintenancecontroller"); using document::BucketId; using vespalib::Executor; +using vespalib::MonitoredRefCount; using vespalib::makeLambdaTask; namespace proton { diff --git a/searchcore/src/vespa/searchcore/proton/server/maintenancecontroller.h b/searchcore/src/vespa/searchcore/proton/server/maintenancecontroller.h index bddd58a349a..8c8cc3e2d43 100644 --- a/searchcore/src/vespa/searchcore/proton/server/maintenancecontroller.h +++ b/searchcore/src/vespa/searchcore/proton/server/maintenancecontroller.h @@ -5,14 +5,17 @@ #include "maintenancedocumentsubdb.h" #include "i_maintenance_job.h" #include <vespa/searchcore/proton/common/doctypename.h> -#include <vespa/searchcore/proton/common/monitored_refcount.h> +#include <vespa/vespalib/util/retain_guard.h> #include <vespa/vespalib/util/scheduledexecutor.h> #include <mutex> namespace vespalib { - class Timer; - class Executor; + +class Executor; +class MonitoredRefCount; +class Timer; + } namespace searchcorespi::index { struct IThreadService; } @@ -20,7 +23,6 @@ namespace proton { class MaintenanceJobRunner; class DocumentDBMaintenanceConfig; -class MonitoredRefCount; /** * Class that controls the bucket moving between ready and notready sub databases @@ -36,7 +38,7 @@ public: using UP = std::unique_ptr<MaintenanceController>; enum class State {INITIALIZING, STARTED, PAUSED, STOPPING}; - MaintenanceController(IThreadService &masterThread, vespalib::Executor & defaultExecutor, MonitoredRefCount & refCount, const DocTypeName &docTypeName); + MaintenanceController(IThreadService &masterThread, vespalib::Executor & defaultExecutor, vespalib::MonitoredRefCount & refCount, const DocTypeName &docTypeName); ~MaintenanceController(); void registerJobInMasterThread(IMaintenanceJob::UP job); @@ -70,14 +72,14 @@ public: const MaintenanceDocumentSubDB & getNotReadySubDB() const { return _notReadySubDB; } IThreadService & masterThread() { return _masterThread; } const DocTypeName & getDocTypeName() const { return _docTypeName; } - RetainGuard retainDB() { return RetainGuard(_refCount); } + vespalib::RetainGuard retainDB() { return vespalib::RetainGuard(_refCount); } private: using Mutex = std::mutex; using Guard = std::lock_guard<Mutex>; IThreadService &_masterThread; vespalib::Executor &_defaultExecutor; - MonitoredRefCount &_refCount; + vespalib::MonitoredRefCount &_refCount; MaintenanceDocumentSubDB _readySubDB; MaintenanceDocumentSubDB _remSubDB; MaintenanceDocumentSubDB _notReadySubDB; diff --git a/searchcore/src/vespa/searchcore/proton/server/persistencehandlerproxy.cpp b/searchcore/src/vespa/searchcore/proton/server/persistencehandlerproxy.cpp index 3d464cced5b..bec9197501b 100644 --- a/searchcore/src/vespa/searchcore/proton/server/persistencehandlerproxy.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/persistencehandlerproxy.cpp @@ -69,9 +69,9 @@ PersistenceHandlerProxy::handleSetClusterState(const storage::spi::ClusterState void PersistenceHandlerProxy::handleSetActiveState(const storage::spi::Bucket &bucket, storage::spi::BucketInfo::ActiveState newState, - IGenericResultHandler &resultHandler) + std::shared_ptr<IGenericResultHandler> resultHandler) { - _bucketHandler.handleSetCurrentState(bucket.getBucketId().stripUnused(), newState, resultHandler); + _bucketHandler.handleSetCurrentState(bucket.getBucketId().stripUnused(), newState, std::move(resultHandler)); } void diff --git a/searchcore/src/vespa/searchcore/proton/server/persistencehandlerproxy.h b/searchcore/src/vespa/searchcore/proton/server/persistencehandlerproxy.h index 96bfbe18423..5c35434aae8 100644 --- a/searchcore/src/vespa/searchcore/proton/server/persistencehandlerproxy.h +++ b/searchcore/src/vespa/searchcore/proton/server/persistencehandlerproxy.h @@ -3,7 +3,7 @@ #pragma once #include <vespa/searchcore/proton/persistenceengine/ipersistencehandler.h> -#include <vespa/searchcore/proton/common/monitored_refcount.h> +#include <vespa/vespalib/util/retain_guard.h> namespace proton { @@ -18,7 +18,7 @@ private: FeedHandler &_feedHandler; BucketHandler &_bucketHandler; ClusterStateHandler &_clusterStateHandler; - RetainGuard _retainGuard; + vespalib::RetainGuard _retainGuard; public: explicit PersistenceHandlerProxy(std::shared_ptr<DocumentDB> documentDB); @@ -40,7 +40,7 @@ public: void handleSetClusterState(const storage::spi::ClusterState &calc, IGenericResultHandler &resultHandler) override; void handleSetActiveState(const storage::spi::Bucket &bucket, storage::spi::BucketInfo::ActiveState newState, - IGenericResultHandler &resultHandler) override; + std::shared_ptr<IGenericResultHandler> resultHandler) override; void handleGetBucketInfo(const storage::spi::Bucket &bucket, IBucketInfoResultHandler &resultHandler) override; void handleCreateBucket(FeedToken token, const storage::spi::Bucket &bucket) override; diff --git a/searchcore/src/vespa/searchcore/proton/server/proton.cpp b/searchcore/src/vespa/searchcore/proton/server/proton.cpp index 17c255d506a..edf68633124 100644 --- a/searchcore/src/vespa/searchcore/proton/server/proton.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/proton.cpp @@ -737,8 +737,7 @@ Proton::prepareRestart() namespace { void -updateExecutorMetrics(ExecutorMetrics &metrics, - const vespalib::ThreadStackExecutor::Stats &stats) +updateExecutorMetrics(ExecutorMetrics &metrics, const vespalib::ExecutorStats &stats) { metrics.update(stats); } diff --git a/searchcore/src/vespa/searchcore/proton/server/pruneremoveddocumentsjob.cpp b/searchcore/src/vespa/searchcore/proton/server/pruneremoveddocumentsjob.cpp index c266a604e33..6fe1947bd1d 100644 --- a/searchcore/src/vespa/searchcore/proton/server/pruneremoveddocumentsjob.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/pruneremoveddocumentsjob.cpp @@ -16,6 +16,7 @@ using document::BucketId; using storage::spi::Timestamp; using storage::spi::Bucket; using vespalib::IDestructorCallback; +using vespalib::RetainGuard; using vespalib::makeLambdaTask; namespace proton { diff --git a/searchcore/src/vespa/searchcore/proton/server/pruneremoveddocumentsjob.h b/searchcore/src/vespa/searchcore/proton/server/pruneremoveddocumentsjob.h index 5c52991d0c3..6198f588a32 100644 --- a/searchcore/src/vespa/searchcore/proton/server/pruneremoveddocumentsjob.h +++ b/searchcore/src/vespa/searchcore/proton/server/pruneremoveddocumentsjob.h @@ -3,9 +3,9 @@ #include "blockable_maintenance_job.h" #include "document_db_maintenance_config.h" -#include <vespa/searchcore/proton/common/monitored_refcount.h> #include <persistence/spi/types.h> #include <vespa/document/bucket/bucketspace.h> +#include <vespa/vespalib/util/retain_guard.h> #include <atomic> namespace storage::spi { struct BucketExecutor; } @@ -36,7 +36,7 @@ private: IThreadService &_master; BucketExecutor &_bucketExecutor; const vespalib::string _docTypeName; - RetainGuard _dbRetainer; + vespalib::RetainGuard _dbRetainer; const vespalib::duration _cfgAgeLimit; const uint32_t _subDbId; const document::BucketSpace _bucketSpace; @@ -45,14 +45,14 @@ private: void remove(uint32_t lid, const RawDocumentMetaData & meta); - PruneRemovedDocumentsJob(const DocumentDBPruneConfig &config, RetainGuard dbRetainer, const IDocumentMetaStore &metaStore, + PruneRemovedDocumentsJob(const DocumentDBPruneConfig &config, vespalib::RetainGuard dbRetainer, const IDocumentMetaStore &metaStore, uint32_t subDbId, document::BucketSpace bucketSpace, const vespalib::string &docTypeName, IPruneRemovedDocumentsHandler &handler, IThreadService & master, BucketExecutor & bucketExecutor); bool run() override; public: static std::shared_ptr<PruneRemovedDocumentsJob> - create(const Config &config, RetainGuard dbRetainer, const IDocumentMetaStore &metaStore, uint32_t subDbId, + create(const Config &config, vespalib::RetainGuard dbRetainer, const IDocumentMetaStore &metaStore, uint32_t subDbId, document::BucketSpace bucketSpace, const vespalib::string &docTypeName, IPruneRemovedDocumentsHandler &handler, IThreadService & master, BucketExecutor & bucketExecutor) { diff --git a/searchcore/src/vespa/searchcore/proton/server/searchhandlerproxy.h b/searchcore/src/vespa/searchcore/proton/server/searchhandlerproxy.h index 4eaf15b3b1c..f7c52ed4baa 100644 --- a/searchcore/src/vespa/searchcore/proton/server/searchhandlerproxy.h +++ b/searchcore/src/vespa/searchcore/proton/server/searchhandlerproxy.h @@ -3,7 +3,7 @@ #pragma once #include <vespa/searchcore/proton/summaryengine/isearchhandler.h> -#include <vespa/searchcore/proton/common/monitored_refcount.h> +#include <vespa/vespalib/util/retain_guard.h> namespace proton { @@ -13,7 +13,7 @@ class SearchHandlerProxy : public ISearchHandler { private: std::shared_ptr<DocumentDB> _documentDB; - RetainGuard _retainGuard; + vespalib::RetainGuard _retainGuard; public: SearchHandlerProxy(std::shared_ptr<DocumentDB> documentDB); diff --git a/searchcore/src/vespa/searchcore/proton/summaryengine/summaryengine.h b/searchcore/src/vespa/searchcore/proton/summaryengine/summaryengine.h index a439306b69b..34eebdc839d 100644 --- a/searchcore/src/vespa/searchcore/proton/summaryengine/summaryengine.h +++ b/searchcore/src/vespa/searchcore/proton/summaryengine/summaryengine.h @@ -66,7 +66,7 @@ public: * * @return executor stats **/ - vespalib::ThreadStackExecutor::Stats getExecutorStats() { return _executor.getStats(); } + vespalib::ExecutorStats getExecutorStats() { return _executor.getStats(); } /** * Returns the underlying executor. Only used for state explorers. diff --git a/searchcore/src/vespa/searchcore/proton/test/thread_service_observer.h b/searchcore/src/vespa/searchcore/proton/test/thread_service_observer.h index f2e1e64eeb3..26a92841999 100644 --- a/searchcore/src/vespa/searchcore/proton/test/thread_service_observer.h +++ b/searchcore/src/vespa/searchcore/proton/test/thread_service_observer.h @@ -40,7 +40,7 @@ public: } size_t getNumThreads() const override { return _service.getNumThreads(); } - Stats getStats() override { + vespalib::ExecutorStats getStats() override { return _service.getStats(); } diff --git a/staging_vespalib/src/vespa/vespalib/util/adaptive_sequenced_executor.cpp b/staging_vespalib/src/vespa/vespalib/util/adaptive_sequenced_executor.cpp index e4b64b19739..869ff0456e1 100644 --- a/staging_vespalib/src/vespa/vespalib/util/adaptive_sequenced_executor.cpp +++ b/staging_vespalib/src/vespa/vespalib/util/adaptive_sequenced_executor.cpp @@ -324,12 +324,12 @@ AdaptiveSequencedExecutor::setTaskLimit(uint32_t task_limit) } } -AdaptiveSequencedExecutor::Stats +ExecutorStats AdaptiveSequencedExecutor::getStats() { auto guard = std::lock_guard(_mutex); - Stats stats = _stats; - _stats = Stats(); + ExecutorStats stats = _stats; + _stats = ExecutorStats(); _stats.queueSize.add(_self.pending_tasks); return stats; } diff --git a/staging_vespalib/src/vespa/vespalib/util/adaptive_sequenced_executor.h b/staging_vespalib/src/vespa/vespalib/util/adaptive_sequenced_executor.h index 4e0388caf8a..fdcdf35fbbb 100644 --- a/staging_vespalib/src/vespa/vespalib/util/adaptive_sequenced_executor.h +++ b/staging_vespalib/src/vespa/vespalib/util/adaptive_sequenced_executor.h @@ -22,8 +22,7 @@ namespace vespalib { class AdaptiveSequencedExecutor : public ISequencedTaskExecutor { private: - using Stats = vespalib::ExecutorStats; - using Task = vespalib::Executor::Task; + using Task = Executor::Task; struct TaggedTask { Task::UP task; @@ -128,7 +127,7 @@ private: vespalib::ArrayQueue<Worker*> _worker_stack; EventBarrier<BarrierCompletion> _barrier; Self _self; - Stats _stats; + ExecutorStats _stats; Config _cfg; void maybe_block_self(std::unique_lock<std::mutex> &lock); @@ -147,7 +146,7 @@ public: void executeTask(ExecutorId id, Task::UP task) override; void sync() override; void setTaskLimit(uint32_t task_limit) override; - vespalib::ExecutorStats getStats() override; + ExecutorStats getStats() override; Config get_config() const; }; diff --git a/staging_vespalib/src/vespa/vespalib/util/foreground_thread_executor.h b/staging_vespalib/src/vespa/vespalib/util/foreground_thread_executor.h index f3dbd512047..3bd5ca3d49a 100644 --- a/staging_vespalib/src/vespa/vespalib/util/foreground_thread_executor.h +++ b/staging_vespalib/src/vespa/vespalib/util/foreground_thread_executor.h @@ -22,7 +22,7 @@ public: return Task::UP(); } size_t getNumThreads() const override { return 0; } - Stats getStats() override { + ExecutorStats getStats() override { return ExecutorStats(ExecutorStats::QueueSizeT(), _accepted.load(std::memory_order_relaxed), 0); } void setTaskLimit(uint32_t taskLimit) override { (void) taskLimit; } diff --git a/staging_vespalib/src/vespa/vespalib/util/sequencedtaskexecutor.cpp b/staging_vespalib/src/vespa/vespalib/util/sequencedtaskexecutor.cpp index adb4356c4c8..727894397a7 100644 --- a/staging_vespalib/src/vespa/vespalib/util/sequencedtaskexecutor.cpp +++ b/staging_vespalib/src/vespa/vespalib/util/sequencedtaskexecutor.cpp @@ -99,10 +99,10 @@ SequencedTaskExecutor::wakeup() { } } -SequencedTaskExecutor::Stats +ExecutorStats SequencedTaskExecutor::getStats() { - Stats accumulatedStats; + ExecutorStats accumulatedStats; for (auto &executor :* _executors) { accumulatedStats += executor->getStats(); } diff --git a/staging_vespalib/src/vespa/vespalib/util/sequencedtaskexecutor.h b/staging_vespalib/src/vespa/vespalib/util/sequencedtaskexecutor.h index 38844535021..7b49f7aac75 100644 --- a/staging_vespalib/src/vespa/vespalib/util/sequencedtaskexecutor.h +++ b/staging_vespalib/src/vespa/vespalib/util/sequencedtaskexecutor.h @@ -16,7 +16,6 @@ class SyncableThreadExecutor; class SequencedTaskExecutor final : public ISequencedTaskExecutor { public: - using Stats = vespalib::ExecutorStats; using ISequencedTaskExecutor::getExecutorId; using OptimizeFor = vespalib::Executor::OptimizeFor; @@ -26,7 +25,7 @@ public: void executeTask(ExecutorId id, vespalib::Executor::Task::UP task) override; ExecutorId getExecutorId(uint64_t componentId) const override; void sync() override; - Stats getStats() override; + ExecutorStats getStats() override; void wakeup() override; /* diff --git a/staging_vespalib/src/vespa/vespalib/util/singleexecutor.cpp b/staging_vespalib/src/vespa/vespalib/util/singleexecutor.cpp index 7fba68e2092..803ec4f3f7c 100644 --- a/staging_vespalib/src/vespa/vespalib/util/singleexecutor.cpp +++ b/staging_vespalib/src/vespa/vespalib/util/singleexecutor.cpp @@ -158,13 +158,13 @@ SingleExecutor::wait_for_room(Lock & lock) { } } -ThreadExecutor::Stats +ExecutorStats SingleExecutor::getStats() { Lock lock(_mutex); uint64_t accepted = _wp.load(std::memory_order_relaxed); - Stats stats(_queueSize, (accepted - _lastAccepted), 0); + ExecutorStats stats(_queueSize, (accepted - _lastAccepted), 0); _lastAccepted = accepted; - _queueSize = Stats::QueueSizeT() ; + _queueSize = ExecutorStats::QueueSizeT() ; return stats; } diff --git a/staging_vespalib/src/vespa/vespalib/util/singleexecutor.h b/staging_vespalib/src/vespa/vespalib/util/singleexecutor.h index 20e5a560e52..8e9c1ae3fa1 100644 --- a/staging_vespalib/src/vespa/vespalib/util/singleexecutor.h +++ b/staging_vespalib/src/vespa/vespalib/util/singleexecutor.h @@ -29,7 +29,7 @@ public: uint32_t getTaskLimit() const override { return _taskLimit.load(std::memory_order_relaxed); } uint32_t get_watermark() const { return _watermark; } duration get_reaction_time() const { return _reactionTime; } - Stats getStats() override; + ExecutorStats getStats() override; SingleExecutor & shutdown() override; private: using Lock = std::unique_lock<std::mutex>; @@ -55,7 +55,7 @@ private: std::condition_variable _producerCondition; vespalib::Thread _thread; uint64_t _lastAccepted; - Stats::QueueSizeT _queueSize; + ExecutorStats::QueueSizeT _queueSize; std::atomic<uint64_t> _wakeupConsumerAt; std::atomic<uint64_t> _producerNeedWakeupAt; std::atomic<uint64_t> _wp; diff --git a/storage/src/tests/distributor/idealstatemanagertest.cpp b/storage/src/tests/distributor/idealstatemanagertest.cpp index d521975e0cb..a1263c9433b 100644 --- a/storage/src/tests/distributor/idealstatemanagertest.cpp +++ b/storage/src/tests/distributor/idealstatemanagertest.cpp @@ -220,8 +220,8 @@ TEST_F(IdealStateManagerTest, block_check_for_all_operations_to_specific_bucket) pending_message_tracker().insert(msg); } { - RemoveBucketOperation op(dummy_cluster_context, - BucketAndNodes(makeDocumentBucket(bid), toVector<uint16_t>(7))); + // TODO we might not want this particular behavior for merge operations either + MergeOperation op(BucketAndNodes(makeDocumentBucket(bid), toVector<uint16_t>(2, 3))); // Not blocked for exact node match. EXPECT_FALSE(checkBlock(op, makeDocumentBucket(bid), operation_context(), op_seq)); // But blocked for bucket match! diff --git a/storage/src/tests/distributor/removebucketoperationtest.cpp b/storage/src/tests/distributor/removebucketoperationtest.cpp index e877f4601b7..971ff36c833 100644 --- a/storage/src/tests/distributor/removebucketoperationtest.cpp +++ b/storage/src/tests/distributor/removebucketoperationtest.cpp @@ -119,4 +119,16 @@ TEST_F(RemoveBucketOperationTest, fail_with_invalid_bucket_info) { EXPECT_EQ("NONEXISTING", dumpBucket(document::BucketId(16, 1))); } +TEST_F(RemoveBucketOperationTest, operation_blocked_when_pending_message_to_target_node) { + RemoveBucketOperation op(dummy_cluster_context, + BucketAndNodes(makeDocumentBucket(document::BucketId(16, 1)), + toVector<uint16_t>(1, 3))); + // In node target set + EXPECT_TRUE(op.shouldBlockThisOperation(api::MessageType::PUT_ID, 1, 120)); + EXPECT_TRUE(op.shouldBlockThisOperation(api::MessageType::PUT_ID, 3, 120)); + // Not in node target set + EXPECT_FALSE(op.shouldBlockThisOperation(api::MessageType::PUT_ID, 0, 120)); + EXPECT_FALSE(op.shouldBlockThisOperation(api::MessageType::PUT_ID, 2, 120)); +} + } // storage::distributor diff --git a/storage/src/tests/distributor/statecheckerstest.cpp b/storage/src/tests/distributor/statecheckerstest.cpp index 25009156f18..f5531a134d0 100644 --- a/storage/src/tests/distributor/statecheckerstest.cpp +++ b/storage/src/tests/distributor/statecheckerstest.cpp @@ -38,14 +38,15 @@ struct StateCheckersTest : Test, DistributorStripeTestUtil { struct PendingMessage { uint32_t _msgType; + uint16_t _node; uint8_t _pri; - PendingMessage() : _msgType(UINT32_MAX), _pri(0) {} + constexpr PendingMessage() noexcept : _msgType(UINT32_MAX), _node(0), _pri(0) {} - PendingMessage(uint32_t msgType, uint8_t pri) - : _msgType(msgType), _pri(pri) {} + constexpr PendingMessage(uint32_t msgType, uint8_t pri) noexcept + : _msgType(msgType), _node(0), _pri(pri) {} - bool shouldCheck() const { return _msgType != UINT32_MAX; } + bool shouldCheck() const noexcept { return _msgType != UINT32_MAX; } }; void enableClusterState(const lib::ClusterState& systemState) { @@ -97,8 +98,7 @@ struct StateCheckersTest : Test, DistributorStripeTestUtil { IdealStateOperation::UP op(result.createOperation()); if (op.get()) { if (blocker.shouldCheck() - && op->shouldBlockThisOperation(blocker._msgType, - blocker._pri)) + && op->shouldBlockThisOperation(blocker._msgType, blocker._node, blocker._pri)) { return "BLOCKED"; } diff --git a/storage/src/tests/persistence/common/persistenceproviderwrapper.cpp b/storage/src/tests/persistence/common/persistenceproviderwrapper.cpp index a174d305c27..b3bd1c6a253 100644 --- a/storage/src/tests/persistence/common/persistenceproviderwrapper.cpp +++ b/storage/src/tests/persistence/common/persistenceproviderwrapper.cpp @@ -22,6 +22,15 @@ } \ } +#define CHECK_ERROR_ASYNC(className, failType, onError) \ + { \ + Guard guard(_lock); \ + if (_result.getErrorCode() != spi::Result::ErrorType::NONE && (_failureMask & (failType))) { \ + onError->onComplete(std::make_unique<className>(_result.getErrorCode(), _result.getErrorMessage())); \ + return; \ + } \ + } + namespace storage { namespace { @@ -87,46 +96,40 @@ PersistenceProviderWrapper::getBucketInfo(const spi::Bucket& bucket) const return _spi.getBucketInfo(bucket); } -spi::Result -PersistenceProviderWrapper::put(const spi::Bucket& bucket, spi::Timestamp timestamp, - document::Document::SP doc, spi::Context& context) +void +PersistenceProviderWrapper::putAsync(const spi::Bucket& bucket, spi::Timestamp timestamp, document::Document::SP doc, + spi::Context& context, spi::OperationComplete::UP onComplete) { LOG_SPI("put(" << bucket << ", " << timestamp << ", " << doc->getId() << ")"); - CHECK_ERROR(spi::Result, FAIL_PUT); - return _spi.put(bucket, timestamp, std::move(doc), context); + CHECK_ERROR_ASYNC(spi::Result, FAIL_PUT, onComplete); + _spi.putAsync(bucket, timestamp, std::move(doc), context, std::move(onComplete)); } -spi::RemoveResult -PersistenceProviderWrapper::remove(const spi::Bucket& bucket, - spi::Timestamp timestamp, - const spi::DocumentId& id, - spi::Context& context) +void +PersistenceProviderWrapper::removeAsync(const spi::Bucket& bucket, spi::Timestamp timestamp, const spi::DocumentId& id, + spi::Context& context, spi::OperationComplete::UP onComplete) { LOG_SPI("remove(" << bucket << ", " << timestamp << ", " << id << ")"); - CHECK_ERROR(spi::RemoveResult, FAIL_REMOVE); - return _spi.remove(bucket, timestamp, id, context); + CHECK_ERROR_ASYNC(spi::RemoveResult, FAIL_REMOVE, onComplete); + _spi.removeAsync(bucket, timestamp, id, context, std::move(onComplete)); } -spi::RemoveResult -PersistenceProviderWrapper::removeIfFound(const spi::Bucket& bucket, - spi::Timestamp timestamp, - const spi::DocumentId& id, - spi::Context& context) +void +PersistenceProviderWrapper::removeIfFoundAsync(const spi::Bucket& bucket, spi::Timestamp timestamp, const spi::DocumentId& id, + spi::Context& context, spi::OperationComplete::UP onComplete) { LOG_SPI("removeIfFound(" << bucket << ", " << timestamp << ", " << id << ")"); - CHECK_ERROR(spi::RemoveResult, FAIL_REMOVE_IF_FOUND); - return _spi.removeIfFound(bucket, timestamp, id, context); + CHECK_ERROR_ASYNC(spi::RemoveResult, FAIL_REMOVE_IF_FOUND, onComplete); + _spi.removeIfFoundAsync(bucket, timestamp, id, context, std::move(onComplete)); } -spi::UpdateResult -PersistenceProviderWrapper::update(const spi::Bucket& bucket, - spi::Timestamp timestamp, - document::DocumentUpdate::SP upd, - spi::Context& context) +void +PersistenceProviderWrapper::updateAsync(const spi::Bucket& bucket, spi::Timestamp timestamp, document::DocumentUpdate::SP upd, + spi::Context& context, spi::OperationComplete::UP onComplete) { LOG_SPI("update(" << bucket << ", " << timestamp << ", " << upd->getId() << ")"); - CHECK_ERROR(spi::UpdateResult, FAIL_UPDATE); - return _spi.update(bucket, timestamp, std::move(upd), context); + CHECK_ERROR_ASYNC(spi::UpdateResult, FAIL_UPDATE, onComplete); + _spi.updateAsync(bucket, timestamp, std::move(upd), context, std::move(onComplete)); } spi::GetResult @@ -172,13 +175,13 @@ PersistenceProviderWrapper::destroyIterator(spi::IteratorId iterId, return _spi.destroyIterator(iterId, context); } -spi::Result -PersistenceProviderWrapper::deleteBucket(const spi::Bucket& bucket, - spi::Context& context) +void +PersistenceProviderWrapper::deleteBucketAsync(const spi::Bucket& bucket, spi::Context& context, + spi::OperationComplete::UP operationComplete) { LOG_SPI("deleteBucket(" << bucket << ")"); - CHECK_ERROR(spi::Result, FAIL_DELETE_BUCKET); - return _spi.deleteBucket(bucket, context); + CHECK_ERROR_ASYNC(spi::Result, FAIL_DELETE_BUCKET, operationComplete); + _spi.deleteBucketAsync(bucket, context, std::move(operationComplete)); } spi::Result @@ -225,4 +228,26 @@ PersistenceProviderWrapper::removeEntry(const spi::Bucket& bucket, return _spi.removeEntry(bucket, timestamp, context); } +spi::Result +PersistenceProviderWrapper::initialize() { + return _spi.initialize(); +} + +spi::BucketIdListResult +PersistenceProviderWrapper::getModifiedBuckets(spi::PersistenceProvider::BucketSpace bucketSpace) const { + return _spi.getModifiedBuckets(bucketSpace); +} + +spi::Result +PersistenceProviderWrapper::setClusterState(spi::PersistenceProvider::BucketSpace bucketSpace, const spi::ClusterState &state) { + return _spi.setClusterState(bucketSpace, state); +} + +void +PersistenceProviderWrapper::setActiveStateAsync(const spi::Bucket &bucket, spi::BucketInfo::ActiveState state, + spi::OperationComplete::UP onComplete) +{ + _spi.setActiveStateAsync(bucket, state, std::move(onComplete)); +} + } diff --git a/storage/src/tests/persistence/common/persistenceproviderwrapper.h b/storage/src/tests/persistence/common/persistenceproviderwrapper.h index d90fa7b2eaa..c6628814dba 100644 --- a/storage/src/tests/persistence/common/persistenceproviderwrapper.h +++ b/storage/src/tests/persistence/common/persistenceproviderwrapper.h @@ -21,7 +21,7 @@ namespace storage { -class PersistenceProviderWrapper : public spi::AbstractPersistenceProvider +class PersistenceProviderWrapper : public spi::PersistenceProvider { public: enum OPERATION_FAILURE_FLAGS @@ -47,11 +47,11 @@ public: // TODO: add more as needed }; private: - spi::PersistenceProvider& _spi; - spi::Result _result; - mutable std::mutex _lock; + spi::PersistenceProvider& _spi; + spi::Result _result; + mutable std::mutex _lock; mutable std::vector<std::string> _log; - uint32_t _failureMask; + uint32_t _failureMask; using Guard = std::lock_guard<std::mutex>; public: PersistenceProviderWrapper(spi::PersistenceProvider& spi); @@ -88,13 +88,21 @@ public: _log.clear(); } + spi::Result initialize() override; + spi::BucketIdListResult getModifiedBuckets(BucketSpace bucketSpace) const override; + + spi::Result setClusterState(BucketSpace bucketSpace, const spi::ClusterState &state) override; + + void setActiveStateAsync(const spi::Bucket &bucket, spi::BucketInfo::ActiveState state, + spi::OperationComplete::UP up) override; + spi::Result createBucket(const spi::Bucket&, spi::Context&) override; spi::BucketIdListResult listBuckets(BucketSpace bucketSpace) const override; spi::BucketInfoResult getBucketInfo(const spi::Bucket&) const override; - spi::Result put(const spi::Bucket&, spi::Timestamp, spi::DocumentSP, spi::Context&) override; - spi::RemoveResult remove(const spi::Bucket&, spi::Timestamp, const spi::DocumentId&, spi::Context&) override; - spi::RemoveResult removeIfFound(const spi::Bucket&, spi::Timestamp, const spi::DocumentId&, spi::Context&) override; - spi::UpdateResult update(const spi::Bucket&, spi::Timestamp, spi::DocumentUpdateSP, spi::Context&) override; + void putAsync(const spi::Bucket&, spi::Timestamp, spi::DocumentSP, spi::Context&, spi::OperationComplete::UP) override; + void removeAsync(const spi::Bucket&, spi::Timestamp, const spi::DocumentId&, spi::Context&, spi::OperationComplete::UP) override; + void removeIfFoundAsync(const spi::Bucket&, spi::Timestamp, const spi::DocumentId&, spi::Context&, spi::OperationComplete::UP) override; + void updateAsync(const spi::Bucket&, spi::Timestamp, spi::DocumentUpdateSP, spi::Context&, spi::OperationComplete::UP) override; spi::GetResult get(const spi::Bucket&, const document::FieldSet&, const spi::DocumentId&, spi::Context&) const override; spi::CreateIteratorResult @@ -103,7 +111,7 @@ public: spi::IterateResult iterate(spi::IteratorId, uint64_t maxByteSize, spi::Context&) const override; spi::Result destroyIterator(spi::IteratorId, spi::Context&) override; - spi::Result deleteBucket(const spi::Bucket&, spi::Context&) override; + void deleteBucketAsync(const spi::Bucket&, spi::Context&, spi::OperationComplete::UP) override; spi::Result split(const spi::Bucket& source, const spi::Bucket& target1, const spi::Bucket& target2, spi::Context&) override; spi::Result join(const spi::Bucket& source1, const spi::Bucket& source2, diff --git a/storage/src/tests/persistence/filestorage/operationabortingtest.cpp b/storage/src/tests/persistence/filestorage/operationabortingtest.cpp index c51357cacd1..07d2b24d536 100644 --- a/storage/src/tests/persistence/filestorage/operationabortingtest.cpp +++ b/storage/src/tests/persistence/filestorage/operationabortingtest.cpp @@ -46,14 +46,15 @@ public: _deleteBucketInvocations(0) {} - spi::Result put(const spi::Bucket&, spi::Timestamp, document::Document::SP, spi::Context&) override + void + putAsync(const spi::Bucket&, spi::Timestamp, document::Document::SP, spi::Context&, spi::OperationComplete::UP onComplete) override { _queueBarrier.await(); // message abort stage with active opertion in disk queue std::this_thread::sleep_for(75ms); _completionBarrier.await(); // test finished - return spi::Result(); + onComplete->onComplete(std::make_unique<spi::Result>()); } spi::BucketInfoResult getBucketInfo(const spi::Bucket& bucket) const override { @@ -66,9 +67,10 @@ public: return PersistenceProviderWrapper::createBucket(bucket, ctx); } - spi::Result deleteBucket(const spi::Bucket& bucket, spi::Context& ctx) override { + void + deleteBucketAsync(const spi::Bucket& bucket, spi::Context& ctx, spi::OperationComplete::UP onComplete) override { ++_deleteBucketInvocations; - return PersistenceProviderWrapper::deleteBucket(bucket, ctx); + PersistenceProviderWrapper::deleteBucketAsync(bucket, ctx, std::move(onComplete)); } }; diff --git a/storage/src/tests/persistence/mergehandlertest.cpp b/storage/src/tests/persistence/mergehandlertest.cpp index fa989410137..60030004594 100644 --- a/storage/src/tests/persistence/mergehandlertest.cpp +++ b/storage/src/tests/persistence/mergehandlertest.cpp @@ -304,10 +304,10 @@ MergeHandlerTest::testApplyBucketDiffChain(bool midChain) EXPECT_FALSE(replySent.get()); LOG(debug, "Verifying that replying the diff sends on back"); - auto reply = std::make_unique<api::ApplyBucketDiffReply>(cmd2); + auto reply = std::make_shared<api::ApplyBucketDiffReply>(cmd2); MessageSenderStub stub; - handler.handleApplyBucketDiffReply(*reply, stub); + handler.handleApplyBucketDiffReply(*reply, stub, createTracker(reply, _bucket)); ASSERT_EQ(1, stub.replies.size()); replySent = stub.replies[0]; } @@ -353,12 +353,12 @@ TEST_F(MergeHandlerTest, master_message_flow) { ASSERT_EQ(2, messageKeeper()._msgs.size()); ASSERT_EQ(api::MessageType::APPLYBUCKETDIFF, messageKeeper()._msgs[1]->getType()); auto& cmd3 = dynamic_cast<api::ApplyBucketDiffCommand&>(*messageKeeper()._msgs[1]); - auto reply2 = std::make_unique<api::ApplyBucketDiffReply>(cmd3); + auto reply2 = std::make_shared<api::ApplyBucketDiffReply>(cmd3); ASSERT_EQ(1, reply2->getDiff().size()); reply2->getDiff()[0]._entry._hasMask |= 2u; MessageSenderStub stub; - handler.handleApplyBucketDiffReply(*reply2, stub); + handler.handleApplyBucketDiffReply(*reply2, stub, createTracker(reply2, _bucket)); ASSERT_EQ(1, stub.replies.size()); @@ -470,9 +470,9 @@ TEST_F(MergeHandlerTest, chunked_apply_bucket_diff) { } } - auto applyBucketDiffReply = std::make_unique<api::ApplyBucketDiffReply>(*applyBucketDiffCmd); + auto applyBucketDiffReply = std::make_shared<api::ApplyBucketDiffReply>(*applyBucketDiffCmd); { - handler.handleApplyBucketDiffReply(*applyBucketDiffReply, messageKeeper()); + handler.handleApplyBucketDiffReply(*applyBucketDiffReply, messageKeeper(), createTracker(applyBucketDiffReply, _bucket)); if (!messageKeeper()._msgs.empty()) { ASSERT_FALSE(reply.get()); @@ -672,10 +672,10 @@ TEST_F(MergeHandlerTest, merge_progress_safe_guard) { handler.handleGetBucketDiffReply(*getBucketDiffReply, messageKeeper()); auto applyBucketDiffCmd = fetchSingleMessage<api::ApplyBucketDiffCommand>(); - auto applyBucketDiffReply = std::make_unique<api::ApplyBucketDiffReply>(*applyBucketDiffCmd); + auto applyBucketDiffReply = std::make_shared<api::ApplyBucketDiffReply>(*applyBucketDiffCmd); MessageSenderStub stub; - handler.handleApplyBucketDiffReply(*applyBucketDiffReply, stub); + handler.handleApplyBucketDiffReply(*applyBucketDiffReply, stub, createTracker(applyBucketDiffReply, _bucket)); ASSERT_EQ(1, stub.replies.size()); @@ -699,14 +699,14 @@ TEST_F(MergeHandlerTest, safe_guard_not_invoked_when_has_mask_changes) { handler.handleGetBucketDiffReply(*getBucketDiffReply, messageKeeper()); auto applyBucketDiffCmd = fetchSingleMessage<api::ApplyBucketDiffCommand>(); - auto applyBucketDiffReply = std::make_unique<api::ApplyBucketDiffReply>(*applyBucketDiffCmd); + auto applyBucketDiffReply = std::make_shared<api::ApplyBucketDiffReply>(*applyBucketDiffCmd); ASSERT_FALSE(applyBucketDiffReply->getDiff().empty()); // Change a hasMask to indicate something changed during merging. applyBucketDiffReply->getDiff()[0]._entry._hasMask = 0x5; MessageSenderStub stub; LOG(debug, "sending apply bucket diff reply"); - handler.handleApplyBucketDiffReply(*applyBucketDiffReply, stub); + handler.handleApplyBucketDiffReply(*applyBucketDiffReply, stub, createTracker(applyBucketDiffReply, _bucket)); ASSERT_EQ(1, stub.commands.size()); @@ -1012,10 +1012,10 @@ MergeHandlerTest::HandleApplyBucketDiffReplyInvoker::invoke( spi::Context&) { (void) test; - api::ApplyBucketDiffReply reply(*_applyCmd); - test.fillDummyApplyDiff(reply.getDiff()); + auto reply = std::make_shared<api::ApplyBucketDiffReply>(*_applyCmd); + test.fillDummyApplyDiff(reply->getDiff()); _stub.clear(); - handler.handleApplyBucketDiffReply(reply, _stub); + handler.handleApplyBucketDiffReply(*reply, _stub, test.createTracker(reply, test._bucket)); } std::string @@ -1275,8 +1275,8 @@ TEST_F(MergeHandlerTest, partially_filled_apply_bucket_diff_reply) EXPECT_EQ(1, cmd2.getAddress()->getIndex()); EXPECT_EQ(1234, cmd2.getSourceIndex()); EXPECT_TRUE(getEnv()._fileStorHandler.isMerging(_bucket)); - auto &s = getEnv()._fileStorHandler.editMergeStatus(_bucket); - EXPECT_EQ((NodeList{{0, false}, {1, false}, {2, true}, {3, true}, {4, true}}), s.nodeList); + auto s = getEnv()._fileStorHandler.editMergeStatus(_bucket); + EXPECT_EQ((NodeList{{0, false}, {1, false}, {2, true}, {3, true}, {4, true}}), s->nodeList); baseline_diff_size = cmd2.getDiff().size(); auto reply = std::make_unique<api::GetBucketDiffReply>(cmd2); auto &diff = reply->getDiff(); @@ -1292,16 +1292,16 @@ TEST_F(MergeHandlerTest, partially_filled_apply_bucket_diff_reply) { LOG(debug, "checking first ApplyBucketDiff command"); EXPECT_TRUE(getEnv()._fileStorHandler.isMerging(_bucket)); - auto &s = getEnv()._fileStorHandler.editMergeStatus(_bucket); + auto s = getEnv()._fileStorHandler.editMergeStatus(_bucket); // Node 4 has been eliminated before the first ApplyBucketDiff command - EXPECT_EQ((NodeList{{0, false}, {1, false}, {2, true}, {3, true}}), s.nodeList); - EXPECT_EQ(baseline_diff_size + 2u, s.diff.size()); - EXPECT_EQ(EntryCheck(20000, 24u), s.diff[baseline_diff_size]); - EXPECT_EQ(EntryCheck(20100, 24u), s.diff[baseline_diff_size + 1]); + EXPECT_EQ((NodeList{{0, false}, {1, false}, {2, true}, {3, true}}), s->nodeList); + EXPECT_EQ(baseline_diff_size + 2u, s->diff.size()); + EXPECT_EQ(EntryCheck(20000, 24u), s->diff[baseline_diff_size]); + EXPECT_EQ(EntryCheck(20100, 24u), s->diff[baseline_diff_size + 1]); auto& cmd3 = dynamic_cast<api::ApplyBucketDiffCommand&>(*messageKeeper()._msgs[1]); // ApplyBucketDiffCommand has a shorter node list, node 2 is not present EXPECT_EQ((NodeList{{0, false}, {1, false}, {3, true}}), cmd3.getNodes()); - auto reply = std::make_unique<api::ApplyBucketDiffReply>(cmd3); + auto reply = std::make_shared<api::ApplyBucketDiffReply>(cmd3); auto& diff = reply->getDiff(); EXPECT_EQ(2u, diff.size()); EXPECT_EQ(EntryCheck(20000u, 4u), diff[0]._entry); @@ -1312,7 +1312,7 @@ TEST_F(MergeHandlerTest, partially_filled_apply_bucket_diff_reply) */ fill_entry(diff[0], *doc1, getEnv().getDocumentTypeRepo()); diff[0]._entry._hasMask |= 2u; // Simulate diff entry having been applied on node 1. - handler.handleApplyBucketDiffReply(*reply, messageKeeper()); + handler.handleApplyBucketDiffReply(*reply, messageKeeper(), createTracker(reply, _bucket)); LOG(debug, "handled first ApplyBucketDiffReply"); } ASSERT_EQ(3u, messageKeeper()._msgs.size()); @@ -1320,19 +1320,19 @@ TEST_F(MergeHandlerTest, partially_filled_apply_bucket_diff_reply) { LOG(debug, "checking second ApplyBucketDiff command"); EXPECT_TRUE(getEnv()._fileStorHandler.isMerging(_bucket)); - auto &s = getEnv()._fileStorHandler.editMergeStatus(_bucket); - EXPECT_EQ((NodeList{{0, false}, {1, false}, {2, true}, {3, true}}), s.nodeList); - EXPECT_EQ(baseline_diff_size + 1u, s.diff.size()); - EXPECT_EQ(EntryCheck(20100, 24u), s.diff[baseline_diff_size]); + auto s = getEnv()._fileStorHandler.editMergeStatus(_bucket); + EXPECT_EQ((NodeList{{0, false}, {1, false}, {2, true}, {3, true}}), s->nodeList); + EXPECT_EQ(baseline_diff_size + 1u, s->diff.size()); + EXPECT_EQ(EntryCheck(20100, 24u), s->diff[baseline_diff_size]); auto& cmd4 = dynamic_cast<api::ApplyBucketDiffCommand&>(*messageKeeper()._msgs[2]); EXPECT_EQ((NodeList{{0, false}, {1, false}, {3, true}}), cmd4.getNodes()); - auto reply = std::make_unique<api::ApplyBucketDiffReply>(cmd4); + auto reply = std::make_shared<api::ApplyBucketDiffReply>(cmd4); auto& diff = reply->getDiff(); EXPECT_EQ(1u, diff.size()); EXPECT_EQ(EntryCheck(20100u, 4u), diff[0]._entry); // Simulate that node 3 somehow lost doc2 when trying to fill diff entry. diff[0]._entry._hasMask &= ~4u; - handler.handleApplyBucketDiffReply(*reply, messageKeeper()); + handler.handleApplyBucketDiffReply(*reply, messageKeeper(), createTracker(reply, _bucket)); LOG(debug, "handled second ApplyBucketDiffReply"); } ASSERT_EQ(4u, messageKeeper()._msgs.size()); @@ -1340,21 +1340,21 @@ TEST_F(MergeHandlerTest, partially_filled_apply_bucket_diff_reply) { LOG(debug, "checking third ApplyBucketDiff command"); EXPECT_TRUE(getEnv()._fileStorHandler.isMerging(_bucket)); - auto &s = getEnv()._fileStorHandler.editMergeStatus(_bucket); + auto s = getEnv()._fileStorHandler.editMergeStatus(_bucket); // Nodes 3 and 2 have been eliminated before the third ApplyBucketDiff command - EXPECT_EQ((NodeList{{0, false}, {1, false}}), s.nodeList); - EXPECT_EQ(baseline_diff_size + 1u, s.diff.size()); - EXPECT_EQ(EntryCheck(20100, 16u), s.diff[baseline_diff_size]); + EXPECT_EQ((NodeList{{0, false}, {1, false}}), s->nodeList); + EXPECT_EQ(baseline_diff_size + 1u, s->diff.size()); + EXPECT_EQ(EntryCheck(20100, 16u), s->diff[baseline_diff_size]); auto& cmd5 = dynamic_cast<api::ApplyBucketDiffCommand&>(*messageKeeper()._msgs[3]); EXPECT_EQ((NodeList{{0, false}, {1, false}}), cmd5.getNodes()); - auto reply = std::make_unique<api::ApplyBucketDiffReply>(cmd5); + auto reply = std::make_shared<api::ApplyBucketDiffReply>(cmd5); auto& diff = reply->getDiff(); EXPECT_EQ(baseline_diff_size, diff.size()); for (auto& e : diff) { EXPECT_EQ(1u, e._entry._hasMask); e._entry._hasMask |= 2u; } - handler.handleApplyBucketDiffReply(*reply, messageKeeper()); + handler.handleApplyBucketDiffReply(*reply, messageKeeper(), createTracker(reply, _bucket)); LOG(debug, "handled third ApplyBucketDiffReply"); } ASSERT_EQ(5u, messageKeeper()._msgs.size()); @@ -1362,19 +1362,19 @@ TEST_F(MergeHandlerTest, partially_filled_apply_bucket_diff_reply) { LOG(debug, "checking fourth ApplyBucketDiff command"); EXPECT_TRUE(getEnv()._fileStorHandler.isMerging(_bucket)); - auto &s = getEnv()._fileStorHandler.editMergeStatus(_bucket); + auto s = getEnv()._fileStorHandler.editMergeStatus(_bucket); // All nodes in use again due to failure to fill diff entry for doc2 - EXPECT_EQ((NodeList{{0, false}, {1, false}, {2, true}, {3, true}, {4, true}}), s.nodeList); - EXPECT_EQ(1u, s.diff.size()); - EXPECT_EQ(EntryCheck(20100, 16u), s.diff[0]); + EXPECT_EQ((NodeList{{0, false}, {1, false}, {2, true}, {3, true}, {4, true}}), s->nodeList); + EXPECT_EQ(1u, s->diff.size()); + EXPECT_EQ(EntryCheck(20100, 16u), s->diff[0]); auto& cmd6 = dynamic_cast<api::ApplyBucketDiffCommand&>(*messageKeeper()._msgs[4]); EXPECT_EQ((NodeList{{0, false}, {1, false}, {4, true}}), cmd6.getNodes()); - auto reply = std::make_unique<api::ApplyBucketDiffReply>(cmd6); + auto reply = std::make_shared<api::ApplyBucketDiffReply>(cmd6); auto& diff = reply->getDiff(); EXPECT_EQ(1u, diff.size()); fill_entry(diff[0], *doc2, getEnv().getDocumentTypeRepo()); diff[0]._entry._hasMask |= 2u; - handler.handleApplyBucketDiffReply(*reply, messageKeeper()); + handler.handleApplyBucketDiffReply(*reply, messageKeeper(), createTracker(reply, _bucket)); LOG(debug, "handled fourth ApplyBucketDiffReply"); } ASSERT_EQ(6u, messageKeeper()._msgs.size()); diff --git a/storage/src/tests/storageserver/mergethrottlertest.cpp b/storage/src/tests/storageserver/mergethrottlertest.cpp index 568eee1e92c..5c192942521 100644 --- a/storage/src/tests/storageserver/mergethrottlertest.cpp +++ b/storage/src/tests/storageserver/mergethrottlertest.cpp @@ -1151,6 +1151,10 @@ TEST_F(MergeThrottlerTest, busy_returned_on_full_queue_for_merges_sent_from_dist size_t maxQueue = _throttlers[0]->getMaxQueueSize(); ASSERT_EQ(20, maxQueue); ASSERT_LT(maxPending, 100); + + EXPECT_EQ(_throttlers[0]->getMetrics().active_window_size.getLast(), 0); + EXPECT_EQ(_throttlers[0]->getMetrics().queueSize.getLast(), 0); + for (size_t i = 0; i < maxPending + maxQueue; ++i) { std::vector<MergeBucketCommand::Node> nodes({{0}, {1}, {2}}); // No chain set, i.e. merge command is freshly squeezed from a distributor. @@ -1162,6 +1166,7 @@ TEST_F(MergeThrottlerTest, busy_returned_on_full_queue_for_merges_sent_from_dist // Wait till we have maxPending replies and maxQueue queued _topLinks[0]->waitForMessages(maxPending, _messageWaitTime); waitUntilMergeQueueIs(*_throttlers[0], maxQueue, _messageWaitTime); + EXPECT_EQ(_throttlers[0]->getMetrics().active_window_size.getLast(), maxPending); EXPECT_EQ(maxQueue, _throttlers[0]->getMetrics().queueSize.getMaximum()); // Clear all forwarded merges diff --git a/storage/src/vespa/storage/distributor/idealstatemetricsset.cpp b/storage/src/vespa/storage/distributor/idealstatemetricsset.cpp index d3096e6864e..618e49c4238 100644 --- a/storage/src/vespa/storage/distributor/idealstatemetricsset.cpp +++ b/storage/src/vespa/storage/distributor/idealstatemetricsset.cpp @@ -39,13 +39,13 @@ MergeBucketMetricSet::MergeBucketMetricSet(const std::string& name, metrics::Met : OperationMetricSet(name, std::move(tags), description, owner), source_only_copy_changed("source_only_copy_changed", {{"logdefault"},{"yamasdefault"}}, - "The number of merge operations where source-only copy changed"), + "The number of merge operations where source-only copy changed", this), source_only_copy_delete_blocked("source_only_copy_delete_blocked", {{"logdefault"},{"yamasdefault"}}, - "The number of merge operations where delete of unchanged source-only copies was blocked"), + "The number of merge operations where delete of unchanged source-only copies was blocked", this), source_only_copy_delete_failed("source_only_copy_delete_failed", {{"logdefault"},{"yamasdefault"}}, - "The number of merge operations where delete of unchanged source-only copies failed") + "The number of merge operations where delete of unchanged source-only copies failed", this) { } diff --git a/storage/src/vespa/storage/distributor/operations/idealstate/garbagecollectionoperation.cpp b/storage/src/vespa/storage/distributor/operations/idealstate/garbagecollectionoperation.cpp index fbe1c142b09..7f66d1effd5 100644 --- a/storage/src/vespa/storage/distributor/operations/idealstate/garbagecollectionoperation.cpp +++ b/storage/src/vespa/storage/distributor/operations/idealstate/garbagecollectionoperation.cpp @@ -86,7 +86,7 @@ void GarbageCollectionOperation::update_gc_metrics() { } bool -GarbageCollectionOperation::shouldBlockThisOperation(uint32_t, uint8_t) const { +GarbageCollectionOperation::shouldBlockThisOperation(uint32_t, uint16_t, uint8_t) const { return true; } diff --git a/storage/src/vespa/storage/distributor/operations/idealstate/garbagecollectionoperation.h b/storage/src/vespa/storage/distributor/operations/idealstate/garbagecollectionoperation.h index 2e010a61bde..f51739242b7 100644 --- a/storage/src/vespa/storage/distributor/operations/idealstate/garbagecollectionoperation.h +++ b/storage/src/vespa/storage/distributor/operations/idealstate/garbagecollectionoperation.h @@ -21,7 +21,7 @@ public: void onReceive(DistributorStripeMessageSender& sender, const std::shared_ptr<api::StorageReply> &) override; const char* getName() const override { return "garbagecollection"; }; Type getType() const override { return GARBAGE_COLLECTION; } - bool shouldBlockThisOperation(uint32_t, uint8_t) const override; + bool shouldBlockThisOperation(uint32_t, uint16_t, uint8_t) const override; protected: MessageTracker _tracker; diff --git a/storage/src/vespa/storage/distributor/operations/idealstate/idealstateoperation.cpp b/storage/src/vespa/storage/distributor/operations/idealstate/idealstateoperation.cpp index b1231fafcd9..744b24b593e 100644 --- a/storage/src/vespa/storage/distributor/operations/idealstate/idealstateoperation.cpp +++ b/storage/src/vespa/storage/distributor/operations/idealstate/idealstateoperation.cpp @@ -138,8 +138,7 @@ public: bool check(uint32_t messageType, uint16_t node, uint8_t priority) override { - (void) node; - if (op.shouldBlockThisOperation(messageType, priority)) { + if (op.shouldBlockThisOperation(messageType, node, priority)) { blocked = true; return false; } @@ -232,6 +231,7 @@ IdealStateOperation::toString() const bool IdealStateOperation::shouldBlockThisOperation(uint32_t messageType, + [[maybe_unused]] uint16_t node, uint8_t) const { for (uint32_t i = 0; MAINTENANCE_MESSAGE_TYPES[i] != 0; ++i) { diff --git a/storage/src/vespa/storage/distributor/operations/idealstate/idealstateoperation.h b/storage/src/vespa/storage/distributor/operations/idealstate/idealstateoperation.h index d4dc4e405df..f8f35afe821 100644 --- a/storage/src/vespa/storage/distributor/operations/idealstate/idealstateoperation.h +++ b/storage/src/vespa/storage/distributor/operations/idealstate/idealstateoperation.h @@ -217,7 +217,7 @@ public: /** * Should return true if the given message type should block this operation. */ - virtual bool shouldBlockThisOperation(uint32_t messageType, uint8_t priority) const; + virtual bool shouldBlockThisOperation(uint32_t messageType, uint16_t node, uint8_t priority) const; protected: friend struct IdealStateManagerTest; diff --git a/storage/src/vespa/storage/distributor/operations/idealstate/mergeoperation.cpp b/storage/src/vespa/storage/distributor/operations/idealstate/mergeoperation.cpp index 7cfe4172b2c..f951a880e5d 100644 --- a/storage/src/vespa/storage/distributor/operations/idealstate/mergeoperation.cpp +++ b/storage/src/vespa/storage/distributor/operations/idealstate/mergeoperation.cpp @@ -329,14 +329,14 @@ constexpr std::array<uint32_t, 7> WRITE_FEED_MESSAGE_TYPES {{ } -bool MergeOperation::shouldBlockThisOperation(uint32_t messageType, uint8_t pri) const { +bool MergeOperation::shouldBlockThisOperation(uint32_t messageType, uint16_t node, uint8_t pri) const { for (auto blocking_type : WRITE_FEED_MESSAGE_TYPES) { if (messageType == blocking_type) { return true; } } - return IdealStateOperation::shouldBlockThisOperation(messageType, pri); + return IdealStateOperation::shouldBlockThisOperation(messageType, node, pri); } bool MergeOperation::isBlocked(const DistributorStripeOperationContext& ctx, diff --git a/storage/src/vespa/storage/distributor/operations/idealstate/mergeoperation.h b/storage/src/vespa/storage/distributor/operations/idealstate/mergeoperation.h index 1bca1f7389f..832c0f99681 100644 --- a/storage/src/vespa/storage/distributor/operations/idealstate/mergeoperation.h +++ b/storage/src/vespa/storage/distributor/operations/idealstate/mergeoperation.h @@ -48,7 +48,7 @@ public: const document::BucketId&, MergeLimiter&, std::vector<MergeMetaData>&); - bool shouldBlockThisOperation(uint32_t messageType, uint8_t pri) const override; + bool shouldBlockThisOperation(uint32_t messageType, uint16_t node, uint8_t pri) const override; bool isBlocked(const DistributorStripeOperationContext& ctx, const OperationSequencer&) const override; private: static void addIdealNodes( diff --git a/storage/src/vespa/storage/distributor/operations/idealstate/removebucketoperation.cpp b/storage/src/vespa/storage/distributor/operations/idealstate/removebucketoperation.cpp index 9a57722dc7e..25cae5b9979 100644 --- a/storage/src/vespa/storage/distributor/operations/idealstate/removebucketoperation.cpp +++ b/storage/src/vespa/storage/distributor/operations/idealstate/removebucketoperation.cpp @@ -61,8 +61,7 @@ RemoveBucketOperation::onStart(DistributorStripeMessageSender& sender) bool RemoveBucketOperation::onReceiveInternal(const std::shared_ptr<api::StorageReply> &msg) { - api::DeleteBucketReply* rep = - dynamic_cast<api::DeleteBucketReply*>(msg.get()); + auto* rep = dynamic_cast<api::DeleteBucketReply*>(msg.get()); uint16_t node = _tracker.handleReply(*rep); @@ -112,8 +111,15 @@ RemoveBucketOperation::onReceive(DistributorStripeMessageSender&, const std::sha } bool -RemoveBucketOperation::shouldBlockThisOperation(uint32_t, uint8_t) const +RemoveBucketOperation::shouldBlockThisOperation(uint32_t, uint16_t target_node, uint8_t) const { - return true; + // Number of nodes is expected to be 1 in the vastly common case (and a highly bounded + // number in the worst case), so a simple linear scan suffices. + for (uint16_t node : getNodes()) { + if (target_node == node) { + return true; + } + } + return false; } diff --git a/storage/src/vespa/storage/distributor/operations/idealstate/removebucketoperation.h b/storage/src/vespa/storage/distributor/operations/idealstate/removebucketoperation.h index a0d496f948a..5e0922d5685 100644 --- a/storage/src/vespa/storage/distributor/operations/idealstate/removebucketoperation.h +++ b/storage/src/vespa/storage/distributor/operations/idealstate/removebucketoperation.h @@ -30,7 +30,7 @@ public: void onReceive(DistributorStripeMessageSender& sender, const std::shared_ptr<api::StorageReply> &) override; const char* getName() const override { return "remove"; }; Type getType() const override { return DELETE_BUCKET; } - bool shouldBlockThisOperation(uint32_t, uint8_t) const override; + bool shouldBlockThisOperation(uint32_t, uint16_t, uint8_t) const override; protected: MessageTracker _tracker; }; diff --git a/storage/src/vespa/storage/distributor/operations/idealstate/splitoperation.cpp b/storage/src/vespa/storage/distributor/operations/idealstate/splitoperation.cpp index 649503cf0f5..6f3924535ef 100644 --- a/storage/src/vespa/storage/distributor/operations/idealstate/splitoperation.cpp +++ b/storage/src/vespa/storage/distributor/operations/idealstate/splitoperation.cpp @@ -150,6 +150,7 @@ SplitOperation::isBlocked(const DistributorStripeOperationContext& ctx, const Op bool SplitOperation::shouldBlockThisOperation(uint32_t msgType, + [[maybe_unused]] uint16_t node, uint8_t pri) const { if (msgType == api::MessageType::SPLITBUCKET_ID && _priority >= pri) { diff --git a/storage/src/vespa/storage/distributor/operations/idealstate/splitoperation.h b/storage/src/vespa/storage/distributor/operations/idealstate/splitoperation.h index ee957309088..6a268155fc8 100644 --- a/storage/src/vespa/storage/distributor/operations/idealstate/splitoperation.h +++ b/storage/src/vespa/storage/distributor/operations/idealstate/splitoperation.h @@ -21,7 +21,7 @@ public: const char* getName() const override { return "split"; }; Type getType() const override { return SPLIT_BUCKET; } bool isBlocked(const DistributorStripeOperationContext&, const OperationSequencer&) const override; - bool shouldBlockThisOperation(uint32_t, uint8_t) const override; + bool shouldBlockThisOperation(uint32_t, uint16_t, uint8_t) const override; protected: MessageTracker _tracker; diff --git a/storage/src/vespa/storage/persistence/asynchandler.cpp b/storage/src/vespa/storage/persistence/asynchandler.cpp index 4a28e650fac..d150f5600e5 100644 --- a/storage/src/vespa/storage/persistence/asynchandler.cpp +++ b/storage/src/vespa/storage/persistence/asynchandler.cpp @@ -3,11 +3,16 @@ #include "asynchandler.h" #include "persistenceutil.h" #include "testandsethelper.h" +#include "bucketownershipnotifier.h" #include <vespa/persistence/spi/persistenceprovider.h> +#include <vespa/storageapi/message/bucket.h> #include <vespa/document/update/documentupdate.h> #include <vespa/vespalib/util/isequencedtaskexecutor.h> #include <vespa/vespalib/util/destructor_callbacks.h> +#include <vespa/log/log.h> +LOG_SETUP(".storage.persistence.asynchandler"); + namespace storage { namespace { @@ -86,12 +91,26 @@ private: vespalib::ISequencedTaskExecutor::ExecutorId _executorId; }; +bool +bucketStatesAreSemanticallyEqual(const api::BucketInfo& a, const api::BucketInfo& b) { + // Don't check document sizes, as background moving of documents in Proton + // may trigger a change in size without any mutations taking place. This will + // only take place when a document being moved was fed _prior_ to the change + // where Proton starts reporting actual document sizes, and will eventually + // converge to a stable value. But for now, ignore it to prevent false positive + // error logs and non-deleted buckets. + return ((a.getChecksum() == b.getChecksum()) && (a.getDocumentCount() == b.getDocumentCount())); } + +} + AsyncHandler::AsyncHandler(const PersistenceUtil & env, spi::PersistenceProvider & spi, + BucketOwnershipNotifier &bucketOwnershipNotifier, vespalib::ISequencedTaskExecutor & executor, const document::BucketIdFactory & bucketIdFactory) : _env(env), _spi(spi), + _bucketOwnershipNotifier(bucketOwnershipNotifier), _sequencedExecutor(executor), _bucketIdFactory(bucketIdFactory) {} @@ -135,6 +154,79 @@ AsyncHandler::handlePut(api::PutCommand& cmd, MessageTracker::UP trackerUP) cons } MessageTracker::UP +AsyncHandler::handleDeleteBucket(api::DeleteBucketCommand& cmd, MessageTracker::UP tracker) const +{ + tracker->setMetric(_env._metrics.deleteBuckets); + LOG(debug, "DeletingBucket(%s)", cmd.getBucketId().toString().c_str()); + if (_env._fileStorHandler.isMerging(cmd.getBucket())) { + _env._fileStorHandler.clearMergeStatus(cmd.getBucket(), + api::ReturnCode(api::ReturnCode::ABORTED, "Bucket was deleted during the merge")); + } + spi::Bucket bucket(cmd.getBucket()); + if (!checkProviderBucketInfoMatches(bucket, cmd.getBucketInfo())) { + return tracker; + } + + auto task = makeResultTask([this, tracker = std::move(tracker), bucket=cmd.getBucket()](spi::Result::UP ignored) { + // TODO Even if an non OK response can not be handled sanely we might probably log a message, or increment a metric + (void) ignored; + StorBucketDatabase &db(_env.getBucketDatabase(bucket.getBucketSpace())); + StorBucketDatabase::WrappedEntry entry = db.get(bucket.getBucketId(), "onDeleteBucket"); + if (entry.exist() && entry->getMetaCount() > 0) { + LOG(debug, "onDeleteBucket(%s): Bucket DB entry existed. Likely " + "active operation when delete bucket was queued. " + "Updating bucket database to keep it in sync with file. " + "Cannot delete bucket from bucket database at this " + "point, as it can have been intentionally recreated " + "after delete bucket had been sent", + bucket.getBucketId().toString().c_str()); + api::BucketInfo info(0, 0, 0); + // Only set document counts/size; retain ready/active state. + info.setReady(entry->getBucketInfo().isReady()); + info.setActive(entry->getBucketInfo().isActive()); + + entry->setBucketInfo(info); + entry.write(); + } + tracker->sendReply(); + }); + _spi.deleteBucketAsync(bucket, tracker->context(), std::make_unique<ResultTaskOperationDone>(_sequencedExecutor, cmd.getBucketId(), std::move(task))); + return tracker; +} + +MessageTracker::UP +AsyncHandler::handleSetBucketState(api::SetBucketStateCommand& cmd, MessageTracker::UP trackerUP) const +{ + trackerUP->setMetric(_env._metrics.setBucketStates); + + //LOG(debug, "handleSetBucketState(): %s", cmd.toString().c_str()); + spi::Bucket bucket(cmd.getBucket()); + bool shouldBeActive(cmd.getState() == api::SetBucketStateCommand::ACTIVE); + spi::BucketInfo::ActiveState newState(shouldBeActive ? spi::BucketInfo::ACTIVE : spi::BucketInfo::NOT_ACTIVE); + + auto task = makeResultTask([this, &cmd, newState, tracker = std::move(trackerUP), bucket, + notifyGuard = std::make_unique<NotificationGuard>(_bucketOwnershipNotifier)](spi::Result::UP response) mutable { + if (tracker->checkForError(*response)) { + StorBucketDatabase &db(_env.getBucketDatabase(bucket.getBucketSpace())); + StorBucketDatabase::WrappedEntry entry = db.get(bucket.getBucketId(),"handleSetBucketState"); + if (entry.exist()) { + entry->info.setActive(newState == spi::BucketInfo::ACTIVE); + notifyGuard->notifyIfOwnershipChanged(cmd.getBucket(), cmd.getSourceIndex(), entry->info); + entry.write(); + } else { + LOG(warning, "Got OK setCurrentState result from provider for %s, " + "but bucket has disappeared from service layer database", + cmd.getBucketId().toString().c_str()); + } + tracker->setReply(std::make_shared<api::SetBucketStateReply>(cmd)); + } + tracker->sendReply(); + }); + _spi.setActiveStateAsync(bucket, newState, std::make_unique<ResultTaskOperationDone>(_sequencedExecutor, cmd.getBucketId(), std::move(task))); + return trackerUP; +} + +MessageTracker::UP AsyncHandler::handleUpdate(api::UpdateCommand& cmd, MessageTracker::UP trackerUP) const { MessageTracker & tracker = *trackerUP; @@ -233,4 +325,31 @@ AsyncHandler::tasConditionMatches(const api::TestAndSetCommand & cmd, MessageTra return true; } +bool +AsyncHandler::checkProviderBucketInfoMatches(const spi::Bucket& bucket, const api::BucketInfo& info) const +{ + spi::BucketInfoResult result(_spi.getBucketInfo(bucket)); + if (result.hasError()) { + LOG(error, "getBucketInfo(%s) failed before deleting bucket; got error '%s'", + bucket.toString().c_str(), result.getErrorMessage().c_str()); + return false; + } + api::BucketInfo providerInfo(PersistenceUtil::convertBucketInfo(result.getBucketInfo())); + // Don't check meta fields or active/ready fields since these are not + // that important and ready may change under the hood in a race with + // getModifiedBuckets(). If bucket is empty it means it has already + // been deleted by a racing split/join. + if (!bucketStatesAreSemanticallyEqual(info, providerInfo) && !providerInfo.empty()) { + LOG(error, + "Service layer bucket database and provider out of sync before " + "deleting bucket %s! Service layer db had %s while provider says " + "bucket has %s. Deletion has been rejected to ensure data is not " + "lost, but bucket may remain out of sync until service has been " + "restarted.", + bucket.toString().c_str(), info.toString().c_str(), providerInfo.toString().c_str()); + return false; + } + return true; +} + } diff --git a/storage/src/vespa/storage/persistence/asynchandler.h b/storage/src/vespa/storage/persistence/asynchandler.h index 23f3605dca1..4f5c242570c 100644 --- a/storage/src/vespa/storage/persistence/asynchandler.h +++ b/storage/src/vespa/storage/persistence/asynchandler.h @@ -14,6 +14,7 @@ namespace spi { class Context; } class PersistenceUtil; +class BucketOwnershipNotifier; /** * Handle async operations that uses a sequenced executor. @@ -21,19 +22,23 @@ class PersistenceUtil; */ class AsyncHandler : public Types { public: - AsyncHandler(const PersistenceUtil&, spi::PersistenceProvider&, vespalib::ISequencedTaskExecutor & executor, - const document::BucketIdFactory & bucketIdFactory); + AsyncHandler(const PersistenceUtil&, spi::PersistenceProvider&, BucketOwnershipNotifier &, + vespalib::ISequencedTaskExecutor & executor, const document::BucketIdFactory & bucketIdFactory); MessageTrackerUP handlePut(api::PutCommand& cmd, MessageTrackerUP tracker) const; MessageTrackerUP handleRemove(api::RemoveCommand& cmd, MessageTrackerUP tracker) const; MessageTrackerUP handleUpdate(api::UpdateCommand& cmd, MessageTrackerUP tracker) const; MessageTrackerUP handleRunTask(RunTaskCommand & cmd, MessageTrackerUP tracker) const; + MessageTrackerUP handleSetBucketState(api::SetBucketStateCommand& cmd, MessageTrackerUP tracker) const; + MessageTrackerUP handleDeleteBucket(api::DeleteBucketCommand& cmd, MessageTrackerUP tracker) const; static bool is_async_message(api::MessageType::Id type_id) noexcept; private: + bool checkProviderBucketInfoMatches(const spi::Bucket&, const api::BucketInfo&) const; static bool tasConditionExists(const api::TestAndSetCommand & cmd); bool tasConditionMatches(const api::TestAndSetCommand & cmd, MessageTracker & tracker, spi::Context & context, bool missingDocumentImpliesMatch = false) const; const PersistenceUtil & _env; spi::PersistenceProvider & _spi; + BucketOwnershipNotifier & _bucketOwnershipNotifier; vespalib::ISequencedTaskExecutor & _sequencedExecutor; const document::BucketIdFactory & _bucketIdFactory; }; diff --git a/storage/src/vespa/storage/persistence/filestorage/filestorhandler.h b/storage/src/vespa/storage/persistence/filestorage/filestorhandler.h index 0dcb8539bff..70ed9845cb0 100644 --- a/storage/src/vespa/storage/persistence/filestorage/filestorhandler.h +++ b/storage/src/vespa/storage/persistence/filestorage/filestorhandler.h @@ -200,13 +200,13 @@ public: virtual void addMergeStatus(const document::Bucket&, std::shared_ptr<MergeStatus>) = 0; /** - * Returns the reference to the current merge status for the given bucket. + * Returns a shared pointer to the current merge status for the given bucket. * This allows unlocked access to an internal variable, so users should * first check that noone else is using it by calling isMerging() first. * * @param bucket The bucket to start merging. */ - virtual MergeStatus& editMergeStatus(const document::Bucket& bucket) = 0; + virtual std::shared_ptr<MergeStatus> editMergeStatus(const document::Bucket& bucket) = 0; /** * Returns true if the bucket is currently being merged on this node. diff --git a/storage/src/vespa/storage/persistence/filestorage/filestorhandlerimpl.cpp b/storage/src/vespa/storage/persistence/filestorage/filestorhandlerimpl.cpp index 62f9fa12a21..e395a7df9e0 100644 --- a/storage/src/vespa/storage/persistence/filestorage/filestorhandlerimpl.cpp +++ b/storage/src/vespa/storage/persistence/filestorage/filestorhandlerimpl.cpp @@ -86,7 +86,7 @@ FileStorHandlerImpl::addMergeStatus(const document::Bucket& bucket, std::shared_ _mergeStates[bucket] = status; } -MergeStatus& +std::shared_ptr<MergeStatus> FileStorHandlerImpl::editMergeStatus(const document::Bucket& bucket) { std::lock_guard mlock(_mergeStatesLock); @@ -94,7 +94,7 @@ FileStorHandlerImpl::editMergeStatus(const document::Bucket& bucket) if ( ! status ) { throw vespalib::IllegalStateException("No merge state exist for " + bucket.toString(), VESPA_STRLOC); } - return *status; + return status; } bool diff --git a/storage/src/vespa/storage/persistence/filestorage/filestorhandlerimpl.h b/storage/src/vespa/storage/persistence/filestorage/filestorhandlerimpl.h index a3dc316cdde..5f212b18a7f 100644 --- a/storage/src/vespa/storage/persistence/filestorage/filestorhandlerimpl.h +++ b/storage/src/vespa/storage/persistence/filestorage/filestorhandlerimpl.h @@ -218,7 +218,7 @@ public: } void addMergeStatus(const document::Bucket&, std::shared_ptr<MergeStatus>) override; - MergeStatus& editMergeStatus(const document::Bucket&) override; + std::shared_ptr<MergeStatus> editMergeStatus(const document::Bucket&) override; bool isMerging(const document::Bucket&) const override; void clearMergeStatus(const document::Bucket& bucket) override; void clearMergeStatus(const document::Bucket& bucket, const api::ReturnCode& code) override; diff --git a/storage/src/vespa/storage/persistence/filestorage/filestormanager.cpp b/storage/src/vespa/storage/persistence/filestorage/filestormanager.cpp index ddcab0f3659..2ffb827accf 100644 --- a/storage/src/vespa/storage/persistence/filestorage/filestormanager.cpp +++ b/storage/src/vespa/storage/persistence/filestorage/filestormanager.cpp @@ -10,7 +10,6 @@ #include <vespa/vdslib/state/clusterstate.h> #include <vespa/storage/common/hostreporter/hostinfo.h> #include <vespa/storage/common/messagebucket.h> -#include <vespa/storage/config/config-stor-server.h> #include <vespa/storage/persistence/bucketownershipnotifier.h> #include <vespa/storage/persistence/persistencethread.h> #include <vespa/storage/persistence/persistencehandler.h> diff --git a/storage/src/vespa/storage/persistence/mergehandler.cpp b/storage/src/vespa/storage/persistence/mergehandler.cpp index 7693156ae30..963fddd9fb5 100644 --- a/storage/src/vespa/storage/persistence/mergehandler.cpp +++ b/storage/src/vespa/storage/persistence/mergehandler.cpp @@ -23,13 +23,15 @@ namespace storage { MergeHandler::MergeHandler(PersistenceUtil& env, spi::PersistenceProvider& spi, const ClusterContext& cluster_context, const framework::Clock & clock, uint32_t maxChunkSize, - uint32_t commonMergeChainOptimalizationMinimumSize) + uint32_t commonMergeChainOptimalizationMinimumSize, + bool async_apply_bucket_diff) : _clock(clock), _cluster_context(cluster_context), _env(env), _spi(spi), _maxChunkSize(maxChunkSize), - _commonMergeChainOptimalizationMinimumSize(commonMergeChainOptimalizationMinimumSize) + _commonMergeChainOptimalizationMinimumSize(commonMergeChainOptimalizationMinimumSize), + _async_apply_bucket_diff(async_apply_bucket_diff) { } @@ -1136,49 +1138,49 @@ MergeHandler::handleGetBucketDiffReply(api::GetBucketDiffReply& reply, MessageSe return; } - MergeStatus& s = _env._fileStorHandler.editMergeStatus(bucket.getBucket()); - if (s.pendingId != reply.getMsgId()) { + auto s = _env._fileStorHandler.editMergeStatus(bucket.getBucket()); + if (s->pendingId != reply.getMsgId()) { LOG(warning, "Got GetBucketDiffReply for %s which had message " "id %" PRIu64 " when we expected %" PRIu64 ". Ignoring reply.", - bucket.toString().c_str(), reply.getMsgId(), s.pendingId); + bucket.toString().c_str(), reply.getMsgId(), s->pendingId); return; } api::StorageReply::SP replyToSend; bool clearState = true; try { - if (s.isFirstNode()) { + if (s->isFirstNode()) { if (reply.getResult().failed()) { // We failed, so we should reply to the pending message. - replyToSend = s.reply; + replyToSend = s->reply; } else { // If we didn't fail, reply should have good content // Sanity check for nodes assert(reply.getNodes().size() >= 2); // Get bucket diff should retrieve all info at once - assert(s.diff.size() == 0); - s.diff.insert(s.diff.end(), + assert(s->diff.size() == 0); + s->diff.insert(s->diff.end(), reply.getDiff().begin(), reply.getDiff().end()); - replyToSend = processBucketMerge(bucket, s, sender, s.context); + replyToSend = processBucketMerge(bucket, *s, sender, s->context); if (!replyToSend.get()) { // We have sent something on, and shouldn't reply now. clearState = false; } else { _env._metrics.merge_handler_metrics.mergeLatencyTotal.addValue( - s.startTime.getElapsedTimeAsDouble()); + s->startTime.getElapsedTimeAsDouble()); } } } else { // Exists in send on list, send on! - replyToSend = s.pendingGetDiff; + replyToSend = s->pendingGetDiff; LOG(spam, "Received GetBucketDiffReply for %s with diff of " "size %zu. Sending it on.", bucket.toString().c_str(), reply.getDiff().size()); - s.pendingGetDiff->getDiff().swap(reply.getDiff()); + s->pendingGetDiff->getDiff().swap(reply.getDiff()); } } catch (std::exception& e) { _env._fileStorHandler.clearMergeStatus( @@ -1282,8 +1284,9 @@ MergeHandler::handleApplyBucketDiff(api::ApplyBucketDiffCommand& cmd, MessageTra } void -MergeHandler::handleApplyBucketDiffReply(api::ApplyBucketDiffReply& reply,MessageSender& sender) const +MergeHandler::handleApplyBucketDiffReply(api::ApplyBucketDiffReply& reply,MessageSender& sender, MessageTracker::UP tracker) const { + (void) tracker; _env._metrics.applyBucketDiffReply.inc(); spi::Bucket bucket(reply.getBucket()); ApplyBucketDiffState async_results(*this, bucket); @@ -1296,11 +1299,11 @@ MergeHandler::handleApplyBucketDiffReply(api::ApplyBucketDiffReply& reply,Messag return; } - MergeStatus& s = _env._fileStorHandler.editMergeStatus(bucket.getBucket()); - if (s.pendingId != reply.getMsgId()) { + auto s = _env._fileStorHandler.editMergeStatus(bucket.getBucket()); + if (s->pendingId != reply.getMsgId()) { LOG(warning, "Got ApplyBucketDiffReply for %s which had message " "id %" PRIu64 " when we expected %" PRIu64 ". Ignoring reply.", - bucket.toString().c_str(), reply.getMsgId(), s.pendingId); + bucket.toString().c_str(), reply.getMsgId(), s->pendingId); return; } bool clearState = true; @@ -1315,12 +1318,12 @@ MergeHandler::handleApplyBucketDiffReply(api::ApplyBucketDiffReply& reply,Messag uint8_t index = findOwnIndex(reply.getNodes(), _env._nodeIndex); if (applyDiffNeedLocalData(diff, index, false)) { framework::MilliSecTimer startTime(_clock); - fetchLocalData(bucket, diff, index, s.context); + fetchLocalData(bucket, diff, index, s->context); _env._metrics.merge_handler_metrics.mergeDataReadLatency.addValue(startTime.getElapsedTimeAsDouble()); } if (applyDiffHasLocallyNeededData(diff, index)) { framework::MilliSecTimer startTime(_clock); - applyDiffLocally(bucket, diff, index, s.context, async_results); + applyDiffLocally(bucket, diff, index, s->context, async_results); _env._metrics.merge_handler_metrics.mergeDataWriteLatency.addValue(startTime.getElapsedTimeAsDouble()); async_results.check(); async_results.sync_bucket_info(); @@ -1332,50 +1335,50 @@ MergeHandler::handleApplyBucketDiffReply(api::ApplyBucketDiffReply& reply,Messag } } - if (s.isFirstNode()) { + if (s->isFirstNode()) { uint16_t hasMask = 0; for (uint16_t i=0; i<reply.getNodes().size(); ++i) { hasMask |= (1 << i); } - const size_t diffSizeBefore = s.diff.size(); - const bool altered = s.removeFromDiff(diff, hasMask, reply.getNodes()); + const size_t diffSizeBefore = s->diff.size(); + const bool altered = s->removeFromDiff(diff, hasMask, reply.getNodes()); if (reply.getResult().success() - && s.diff.size() == diffSizeBefore + && s->diff.size() == diffSizeBefore && !altered) { std::string msg( vespalib::make_string( "Completed merge cycle without fixing " "any entries (merge state diff at %zu entries)", - s.diff.size())); + s->diff.size())); returnCode = api::ReturnCode(api::ReturnCode::INTERNAL_FAILURE, msg); LOG(warning, "Got reply indicating merge cycle did not fix any entries: %s", reply.toString(true).c_str()); LOG(warning, "Merge state for which there was no progress across a full merge cycle: %s", - s.toString().c_str()); + s->toString().c_str()); } if (returnCode.failed()) { // Should reply now, since we failed. - replyToSend = s.reply; + replyToSend = s->reply; } else { - replyToSend = processBucketMerge(bucket, s, sender, s.context); + replyToSend = processBucketMerge(bucket, *s, sender, s->context); if (!replyToSend.get()) { // We have sent something on and shouldn't reply now. clearState = false; } else { - _env._metrics.merge_handler_metrics.mergeLatencyTotal.addValue(s.startTime.getElapsedTimeAsDouble()); + _env._metrics.merge_handler_metrics.mergeLatencyTotal.addValue(s->startTime.getElapsedTimeAsDouble()); } } } else { - replyToSend = s.pendingApplyDiff; + replyToSend = s->pendingApplyDiff; LOG(debug, "ApplyBucketDiff(%s) finished. Sending reply.", bucket.toString().c_str()); - s.pendingApplyDiff->getDiff().swap(reply.getDiff()); + s->pendingApplyDiff->getDiff().swap(reply.getDiff()); } } catch (std::exception& e) { _env._fileStorHandler.clearMergeStatus( diff --git a/storage/src/vespa/storage/persistence/mergehandler.h b/storage/src/vespa/storage/persistence/mergehandler.h index fa7e21dae78..0ff8f3c0ef8 100644 --- a/storage/src/vespa/storage/persistence/mergehandler.h +++ b/storage/src/vespa/storage/persistence/mergehandler.h @@ -45,7 +45,8 @@ public: MergeHandler(PersistenceUtil& env, spi::PersistenceProvider& spi, const ClusterContext& cluster_context, const framework::Clock & clock, uint32_t maxChunkSize = 4190208, - uint32_t commonMergeChainOptimalizationMinimumSize = 64); + uint32_t commonMergeChainOptimalizationMinimumSize = 64, + bool async_apply_bucket_diff = false); bool buildBucketInfoList( const spi::Bucket& bucket, @@ -68,7 +69,7 @@ public: MessageTrackerUP handleGetBucketDiff(api::GetBucketDiffCommand&, MessageTrackerUP) const; void handleGetBucketDiffReply(api::GetBucketDiffReply&, MessageSender&) const; MessageTrackerUP handleApplyBucketDiff(api::ApplyBucketDiffCommand&, MessageTrackerUP) const; - void handleApplyBucketDiffReply(api::ApplyBucketDiffReply&, MessageSender&) const; + void handleApplyBucketDiffReply(api::ApplyBucketDiffReply&, MessageSender&, MessageTrackerUP) const; private: const framework::Clock &_clock; @@ -77,6 +78,7 @@ private: spi::PersistenceProvider &_spi; const uint32_t _maxChunkSize; const uint32_t _commonMergeChainOptimalizationMinimumSize; + const bool _async_apply_bucket_diff; /** Returns a reply if merge is complete */ api::StorageReply::SP processBucketMerge(const spi::Bucket& bucket, diff --git a/storage/src/vespa/storage/persistence/persistencehandler.cpp b/storage/src/vespa/storage/persistence/persistencehandler.cpp index 5315d3ec0bc..1ef883fc810 100644 --- a/storage/src/vespa/storage/persistence/persistencehandler.cpp +++ b/storage/src/vespa/storage/persistence/persistencehandler.cpp @@ -19,8 +19,9 @@ PersistenceHandler::PersistenceHandler(vespalib::ISequencedTaskExecutor & sequen _processAllHandler(_env, provider), _mergeHandler(_env, provider, component.cluster_context(), _clock, cfg.bucketMergeChunkSize, - cfg.commonMergeChainOptimalizationMinimumSize), - _asyncHandler(_env, provider, sequencedExecutor, component.getBucketIdFactory()), + cfg.commonMergeChainOptimalizationMinimumSize, + cfg.asyncApplyBucketDiff), + _asyncHandler(_env, provider, bucketOwnershipNotifier, sequencedExecutor, component.getBucketIdFactory()), _splitJoinHandler(_env, provider, bucketOwnershipNotifier, cfg.enableMultibitSplitOptimalization), _simpleHandler(_env, provider) { @@ -45,7 +46,7 @@ PersistenceHandler::handleCommandSplitByType(api::StorageCommand& msg, MessageTr case api::MessageType::CREATEBUCKET_ID: return _simpleHandler.handleCreateBucket(static_cast<api::CreateBucketCommand&>(msg), std::move(tracker)); case api::MessageType::DELETEBUCKET_ID: - return _simpleHandler.handleDeleteBucket(static_cast<api::DeleteBucketCommand&>(msg), std::move(tracker)); + return _asyncHandler.handleDeleteBucket(static_cast<api::DeleteBucketCommand&>(msg), std::move(tracker)); case api::MessageType::JOINBUCKETS_ID: return _splitJoinHandler.handleJoinBuckets(static_cast<api::JoinBucketsCommand&>(msg), std::move(tracker)); case api::MessageType::SPLITBUCKET_ID: @@ -62,7 +63,7 @@ PersistenceHandler::handleCommandSplitByType(api::StorageCommand& msg, MessageTr case api::MessageType::APPLYBUCKETDIFF_ID: return _mergeHandler.handleApplyBucketDiff(static_cast<api::ApplyBucketDiffCommand&>(msg), std::move(tracker)); case api::MessageType::SETBUCKETSTATE_ID: - return _splitJoinHandler.handleSetBucketState(static_cast<api::SetBucketStateCommand&>(msg), std::move(tracker)); + return _asyncHandler.handleSetBucketState(static_cast<api::SetBucketStateCommand&>(msg), std::move(tracker)); case api::MessageType::INTERNAL_ID: switch(static_cast<api::InternalCommand&>(msg).getType()) { case GetIterCommand::ID: @@ -87,19 +88,20 @@ PersistenceHandler::handleCommandSplitByType(api::StorageCommand& msg, MessageTr return MessageTracker::UP(); } -void -PersistenceHandler::handleReply(api::StorageReply& reply) const +MessageTracker::UP +PersistenceHandler::handleReply(api::StorageReply& reply, MessageTracker::UP tracker) const { switch (reply.getType().getId()) { case api::MessageType::GETBUCKETDIFF_REPLY_ID: _mergeHandler.handleGetBucketDiffReply(static_cast<api::GetBucketDiffReply&>(reply), _env._fileStorHandler); break; case api::MessageType::APPLYBUCKETDIFF_REPLY_ID: - _mergeHandler.handleApplyBucketDiffReply(static_cast<api::ApplyBucketDiffReply&>(reply), _env._fileStorHandler); + _mergeHandler.handleApplyBucketDiffReply(static_cast<api::ApplyBucketDiffReply&>(reply), _env._fileStorHandler, std::move(tracker)); break; default: break; } + return tracker; } MessageTracker::UP @@ -112,7 +114,7 @@ PersistenceHandler::processMessage(api::StorageMessage& msg, MessageTracker::UP try{ LOG(debug, "Handling reply: %s", msg.toString().c_str()); LOG(spam, "Message content: %s", msg.toString(true).c_str()); - handleReply(static_cast<api::StorageReply&>(msg)); + return handleReply(static_cast<api::StorageReply&>(msg), std::move(tracker)); } catch (std::exception& e) { // It's a reply, so nothing we can do. LOG(debug, "Caught exception for %s: %s", msg.toString().c_str(), e.what()); diff --git a/storage/src/vespa/storage/persistence/persistencehandler.h b/storage/src/vespa/storage/persistence/persistencehandler.h index a800d1d4053..a92c2dc78ca 100644 --- a/storage/src/vespa/storage/persistence/persistencehandler.h +++ b/storage/src/vespa/storage/persistence/persistencehandler.h @@ -38,7 +38,7 @@ public: private: // Message handling functions MessageTracker::UP handleCommandSplitByType(api::StorageCommand&, MessageTracker::UP tracker) const; - void handleReply(api::StorageReply&) const; + MessageTracker::UP handleReply(api::StorageReply&, MessageTracker::UP) const; MessageTracker::UP processMessage(api::StorageMessage& msg, MessageTracker::UP tracker) const; diff --git a/storage/src/vespa/storage/persistence/provider_error_wrapper.cpp b/storage/src/vespa/storage/persistence/provider_error_wrapper.cpp index 471d3d62a35..ce424f0ce83 100644 --- a/storage/src/vespa/storage/persistence/provider_error_wrapper.cpp +++ b/storage/src/vespa/storage/persistence/provider_error_wrapper.cpp @@ -60,10 +60,11 @@ ProviderErrorWrapper::setClusterState(BucketSpace bucketSpace, const spi::Cluste return checkResult(_impl.setClusterState(bucketSpace, state)); } -spi::Result -ProviderErrorWrapper::setActiveState(const spi::Bucket& bucket, spi::BucketInfo::ActiveState newState) +void +ProviderErrorWrapper::setActiveStateAsync(const spi::Bucket& bucket, spi::BucketInfo::ActiveState newState, spi::OperationComplete::UP onComplete) { - return checkResult(_impl.setActiveState(bucket, newState)); + onComplete->addResultHandler(this); + _impl.setActiveStateAsync(bucket, newState, std::move(onComplete)); } spi::BucketInfoResult @@ -72,32 +73,6 @@ ProviderErrorWrapper::getBucketInfo(const spi::Bucket& bucket) const return checkResult(_impl.getBucketInfo(bucket)); } -spi::Result -ProviderErrorWrapper::put(const spi::Bucket& bucket, spi::Timestamp ts, spi::DocumentSP doc, spi::Context& context) -{ - return checkResult(_impl.put(bucket, ts, std::move(doc), context)); -} - -spi::RemoveResult -ProviderErrorWrapper::remove(const spi::Bucket& bucket, spi::Timestamp ts, const document::DocumentId& docId, spi::Context& context) -{ - return checkResult(_impl.remove(bucket, ts, docId, context)); -} - -spi::RemoveResult -ProviderErrorWrapper::removeIfFound(const spi::Bucket& bucket, spi::Timestamp ts, - const document::DocumentId& docId, spi::Context& context) -{ - return checkResult(_impl.removeIfFound(bucket, ts, docId, context)); -} - -spi::UpdateResult -ProviderErrorWrapper::update(const spi::Bucket& bucket, spi::Timestamp ts, - spi::DocumentUpdateSP docUpdate, spi::Context& context) -{ - return checkResult(_impl.update(bucket, ts, std::move(docUpdate), context)); -} - spi::GetResult ProviderErrorWrapper::get(const spi::Bucket& bucket, const document::FieldSet& fieldSet, const document::DocumentId& docId, spi::Context& context) const @@ -130,10 +105,11 @@ ProviderErrorWrapper::createBucket(const spi::Bucket& bucket, spi::Context& cont return checkResult(_impl.createBucket(bucket, context)); } -spi::Result -ProviderErrorWrapper::deleteBucket(const spi::Bucket& bucket, spi::Context& context) +void +ProviderErrorWrapper::deleteBucketAsync(const spi::Bucket& bucket, spi::Context& context, spi::OperationComplete::UP onComplete) { - return checkResult(_impl.deleteBucket(bucket, context)); + onComplete->addResultHandler(this); + _impl.deleteBucketAsync(bucket, context, std::move(onComplete)); } spi::BucketIdListResult diff --git a/storage/src/vespa/storage/persistence/provider_error_wrapper.h b/storage/src/vespa/storage/persistence/provider_error_wrapper.h index d23cce9172a..c9d2411e372 100644 --- a/storage/src/vespa/storage/persistence/provider_error_wrapper.h +++ b/storage/src/vespa/storage/persistence/provider_error_wrapper.h @@ -41,12 +41,8 @@ public: spi::Result initialize() override; spi::BucketIdListResult listBuckets(BucketSpace bucketSpace) const override; spi::Result setClusterState(BucketSpace bucketSpace, const spi::ClusterState&) override; - spi::Result setActiveState(const spi::Bucket& bucket, spi::BucketInfo::ActiveState newState) override; + spi::BucketInfoResult getBucketInfo(const spi::Bucket&) const override; - spi::Result put(const spi::Bucket&, spi::Timestamp, spi::DocumentSP, spi::Context&) override; - spi::RemoveResult remove(const spi::Bucket&, spi::Timestamp, const document::DocumentId&, spi::Context&) override; - spi::RemoveResult removeIfFound(const spi::Bucket&, spi::Timestamp, const document::DocumentId&, spi::Context&) override; - spi::UpdateResult update(const spi::Bucket&, spi::Timestamp, spi::DocumentUpdateSP, spi::Context&) override; spi::GetResult get(const spi::Bucket&, const document::FieldSet&, const document::DocumentId&, spi::Context&) const override; spi::CreateIteratorResult createIterator(const spi::Bucket &bucket, FieldSetSP, const spi::Selection &, spi::IncludedVersions versions, @@ -54,7 +50,6 @@ public: spi::IterateResult iterate(spi::IteratorId, uint64_t maxByteSize, spi::Context&) const override; spi::Result destroyIterator(spi::IteratorId, spi::Context&) override; spi::Result createBucket(const spi::Bucket&, spi::Context&) override; - spi::Result deleteBucket(const spi::Bucket&, spi::Context&) override; spi::BucketIdListResult getModifiedBuckets(BucketSpace bucketSpace) const override; spi::Result split(const spi::Bucket& source, const spi::Bucket& target1, const spi::Bucket& target2, spi::Context&) override; spi::Result join(const spi::Bucket& source1, const spi::Bucket& source2, const spi::Bucket& target, spi::Context&) override; @@ -67,6 +62,8 @@ public: void removeAsync(const spi::Bucket&, spi::Timestamp, const document::DocumentId&, spi::Context&, spi::OperationComplete::UP) override; void removeIfFoundAsync(const spi::Bucket&, spi::Timestamp, const document::DocumentId&, spi::Context&, spi::OperationComplete::UP) override; void updateAsync(const spi::Bucket &, spi::Timestamp, spi::DocumentUpdateSP, spi::Context &, spi::OperationComplete::UP) override; + void setActiveStateAsync(const spi::Bucket& b, spi::BucketInfo::ActiveState newState, spi::OperationComplete::UP onComplete) override; + void deleteBucketAsync(const spi::Bucket&, spi::Context&, spi::OperationComplete::UP) override; std::unique_ptr<vespalib::IDestructorCallback> register_executor(std::shared_ptr<spi::BucketExecutor> executor) override; private: template <typename ResultType> diff --git a/storage/src/vespa/storage/persistence/simplemessagehandler.cpp b/storage/src/vespa/storage/persistence/simplemessagehandler.cpp index b4c09f09e63..b4fe207e2e5 100644 --- a/storage/src/vespa/storage/persistence/simplemessagehandler.cpp +++ b/storage/src/vespa/storage/persistence/simplemessagehandler.cpp @@ -43,16 +43,6 @@ getFieldSet(const document::FieldSetRepo & repo, vespalib::stringref name, Messa return document::FieldSet::SP(); } -bool -bucketStatesAreSemanticallyEqual(const api::BucketInfo& a, const api::BucketInfo& b) { - // Don't check document sizes, as background moving of documents in Proton - // may trigger a change in size without any mutations taking place. This will - // only take place when a document being moved was fed _prior_ to the change - // where Proton starts reporting actual document sizes, and will eventually - // converge to a stable value. But for now, ignore it to prevent false positive - // error logs and non-deleted buckets. - return ((a.getChecksum() == b.getChecksum()) && (a.getDocumentCount() == b.getDocumentCount())); -} } SimpleMessageHandler::SimpleMessageHandler(const PersistenceUtil& env, spi::PersistenceProvider& spi) : _env(env), @@ -113,70 +103,6 @@ SimpleMessageHandler::handleCreateBucket(api::CreateBucketCommand& cmd, MessageT return tracker; } -bool -SimpleMessageHandler::checkProviderBucketInfoMatches(const spi::Bucket& bucket, const api::BucketInfo& info) const -{ - spi::BucketInfoResult result(_spi.getBucketInfo(bucket)); - if (result.hasError()) { - LOG(error, "getBucketInfo(%s) failed before deleting bucket; got error '%s'", - bucket.toString().c_str(), result.getErrorMessage().c_str()); - return false; - } - api::BucketInfo providerInfo(PersistenceUtil::convertBucketInfo(result.getBucketInfo())); - // Don't check meta fields or active/ready fields since these are not - // that important and ready may change under the hood in a race with - // getModifiedBuckets(). If bucket is empty it means it has already - // been deleted by a racing split/join. - if (!bucketStatesAreSemanticallyEqual(info, providerInfo) && !providerInfo.empty()) { - LOG(error, - "Service layer bucket database and provider out of sync before " - "deleting bucket %s! Service layer db had %s while provider says " - "bucket has %s. Deletion has been rejected to ensure data is not " - "lost, but bucket may remain out of sync until service has been " - "restarted.", - bucket.toString().c_str(), info.toString().c_str(), providerInfo.toString().c_str()); - return false; - } - return true; -} - -MessageTracker::UP -SimpleMessageHandler::handleDeleteBucket(api::DeleteBucketCommand& cmd, MessageTracker::UP tracker) const -{ - tracker->setMetric(_env._metrics.deleteBuckets); - LOG(debug, "DeletingBucket(%s)", cmd.getBucketId().toString().c_str()); - if (_env._fileStorHandler.isMerging(cmd.getBucket())) { - _env._fileStorHandler.clearMergeStatus(cmd.getBucket(), - api::ReturnCode(api::ReturnCode::ABORTED, "Bucket was deleted during the merge")); - } - spi::Bucket bucket(cmd.getBucket()); - if (!checkProviderBucketInfoMatches(bucket, cmd.getBucketInfo())) { - return tracker; - } - _spi.deleteBucket(bucket, tracker->context()); - StorBucketDatabase& db(_env.getBucketDatabase(cmd.getBucket().getBucketSpace())); - { - StorBucketDatabase::WrappedEntry entry(db.get(cmd.getBucketId(), "FileStorThread::onDeleteBucket")); - if (entry.exist() && entry->getMetaCount() > 0) { - LOG(debug, "onDeleteBucket(%s): Bucket DB entry existed. Likely " - "active operation when delete bucket was queued. " - "Updating bucket database to keep it in sync with file. " - "Cannot delete bucket from bucket database at this " - "point, as it can have been intentionally recreated " - "after delete bucket had been sent", - cmd.getBucketId().toString().c_str()); - api::BucketInfo info(0, 0, 0); - // Only set document counts/size; retain ready/active state. - info.setReady(entry->getBucketInfo().isReady()); - info.setActive(entry->getBucketInfo().isActive()); - - entry->setBucketInfo(info); - entry.write(); - } - } - return tracker; -} - MessageTracker::UP SimpleMessageHandler::handleGetIter(GetIterCommand& cmd, MessageTracker::UP tracker) const { diff --git a/storage/src/vespa/storage/persistence/simplemessagehandler.h b/storage/src/vespa/storage/persistence/simplemessagehandler.h index 9f00f67684d..2cfbc7016c0 100644 --- a/storage/src/vespa/storage/persistence/simplemessagehandler.h +++ b/storage/src/vespa/storage/persistence/simplemessagehandler.h @@ -23,13 +23,11 @@ public: MessageTrackerUP handleGet(api::GetCommand& cmd, MessageTrackerUP tracker) const; MessageTrackerUP handleRevert(api::RevertCommand& cmd, MessageTrackerUP tracker) const; MessageTrackerUP handleCreateBucket(api::CreateBucketCommand& cmd, MessageTrackerUP tracker) const; - MessageTrackerUP handleDeleteBucket(api::DeleteBucketCommand& cmd, MessageTrackerUP tracker) const; MessageTrackerUP handleCreateIterator(CreateIteratorCommand& cmd, MessageTrackerUP tracker) const; MessageTrackerUP handleGetIter(GetIterCommand& cmd, MessageTrackerUP tracker) const; MessageTrackerUP handleReadBucketList(ReadBucketList& cmd, MessageTrackerUP tracker) const; MessageTrackerUP handleReadBucketInfo(ReadBucketInfo& cmd, MessageTrackerUP tracker) const; private: - bool checkProviderBucketInfoMatches(const spi::Bucket&, const api::BucketInfo&) const; const PersistenceUtil & _env; spi::PersistenceProvider & _spi; }; diff --git a/storage/src/vespa/storage/persistence/splitjoinhandler.cpp b/storage/src/vespa/storage/persistence/splitjoinhandler.cpp index 0856f45c3ff..d5b44cc1911 100644 --- a/storage/src/vespa/storage/persistence/splitjoinhandler.cpp +++ b/storage/src/vespa/storage/persistence/splitjoinhandler.cpp @@ -5,7 +5,6 @@ #include "bucketownershipnotifier.h" #include "splitbitdetector.h" #include "messages.h" -#include <vespa/storage/common/bucketmessages.h> #include <vespa/persistence/spi/persistenceprovider.h> #include <vespa/storageapi/message/bucket.h> @@ -145,37 +144,6 @@ SplitJoinHandler::handleSplitBucket(api::SplitBucketCommand& cmd, MessageTracker } MessageTracker::UP -SplitJoinHandler::handleSetBucketState(api::SetBucketStateCommand& cmd, MessageTracker::UP tracker) const -{ - tracker->setMetric(_env._metrics.setBucketStates); - NotificationGuard notifyGuard(_bucketOwnershipNotifier); - - LOG(debug, "handleSetBucketState(): %s", cmd.toString().c_str()); - spi::Bucket bucket(cmd.getBucket()); - bool shouldBeActive(cmd.getState() == api::SetBucketStateCommand::ACTIVE); - spi::BucketInfo::ActiveState newState(shouldBeActive ? spi::BucketInfo::ACTIVE : spi::BucketInfo::NOT_ACTIVE); - - spi::Result result(_spi.setActiveState(bucket, newState)); - if (tracker->checkForError(result)) { - StorBucketDatabase::WrappedEntry - entry = _env.getBucketDatabase(bucket.getBucket().getBucketSpace()).get(cmd.getBucketId(), "handleSetBucketState"); - if (entry.exist()) { - entry->info.setActive(newState == spi::BucketInfo::ACTIVE); - notifyGuard.notifyIfOwnershipChanged(cmd.getBucket(), cmd.getSourceIndex(), entry->info); - entry.write(); - } else { - LOG(warning, "Got OK setCurrentState result from provider for %s, " - "but bucket has disappeared from service layer database", - cmd.getBucketId().toString().c_str()); - } - - tracker->setReply(std::make_shared<api::SetBucketStateReply>(cmd)); - } - - return tracker; -} - -MessageTracker::UP SplitJoinHandler::handleRecheckBucketInfo(RecheckBucketInfoCommand& cmd, MessageTracker::UP tracker) const { tracker->setMetric(_env._metrics.recheckBucketInfo); diff --git a/storage/src/vespa/storage/persistence/splitjoinhandler.h b/storage/src/vespa/storage/persistence/splitjoinhandler.h index ddfa22b154c..4521e520ee9 100644 --- a/storage/src/vespa/storage/persistence/splitjoinhandler.h +++ b/storage/src/vespa/storage/persistence/splitjoinhandler.h @@ -21,7 +21,6 @@ public: SplitJoinHandler(PersistenceUtil &, spi::PersistenceProvider &, BucketOwnershipNotifier &, bool enableMultibitSplitOptimalization); MessageTrackerUP handleSplitBucket(api::SplitBucketCommand& cmd, MessageTrackerUP tracker) const; - MessageTrackerUP handleSetBucketState(api::SetBucketStateCommand& cmd, MessageTrackerUP tracker) const; MessageTrackerUP handleRecheckBucketInfo(RecheckBucketInfoCommand& cmd, MessageTrackerUP tracker) const; MessageTrackerUP handleJoinBuckets(api::JoinBucketsCommand& cmd, MessageTrackerUP tracker) const; private: diff --git a/storage/src/vespa/storage/storageserver/mergethrottler.cpp b/storage/src/vespa/storage/storageserver/mergethrottler.cpp index fa9ab22c1cb..a17c77f6ca4 100644 --- a/storage/src/vespa/storage/storageserver/mergethrottler.cpp +++ b/storage/src/vespa/storage/storageserver/mergethrottler.cpp @@ -72,6 +72,7 @@ MergeThrottler::Metrics::Metrics(metrics::MetricSet* owner) : metrics::MetricSet("mergethrottler", {}, "", owner), averageQueueWaitingTime("averagequeuewaitingtime", {}, "Average time a merge spends in the throttler queue", this), queueSize("queuesize", {}, "Length of merge queue", this), + active_window_size("active_window_size", {}, "Number of merges active within the pending window size", this), bounced_due_to_back_pressure("bounced_due_to_back_pressure", {}, "Number of merges bounced due to resource exhaustion back-pressure", this), chaining("mergechains", this), local("locallyexecutedmerges", this) @@ -366,6 +367,7 @@ MergeThrottler::removeActiveMerge(ActiveMergeMap::iterator mergeIter) LOG(debug, "Removed merge for %s from internal state", mergeIter->first.toString().c_str()); _merges.erase(mergeIter); + update_active_merge_window_size_metric(); } api::StorageMessage::SP @@ -815,6 +817,7 @@ MergeThrottler::processNewMergeCommand( // merge throttling window. assert(_merges.find(mergeCmd.getBucket()) == _merges.end()); auto state = _merges.emplace(mergeCmd.getBucket(), ChainedMergeState(msg)).first; + update_active_merge_window_size_metric(); LOG(debug, "Added merge %s to internal state", mergeCmd.toString().c_str()); @@ -1247,6 +1250,11 @@ MergeThrottler::set_disable_queue_limits_for_chained_merges(bool disable_limits) } void +MergeThrottler::update_active_merge_window_size_metric() noexcept { + _metrics->active_window_size.set(static_cast<int64_t>(_merges.size())); +} + +void MergeThrottler::print(std::ostream& out, bool /*verbose*/, const std::string& /*indent*/) const { diff --git a/storage/src/vespa/storage/storageserver/mergethrottler.h b/storage/src/vespa/storage/storageserver/mergethrottler.h index 9b0fb125b2f..997477a4b70 100644 --- a/storage/src/vespa/storage/storageserver/mergethrottler.h +++ b/storage/src/vespa/storage/storageserver/mergethrottler.h @@ -64,6 +64,7 @@ public: public: metrics::DoubleAverageMetric averageQueueWaitingTime; metrics::LongValueMetric queueSize; + metrics::LongValueMetric active_window_size; metrics::LongCountMetric bounced_due_to_back_pressure; MergeOperationMetrics chaining; MergeOperationMetrics local; @@ -388,6 +389,8 @@ private: void rejectOperationsInThreadQueue(MessageGuard&, uint32_t minimumStateVersion); void markActiveMergesAsAborted(uint32_t minimumStateVersion); + void update_active_merge_window_size_metric() noexcept; + // const function, but metrics are mutable void updateOperationMetrics( const api::ReturnCode& result, diff --git a/vespajlib/src/main/java/com/yahoo/concurrent/maintenance/Maintainer.java b/vespajlib/src/main/java/com/yahoo/concurrent/maintenance/Maintainer.java index 1bf84035c82..9f27ac507c9 100644 --- a/vespajlib/src/main/java/com/yahoo/concurrent/maintenance/Maintainer.java +++ b/vespajlib/src/main/java/com/yahoo/concurrent/maintenance/Maintainer.java @@ -4,6 +4,8 @@ package com.yahoo.concurrent.maintenance; import com.google.common.util.concurrent.UncheckedTimeoutException; import com.yahoo.net.HostName; +import java.math.BigDecimal; +import java.math.RoundingMode; import java.time.Duration; import java.time.Instant; import java.util.List; @@ -94,7 +96,8 @@ public abstract class Maintainer implements Runnable { /** Convenience methods to convert attempts and failures into a success factor */ protected final double asSuccessFactor(int attempts, int failures) { - return attempts == 0 ? 1.0 : 1 - (double)failures / attempts; + double factor = attempts == 0 ? 1.0 : 1 - (double)failures / attempts; + return new BigDecimal(factor).setScale(2, RoundingMode.HALF_UP).doubleValue(); } /** Returns the interval at which this job is set to run */ diff --git a/vespajlib/src/main/java/com/yahoo/io/NativeIO.java b/vespajlib/src/main/java/com/yahoo/io/NativeIO.java index 34e68edcedd..3cb1cdf5242 100644 --- a/vespajlib/src/main/java/com/yahoo/io/NativeIO.java +++ b/vespajlib/src/main/java/com/yahoo/io/NativeIO.java @@ -19,14 +19,19 @@ import com.sun.jna.Platform; */ public class NativeIO { private final static Logger logger = Logger.getLogger(NativeIO.class.getName()); + private final static String DISABLE_NATIVE_IO = "DISABLE_NATIVE_IO"; private static final int POSIX_FADV_DONTNEED = 4; // See /usr/include/linux/fadvise.h private static boolean initialized = false; + private static boolean disabled = false; private static Throwable initError = null; static { try { if (Platform.isLinux()) { - Native.register(Platform.C_LIBRARY_NAME); - initialized = true; + disabled = System.getenv().containsKey(DISABLE_NATIVE_IO); + if (!disabled) { + Native.register(Platform.C_LIBRARY_NAME); + initialized = true; + } } } catch (Throwable throwable) { initError = throwable; @@ -40,7 +45,11 @@ public class NativeIO { public NativeIO() { if (!initialized) { - logger.warning("native IO not possible due to " + getError().getMessage()); + if (disabled) { + logger.info("Native IO has been disable explicit via system property " + DISABLE_NATIVE_IO); + } else { + logger.warning("Native IO not possible due to " + getError().getMessage()); + } } } diff --git a/vespalib/src/tests/executor/threadstackexecutor_test.cpp b/vespalib/src/tests/executor/threadstackexecutor_test.cpp index 63c6856afd2..b55f54f9339 100644 --- a/vespalib/src/tests/executor/threadstackexecutor_test.cpp +++ b/vespalib/src/tests/executor/threadstackexecutor_test.cpp @@ -33,17 +33,18 @@ std::atomic<uint32_t> MyTask::runCnt(0); std::atomic<uint32_t> MyTask::deleteCnt(0); struct MyState { + static constexpr uint32_t NUM_THREADS = 10; Gate gate; // to block workers CountDownLatch latch; // to wait for workers ThreadStackExecutor executor; bool checked; - MyState() : gate(), latch(10), executor(10, 128000, 20), checked(false) + MyState() : gate(), latch(10), executor(NUM_THREADS, 128000, 20), checked(false) { MyTask::resetStats(); } MyState &execute(uint32_t cnt) { for (uint32_t i = 0; i < cnt; ++i) { - executor.execute(Task::UP(new MyTask(gate, latch))); + executor.execute(std::make_unique<MyTask>(gate, latch)); } return *this; } @@ -70,7 +71,7 @@ struct MyState { { ASSERT_TRUE(!checked); checked = true; - ThreadStackExecutor::Stats stats = executor.getStats(); + ExecutorStats stats = executor.getStats(); EXPECT_EQUAL(expect_running + expect_deleted, MyTask::runCnt); EXPECT_EQUAL(expect_rejected + expect_deleted, MyTask::deleteCnt); EXPECT_EQUAL(expect_queue + expect_running + expect_deleted, @@ -187,11 +188,11 @@ TEST_F("require that executor thread stack tag can be set", ThreadStackExecutor( } TEST("require that stats can be accumulated") { - ThreadStackExecutor::Stats stats(ThreadExecutor::Stats::QueueSizeT(1) ,2,3); + ExecutorStats stats(ExecutorStats::QueueSizeT(1) ,2,3); EXPECT_EQUAL(1u, stats.queueSize.max()); EXPECT_EQUAL(2u, stats.acceptedTasks); EXPECT_EQUAL(3u, stats.rejectedTasks); - stats += ThreadStackExecutor::Stats(ThreadExecutor::Stats::QueueSizeT(7),8,9); + stats += ExecutorStats(ExecutorStats::QueueSizeT(7),8,9); EXPECT_EQUAL(2u, stats.queueSize.count()); EXPECT_EQUAL(8u, stats.queueSize.total()); EXPECT_EQUAL(8u, stats.queueSize.max()); diff --git a/vespalib/src/vespa/vespalib/util/CMakeLists.txt b/vespalib/src/vespa/vespalib/util/CMakeLists.txt index 9e2917775f0..6115c21623d 100644 --- a/vespalib/src/vespa/vespalib/util/CMakeLists.txt +++ b/vespalib/src/vespa/vespalib/util/CMakeLists.txt @@ -38,6 +38,7 @@ vespa_add_library(vespalib_vespalib_util OBJECT memoryusage.cpp mmap_file_allocator.cpp mmap_file_allocator_factory.cpp + monitored_refcount.cpp printable.cpp priority_queue.cpp random.cpp diff --git a/vespalib/src/vespa/vespalib/util/executor_stats.h b/vespalib/src/vespa/vespalib/util/executor_stats.h index 49d83c96714..f1f58685570 100644 --- a/vespalib/src/vespa/vespalib/util/executor_stats.h +++ b/vespalib/src/vespa/vespalib/util/executor_stats.h @@ -3,6 +3,7 @@ #pragma once #include <limits> +#include <cstdint> namespace vespalib { @@ -15,10 +16,10 @@ public: AggregatedAverage() : AggregatedAverage(0ul, T(0), std::numeric_limits<T>::max(), std::numeric_limits<T>::min()) { } explicit AggregatedAverage(T value) : AggregatedAverage(1, value, value, value) { } AggregatedAverage(size_t count_in, T total_in, T min_in, T max_in) - : _count(count_in), - _total(total_in), - _min(min_in), - _max(max_in) + : _count(count_in), + _total(total_in), + _min(min_in), + _max(max_in) { } AggregatedAverage & operator += (const AggregatedAverage & rhs) { add(rhs); diff --git a/searchcore/src/vespa/searchcore/proton/common/monitored_refcount.cpp b/vespalib/src/vespa/vespalib/util/monitored_refcount.cpp index 97615abba31..4376e26bb66 100644 --- a/searchcore/src/vespa/searchcore/proton/common/monitored_refcount.cpp +++ b/vespalib/src/vespa/vespalib/util/monitored_refcount.cpp @@ -3,7 +3,7 @@ #include "monitored_refcount.h" #include <cassert> -namespace proton { +namespace vespalib { MonitoredRefCount::MonitoredRefCount() : _lock(), @@ -41,4 +41,4 @@ MonitoredRefCount::waitForZeroRefCount() _cv.wait(guard, [this] { return (_refCount == 0u); }); } -} // namespace proton +} diff --git a/vespalib/src/vespa/vespalib/util/monitored_refcount.h b/vespalib/src/vespa/vespalib/util/monitored_refcount.h new file mode 100644 index 00000000000..465284b6fd3 --- /dev/null +++ b/vespalib/src/vespa/vespalib/util/monitored_refcount.h @@ -0,0 +1,29 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#pragma once + +#include <mutex> +#include <condition_variable> + +namespace vespalib { + +class RetainGuard; +/* + * Class containing a reference count that can be waited on to become zero. + * Typically ancestor or member of a class that has to be careful of when + * portions object can be properly torn down before destruction itself. + */ +class MonitoredRefCount +{ + std::mutex _lock; + std::condition_variable _cv; + uint32_t _refCount; + void retain() noexcept; + void release() noexcept; + friend RetainGuard; +public: + MonitoredRefCount(); + virtual ~MonitoredRefCount(); + void waitForZeroRefCount(); +}; + +} diff --git a/searchcore/src/vespa/searchcore/proton/common/monitored_refcount.h b/vespalib/src/vespa/vespalib/util/retain_guard.h index 9eb713b1220..090f3ce75cf 100644 --- a/searchcore/src/vespa/searchcore/proton/common/monitored_refcount.h +++ b/vespalib/src/vespa/vespalib/util/retain_guard.h @@ -1,31 +1,16 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + #pragma once -#include <mutex> -#include <condition_variable> +#include "monitored_refcount.h" -namespace proton { +namespace vespalib { -class RetainGuard; /* - * Class containing a reference count that can be waited on to become zero. - * Typically ancestor or member of a class that has to be careful of when - * portions object can be properly torn down before destruction itself. + * Class containing a reference to a monitored reference count, + * intended to block teardown of the class owning the monitored + * reference count. */ -class MonitoredRefCount -{ - std::mutex _lock; - std::condition_variable _cv; - uint32_t _refCount; - void retain() noexcept; - void release() noexcept; - friend RetainGuard; -public: - MonitoredRefCount(); - virtual ~MonitoredRefCount(); - void waitForZeroRefCount(); -}; - class RetainGuard { public: RetainGuard(MonitoredRefCount & refCount) noexcept @@ -57,4 +42,4 @@ private: MonitoredRefCount * _refCount; }; -} // namespace proton +} diff --git a/vespalib/src/vespa/vespalib/util/threadexecutor.h b/vespalib/src/vespa/vespalib/util/threadexecutor.h index 9ab7aacfc4b..36c72fa4bb0 100644 --- a/vespalib/src/vespa/vespalib/util/threadexecutor.h +++ b/vespalib/src/vespa/vespalib/util/threadexecutor.h @@ -12,11 +12,6 @@ class ThreadExecutor : public Executor { public: /** - * Internal stats that we want to observe externally. Note that - * all stats are reset each time they are observed. - **/ - using Stats = ExecutorStats; - /** * Get number of threads in the executor pool. * @return number of threads in the pool */ @@ -26,7 +21,7 @@ public: * Observe and reset stats for this object. * @return stats **/ - virtual Stats getStats() = 0; + virtual ExecutorStats getStats() = 0; /** * Sets a new upper limit for accepted number of tasks. diff --git a/vespalib/src/vespa/vespalib/util/threadstackexecutorbase.cpp b/vespalib/src/vespa/vespalib/util/threadstackexecutorbase.cpp index 32e47f366cc..f80a5b4ce32 100644 --- a/vespalib/src/vespa/vespalib/util/threadstackexecutorbase.cpp +++ b/vespalib/src/vespa/vespalib/util/threadstackexecutorbase.cpp @@ -203,12 +203,12 @@ ThreadStackExecutorBase::num_idle_workers() const return _workers.size(); } -ThreadStackExecutorBase::Stats +ExecutorStats ThreadStackExecutorBase::getStats() { std::unique_lock guard(_lock); - Stats stats = _stats; - _stats = Stats(); + ExecutorStats stats = _stats; + _stats = ExecutorStats(); _stats.queueSize.add(_taskCount); return stats; } diff --git a/vespalib/src/vespa/vespalib/util/threadstackexecutorbase.h b/vespalib/src/vespa/vespalib/util/threadstackexecutorbase.h index 59ed385b4f4..66a34bfde95 100644 --- a/vespalib/src/vespa/vespalib/util/threadstackexecutorbase.h +++ b/vespalib/src/vespa/vespalib/util/threadstackexecutorbase.h @@ -80,7 +80,7 @@ private: std::unique_ptr<FastOS_ThreadPool> _pool; mutable std::mutex _lock; std::condition_variable _cond; - Stats _stats; + ExecutorStats _stats; Gate _executorCompletion; ArrayQueue<TaggedTask> _tasks; ArrayQueue<Worker*> _workers; @@ -188,7 +188,7 @@ public: **/ size_t num_idle_workers() const; - Stats getStats() override; + ExecutorStats getStats() override; Task::UP execute(Task::UP task) override; |
