diff options
246 files changed, 2812 insertions, 1963 deletions
diff --git a/client/go/cmd/logfmt/cmd.go b/client/go/script-utils/logfmt/cmd.go index 84322c7ff08..84322c7ff08 100644 --- a/client/go/cmd/logfmt/cmd.go +++ b/client/go/script-utils/logfmt/cmd.go diff --git a/client/go/cmd/logfmt/formatflags.go b/client/go/script-utils/logfmt/formatflags.go index 097746d696f..097746d696f 100644 --- a/client/go/cmd/logfmt/formatflags.go +++ b/client/go/script-utils/logfmt/formatflags.go diff --git a/client/go/cmd/logfmt/formatflags_test.go b/client/go/script-utils/logfmt/formatflags_test.go index 53c47d24208..53c47d24208 100644 --- a/client/go/cmd/logfmt/formatflags_test.go +++ b/client/go/script-utils/logfmt/formatflags_test.go diff --git a/client/go/cmd/logfmt/handleline.go b/client/go/script-utils/logfmt/handleline.go index 813ca82acb4..813ca82acb4 100644 --- a/client/go/cmd/logfmt/handleline.go +++ b/client/go/script-utils/logfmt/handleline.go diff --git a/client/go/cmd/logfmt/internal.go b/client/go/script-utils/logfmt/internal.go index 992c537f939..992c537f939 100644 --- a/client/go/cmd/logfmt/internal.go +++ b/client/go/script-utils/logfmt/internal.go diff --git a/client/go/cmd/logfmt/internal_names.txt b/client/go/script-utils/logfmt/internal_names.txt index cc554546fcc..cc554546fcc 100644 --- a/client/go/cmd/logfmt/internal_names.txt +++ b/client/go/script-utils/logfmt/internal_names.txt diff --git a/client/go/cmd/logfmt/internal_notnames.txt b/client/go/script-utils/logfmt/internal_notnames.txt index 49543758ac3..49543758ac3 100644 --- a/client/go/cmd/logfmt/internal_notnames.txt +++ b/client/go/script-utils/logfmt/internal_notnames.txt diff --git a/client/go/cmd/logfmt/internal_test.go b/client/go/script-utils/logfmt/internal_test.go index 9b6b0f8404c..9b6b0f8404c 100644 --- a/client/go/cmd/logfmt/internal_test.go +++ b/client/go/script-utils/logfmt/internal_test.go diff --git a/client/go/cmd/logfmt/levelflags.go b/client/go/script-utils/logfmt/levelflags.go index 4e6c1284753..4e6c1284753 100644 --- a/client/go/cmd/logfmt/levelflags.go +++ b/client/go/script-utils/logfmt/levelflags.go diff --git a/client/go/cmd/logfmt/levelflags_test.go b/client/go/script-utils/logfmt/levelflags_test.go index 186ea2d96b0..186ea2d96b0 100644 --- a/client/go/cmd/logfmt/levelflags_test.go +++ b/client/go/script-utils/logfmt/levelflags_test.go diff --git a/client/go/cmd/logfmt/options.go b/client/go/script-utils/logfmt/options.go index 864868d4ce5..864868d4ce5 100644 --- a/client/go/cmd/logfmt/options.go +++ b/client/go/script-utils/logfmt/options.go diff --git a/client/go/cmd/logfmt/plusminusflag.go b/client/go/script-utils/logfmt/plusminusflag.go index 1768cf0e7be..1768cf0e7be 100644 --- a/client/go/cmd/logfmt/plusminusflag.go +++ b/client/go/script-utils/logfmt/plusminusflag.go diff --git a/client/go/cmd/logfmt/regexflag.go b/client/go/script-utils/logfmt/regexflag.go index 8f7d2a91373..8f7d2a91373 100644 --- a/client/go/cmd/logfmt/regexflag.go +++ b/client/go/script-utils/logfmt/regexflag.go diff --git a/client/go/cmd/logfmt/regexflag_test.go b/client/go/script-utils/logfmt/regexflag_test.go index 489439863a2..489439863a2 100644 --- a/client/go/cmd/logfmt/regexflag_test.go +++ b/client/go/script-utils/logfmt/regexflag_test.go diff --git a/client/go/cmd/logfmt/runlogfmt.go b/client/go/script-utils/logfmt/runlogfmt.go index 5b9a3ac0870..5b9a3ac0870 100644 --- a/client/go/cmd/logfmt/runlogfmt.go +++ b/client/go/script-utils/logfmt/runlogfmt.go diff --git a/client/go/cmd/logfmt/showflags.go b/client/go/script-utils/logfmt/showflags.go index b69860e0312..b69860e0312 100644 --- a/client/go/cmd/logfmt/showflags.go +++ b/client/go/script-utils/logfmt/showflags.go diff --git a/client/go/cmd/logfmt/showflags_test.go b/client/go/script-utils/logfmt/showflags_test.go index d1b66118afd..d1b66118afd 100644 --- a/client/go/cmd/logfmt/showflags_test.go +++ b/client/go/script-utils/logfmt/showflags_test.go diff --git a/client/go/cmd/logfmt/tail.go b/client/go/script-utils/logfmt/tail.go index 75e7cbb0693..75e7cbb0693 100644 --- a/client/go/cmd/logfmt/tail.go +++ b/client/go/script-utils/logfmt/tail.go diff --git a/client/go/cmd/logfmt/tail_not_unix.go b/client/go/script-utils/logfmt/tail_not_unix.go index 7030572575d..7030572575d 100644 --- a/client/go/cmd/logfmt/tail_not_unix.go +++ b/client/go/script-utils/logfmt/tail_not_unix.go diff --git a/client/go/cmd/logfmt/tail_unix.go b/client/go/script-utils/logfmt/tail_unix.go index 7703844da48..7703844da48 100644 --- a/client/go/cmd/logfmt/tail_unix.go +++ b/client/go/script-utils/logfmt/tail_unix.go diff --git a/client/go/script-utils/main.go b/client/go/script-utils/main.go index 79125292632..b0b7edd6f6c 100644 --- a/client/go/script-utils/main.go +++ b/client/go/script-utils/main.go @@ -10,9 +10,9 @@ import ( "github.com/vespa-engine/vespa/client/go/cmd/clusterstate" "github.com/vespa-engine/vespa/client/go/cmd/deploy" - "github.com/vespa-engine/vespa/client/go/cmd/logfmt" "github.com/vespa-engine/vespa/client/go/jvm" "github.com/vespa-engine/vespa/client/go/script-utils/configserver" + "github.com/vespa-engine/vespa/client/go/script-utils/logfmt" "github.com/vespa-engine/vespa/client/go/script-utils/services" "github.com/vespa-engine/vespa/client/go/script-utils/standalone" "github.com/vespa-engine/vespa/client/go/script-utils/startcbinary" diff --git a/config-model-api/src/main/java/com/yahoo/config/application/api/DeploymentSpec.java b/config-model-api/src/main/java/com/yahoo/config/application/api/DeploymentSpec.java index d731e09d4e4..cbdb5bd6bcc 100644 --- a/config-model-api/src/main/java/com/yahoo/config/application/api/DeploymentSpec.java +++ b/config-model-api/src/main/java/com/yahoo/config/application/api/DeploymentSpec.java @@ -194,6 +194,10 @@ public class DeploymentSpec { */ public ZoneEndpoint zoneEndpoint(InstanceName instance, ZoneId zone, ClusterSpec.Id cluster) { // TODO: look up endpoints from <dev> tag, or so, if we're to support non-prod settings. + if ( zone.environment().isTest() + && instances().stream() + .anyMatch(spec -> spec.zoneEndpoints().getOrDefault(cluster, Map.of()).values().stream() + .anyMatch(endpoint -> ! endpoint.isPublicEndpoint()))) return ZoneEndpoint.privateEndpoint; if (zone.environment() != Environment.prod) return ZoneEndpoint.defaultEndpoint; return instance(instance).flatMap(spec -> spec.zoneEndpoint(zone, cluster)) .orElse(ZoneEndpoint.defaultEndpoint); diff --git a/config-model-api/src/test/java/com/yahoo/config/application/api/DeploymentSpecTest.java b/config-model-api/src/test/java/com/yahoo/config/application/api/DeploymentSpecTest.java index 355ce651c34..2e746ff55c8 100644 --- a/config-model-api/src/test/java/com/yahoo/config/application/api/DeploymentSpecTest.java +++ b/config-model-api/src/test/java/com/yahoo/config/application/api/DeploymentSpecTest.java @@ -1217,6 +1217,9 @@ public class DeploymentSpecTest { assertEquals(ZoneEndpoint.defaultEndpoint, spec.zoneEndpoint(InstanceName.defaultName(), defaultId(), ClusterSpec.Id.from("cluster"))); + assertEquals(ZoneEndpoint.defaultEndpoint, spec.zoneEndpoint(InstanceName.defaultName(), + com.yahoo.config.provision.zone.ZoneId.from("test", "us"), + ClusterSpec.Id.from("cluster"))); } @Test @@ -1271,12 +1274,17 @@ public class DeploymentSpecTest { assertEquals(List.of(RegionName.from("us-east")), spec.requireInstance("default").endpoints().get(0).regions()); var zone = from(Environment.prod, RegionName.from("us-east")); + var testZone = from(Environment.test, RegionName.from("us-east")); assertEquals(ZoneEndpoint.defaultEndpoint, spec.zoneEndpoint(InstanceName.from("custom"), zone, ClusterSpec.Id.from("bax"))); assertEquals(ZoneEndpoint.defaultEndpoint, spec.zoneEndpoint(InstanceName.from("default"), defaultId(), ClusterSpec.Id.from("bax"))); assertEquals(ZoneEndpoint.defaultEndpoint, spec.zoneEndpoint(InstanceName.from("default"), zone, ClusterSpec.Id.from("bax"))); + assertEquals(ZoneEndpoint.defaultEndpoint, + spec.zoneEndpoint(InstanceName.from("default"), testZone, ClusterSpec.Id.from("bax"))); + assertEquals(ZoneEndpoint.privateEndpoint, + spec.zoneEndpoint(InstanceName.from("default"), testZone, ClusterSpec.Id.from("froz"))); assertEquals(new ZoneEndpoint(false, true, List.of(new AllowedUrn(AccessType.awsPrivateLink, "barn"), new AllowedUrn(AccessType.gcpServiceConnect, "nine"))), diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/VespaMetricSet.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/VespaMetricSet.java index 416826bcd91..b940cfc0a74 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/VespaMetricSet.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/VespaMetricSet.java @@ -53,12 +53,12 @@ public class VespaMetricSet { private static Set<Metric> getSentinelMetrics() { Set<Metric> metrics = new LinkedHashSet<>(); - metrics.add(new Metric("sentinel.restarts.count")); - metrics.add(new Metric("sentinel.totalRestarts.last")); - metrics.add(new Metric("sentinel.uptime.last")); + addMetric(metrics, "sentinel.restarts.count"); + addMetric(metrics, "sentinel.totalRestarts.last"); + addMetric(metrics, "sentinel.uptime.last"); - metrics.add(new Metric("sentinel.running.count")); - metrics.add(new Metric("sentinel.running.last")); + addMetric(metrics, "sentinel.running.count"); + addMetric(metrics, "sentinel.running.last"); return metrics; } @@ -66,36 +66,36 @@ public class VespaMetricSet { private static Set<Metric> getOtherMetrics() { Set<Metric> metrics = new LinkedHashSet<>(); - metrics.add(new Metric("slobrok.heartbeats.failed.count")); - metrics.add(new Metric("slobrok.missing.consensus.count")); + addMetric(metrics, "slobrok.heartbeats.failed.count"); + addMetric(metrics, "slobrok.missing.consensus.count"); - metrics.add(new Metric("logd.processed.lines.count")); - metrics.add(new Metric("worker.connections.max")); - metrics.add(new Metric("endpoint.certificate.expiry.seconds")); + addMetric(metrics, "logd.processed.lines.count"); + addMetric(metrics, "worker.connections.max"); + addMetric(metrics, "endpoint.certificate.expiry.seconds"); // Java (JRT) TLS metrics - metrics.add(new Metric("jrt.transport.tls-certificate-verification-failures")); - metrics.add(new Metric("jrt.transport.peer-authorization-failures")); - metrics.add(new Metric("jrt.transport.server.tls-connections-established")); - metrics.add(new Metric("jrt.transport.client.tls-connections-established")); - metrics.add(new Metric("jrt.transport.server.unencrypted-connections-established")); - metrics.add(new Metric("jrt.transport.client.unencrypted-connections-established")); + addMetric(metrics, "jrt.transport.tls-certificate-verification-failures"); + addMetric(metrics, "jrt.transport.peer-authorization-failures"); + addMetric(metrics, "jrt.transport.server.tls-connections-established"); + addMetric(metrics, "jrt.transport.client.tls-connections-established"); + addMetric(metrics, "jrt.transport.server.unencrypted-connections-established"); + addMetric(metrics, "jrt.transport.client.unencrypted-connections-established"); // C++ TLS metrics - metrics.add(new Metric("vds.server.network.tls-handshakes-failed")); - metrics.add(new Metric("vds.server.network.peer-authorization-failures")); - metrics.add(new Metric("vds.server.network.client.tls-connections-established")); - metrics.add(new Metric("vds.server.network.server.tls-connections-established")); - metrics.add(new Metric("vds.server.network.client.insecure-connections-established")); - metrics.add(new Metric("vds.server.network.server.insecure-connections-established")); - metrics.add(new Metric("vds.server.network.tls-connections-broken")); - metrics.add(new Metric("vds.server.network.failed-tls-config-reloads")); + addMetric(metrics, "vds.server.network.tls-handshakes-failed"); + addMetric(metrics, "vds.server.network.peer-authorization-failures"); + addMetric(metrics, "vds.server.network.client.tls-connections-established"); + addMetric(metrics, "vds.server.network.server.tls-connections-established"); + addMetric(metrics, "vds.server.network.client.insecure-connections-established"); + addMetric(metrics, "vds.server.network.server.insecure-connections-established"); + addMetric(metrics, "vds.server.network.tls-connections-broken"); + addMetric(metrics, "vds.server.network.failed-tls-config-reloads"); // C++ Fnet metrics - metrics.add(new Metric("vds.server.fnet.num-connections")); + addMetric(metrics, "vds.server.fnet.num-connections"); // Node certificate - metrics.add(new Metric("node-certificate.expiry.seconds")); + addMetric(metrics, "node-certificate.expiry.seconds"); return metrics; } @@ -103,22 +103,22 @@ public class VespaMetricSet { private static Set<Metric> getConfigServerMetrics() { Set<Metric> metrics =new LinkedHashSet<>(); - metrics.add(new Metric("configserver.requests.count")); - metrics.add(new Metric("configserver.failedRequests.count")); - metrics.add(new Metric("configserver.latency.max")); - metrics.add(new Metric("configserver.latency.sum")); - metrics.add(new Metric("configserver.latency.count")); - metrics.add(new Metric("configserver.cacheConfigElems.last")); - metrics.add(new Metric("configserver.cacheChecksumElems.last")); - metrics.add(new Metric("configserver.hosts.last")); - metrics.add(new Metric("configserver.delayedResponses.count")); - metrics.add(new Metric("configserver.sessionChangeErrors.count")); - - metrics.add(new Metric("configserver.zkZNodes.last")); - metrics.add(new Metric("configserver.zkAvgLatency.last")); - metrics.add(new Metric("configserver.zkMaxLatency.last")); - metrics.add(new Metric("configserver.zkConnections.last")); - metrics.add(new Metric("configserver.zkOutstandingRequests.last")); + addMetric(metrics, "configserver.requests.count"); + addMetric(metrics, "configserver.failedRequests.count"); + addMetric(metrics, "configserver.latency.max"); + addMetric(metrics, "configserver.latency.sum"); + addMetric(metrics, "configserver.latency.count"); + addMetric(metrics, "configserver.cacheConfigElems.last"); + addMetric(metrics, "configserver.cacheChecksumElems.last"); + addMetric(metrics, "configserver.hosts.last"); + addMetric(metrics, "configserver.delayedResponses.count"); + addMetric(metrics, "configserver.sessionChangeErrors.count"); + + addMetric(metrics, "configserver.zkZNodes.last"); + addMetric(metrics, "configserver.zkAvgLatency.last"); + addMetric(metrics, "configserver.zkMaxLatency.last"); + addMetric(metrics, "configserver.zkConnections.last"); + addMetric(metrics, "configserver.zkOutstandingRequests.last"); return metrics; } @@ -325,30 +325,30 @@ public class VespaMetricSet { } private static void addSearchNodeExecutorMetrics(Set<Metric> metrics, String prefix) { - metrics.add(new Metric(prefix + ".queuesize.max")); - metrics.add(new Metric(prefix + ".queuesize.sum")); - metrics.add(new Metric(prefix + ".queuesize.count")); - metrics.add(new Metric(prefix + ".accepted.rate")); - metrics.add(new Metric(prefix + ".wakeups.rate")); - metrics.add(new Metric(prefix + ".utilization.max")); - metrics.add(new Metric(prefix + ".utilization.sum")); - metrics.add(new Metric(prefix + ".utilization.count")); + addMetric(metrics, prefix + ".queuesize.max"); + addMetric(metrics, prefix + ".queuesize.sum"); + addMetric(metrics, prefix + ".queuesize.count"); + addMetric(metrics, prefix + ".accepted.rate"); + addMetric(metrics, prefix + ".wakeups.rate"); + addMetric(metrics, prefix + ".utilization.max"); + addMetric(metrics, prefix + ".utilization.sum"); + addMetric(metrics, prefix + ".utilization.count"); } private static Set<Metric> getSearchNodeMetrics() { Set<Metric> metrics = new LinkedHashSet<>(); - metrics.add(new Metric("content.proton.documentdb.documents.total.last")); - metrics.add(new Metric("content.proton.documentdb.documents.ready.last")); - metrics.add(new Metric("content.proton.documentdb.documents.active.last")); - metrics.add(new Metric("content.proton.documentdb.documents.removed.last")); - - metrics.add(new Metric("content.proton.documentdb.index.docs_in_memory.last")); - metrics.add(new Metric("content.proton.documentdb.disk_usage.last")); - metrics.add(new Metric("content.proton.documentdb.memory_usage.allocated_bytes.max")); - metrics.add(new Metric("content.proton.documentdb.heart_beat_age.last")); - metrics.add(new Metric("content.proton.transport.query.count.rate")); - metrics.add(new Metric("content.proton.docsum.docs.rate")); + addMetric(metrics, "content.proton.documentdb.documents.total.last"); + addMetric(metrics, "content.proton.documentdb.documents.ready.last"); + addMetric(metrics, "content.proton.documentdb.documents.active.last"); + addMetric(metrics,"content.proton.documentdb.documents.removed.last"); + + addMetric(metrics, "content.proton.documentdb.index.docs_in_memory.last"); + addMetric(metrics, "content.proton.documentdb.disk_usage.last"); + addMetric(metrics,"content.proton.documentdb.memory_usage.allocated_bytes.max"); + addMetric(metrics, "content.proton.documentdb.heart_beat_age.last"); + addMetric(metrics, "content.proton.transport.query.count.rate"); + addMetric(metrics, "content.proton.docsum.docs.rate"); addMetric(metrics, "content.proton.docsum.latency", List.of("max", "sum", "count")); addMetric(metrics, "content.proton.transport.query.latency", List.of("max", "sum", "count")); @@ -359,7 +359,7 @@ public class VespaMetricSet { addMetric(metrics, "content.proton.search_protocol.docsum.latency", List.of("max", "sum", "count")); addMetric(metrics, "content.proton.search_protocol.docsum.request_size", List.of("max", "sum", "count")); addMetric(metrics, "content.proton.search_protocol.docsum.reply_size", List.of("max", "sum", "count")); - metrics.add(new Metric("content.proton.search_protocol.docsum.requested_documents.count")); + addMetric(metrics, "content.proton.search_protocol.docsum.requested_documents.count"); // Executors shared between all document dbs addSearchNodeExecutorMetrics(metrics, "content.proton.executor.proton"); @@ -371,15 +371,15 @@ public class VespaMetricSet { addSearchNodeExecutorMetrics(metrics, "content.proton.executor.field_writer"); // jobs - metrics.add(new Metric("content.proton.documentdb.job.total.average")); - metrics.add(new Metric("content.proton.documentdb.job.attribute_flush.average")); - metrics.add(new Metric("content.proton.documentdb.job.memory_index_flush.average")); - metrics.add(new Metric("content.proton.documentdb.job.disk_index_fusion.average")); - metrics.add(new Metric("content.proton.documentdb.job.document_store_flush.average")); - metrics.add(new Metric("content.proton.documentdb.job.document_store_compact.average")); - metrics.add(new Metric("content.proton.documentdb.job.bucket_move.average")); - metrics.add(new Metric("content.proton.documentdb.job.lid_space_compact.average")); - metrics.add(new Metric("content.proton.documentdb.job.removed_documents_prune.average")); + addMetric(metrics, "content.proton.documentdb.job.total.average"); + addMetric(metrics, "content.proton.documentdb.job.attribute_flush.average"); + addMetric(metrics, "content.proton.documentdb.job.memory_index_flush.average"); + addMetric(metrics, "content.proton.documentdb.job.disk_index_fusion.average"); + addMetric(metrics, "content.proton.documentdb.job.document_store_flush.average"); + addMetric(metrics, "content.proton.documentdb.job.document_store_compact.average"); + addMetric(metrics, "content.proton.documentdb.job.bucket_move.average"); + addMetric(metrics, "content.proton.documentdb.job.lid_space_compact.average"); + addMetric(metrics, "content.proton.documentdb.job.removed_documents_prune.average"); // Threading service (per document db) addSearchNodeExecutorMetrics(metrics, "content.proton.documentdb.threading_service.master"); @@ -390,40 +390,40 @@ public class VespaMetricSet { addSearchNodeExecutorMetrics(metrics, "content.proton.documentdb.threading_service.attribute_field_writer"); // lid space - metrics.add(new Metric("content.proton.documentdb.ready.lid_space.lid_bloat_factor.average")); - metrics.add(new Metric("content.proton.documentdb.notready.lid_space.lid_bloat_factor.average")); - metrics.add(new Metric("content.proton.documentdb.removed.lid_space.lid_bloat_factor.average")); - metrics.add(new Metric("content.proton.documentdb.ready.lid_space.lid_fragmentation_factor.average")); - metrics.add(new Metric("content.proton.documentdb.notready.lid_space.lid_fragmentation_factor.average")); - metrics.add(new Metric("content.proton.documentdb.removed.lid_space.lid_fragmentation_factor.average")); - metrics.add(new Metric("content.proton.documentdb.ready.lid_space.lid_limit.last")); - metrics.add(new Metric("content.proton.documentdb.notready.lid_space.lid_limit.last")); - metrics.add(new Metric("content.proton.documentdb.removed.lid_space.lid_limit.last")); - metrics.add(new Metric("content.proton.documentdb.ready.lid_space.highest_used_lid.last")); - metrics.add(new Metric("content.proton.documentdb.notready.lid_space.highest_used_lid.last")); - metrics.add(new Metric("content.proton.documentdb.removed.lid_space.highest_used_lid.last")); - metrics.add(new Metric("content.proton.documentdb.ready.lid_space.used_lids.last")); - metrics.add(new Metric("content.proton.documentdb.notready.lid_space.used_lids.last")); - metrics.add(new Metric("content.proton.documentdb.removed.lid_space.used_lids.last")); + addMetric(metrics, "content.proton.documentdb.ready.lid_space.lid_bloat_factor.average"); + addMetric(metrics, "content.proton.documentdb.notready.lid_space.lid_bloat_factor.average"); + addMetric(metrics, "content.proton.documentdb.removed.lid_space.lid_bloat_factor.average"); + addMetric(metrics, "content.proton.documentdb.ready.lid_space.lid_fragmentation_factor.average"); + addMetric(metrics, "content.proton.documentdb.notready.lid_space.lid_fragmentation_factor.average"); + addMetric(metrics, "content.proton.documentdb.removed.lid_space.lid_fragmentation_factor.average"); + addMetric(metrics, "content.proton.documentdb.ready.lid_space.lid_limit.last"); + addMetric(metrics, "content.proton.documentdb.notready.lid_space.lid_limit.last"); + addMetric(metrics, "content.proton.documentdb.removed.lid_space.lid_limit.last"); + addMetric(metrics, "content.proton.documentdb.ready.lid_space.highest_used_lid.last"); + addMetric(metrics, "content.proton.documentdb.notready.lid_space.highest_used_lid.last"); + addMetric(metrics, "content.proton.documentdb.removed.lid_space.highest_used_lid.last"); + addMetric(metrics, "content.proton.documentdb.ready.lid_space.used_lids.last"); + addMetric(metrics, "content.proton.documentdb.notready.lid_space.used_lids.last"); + addMetric(metrics, "content.proton.documentdb.removed.lid_space.used_lids.last"); // bucket move - metrics.add(new Metric("content.proton.documentdb.bucket_move.buckets_pending.last")); + addMetric(metrics, "content.proton.documentdb.bucket_move.buckets_pending.last"); // resource usage - metrics.add(new Metric("content.proton.resource_usage.disk.average")); - metrics.add(new Metric("content.proton.resource_usage.disk_usage.total.max")); - metrics.add(new Metric("content.proton.resource_usage.disk_usage.total_utilization.max")); - metrics.add(new Metric("content.proton.resource_usage.disk_usage.transient.max")); - metrics.add(new Metric("content.proton.resource_usage.memory.average")); - metrics.add(new Metric("content.proton.resource_usage.memory_usage.total.max")); - metrics.add(new Metric("content.proton.resource_usage.memory_usage.total_utilization.max")); - metrics.add(new Metric("content.proton.resource_usage.memory_usage.transient.max")); - metrics.add(new Metric("content.proton.resource_usage.memory_mappings.max")); - metrics.add(new Metric("content.proton.resource_usage.open_file_descriptors.max")); - metrics.add(new Metric("content.proton.resource_usage.feeding_blocked.max")); - metrics.add(new Metric("content.proton.resource_usage.malloc_arena.max")); - metrics.add(new Metric("content.proton.documentdb.attribute.resource_usage.address_space.max")); - metrics.add(new Metric("content.proton.documentdb.attribute.resource_usage.feeding_blocked.max")); + addMetric(metrics, "content.proton.resource_usage.disk.average"); + addMetric(metrics, "content.proton.resource_usage.disk_usage.total.max"); + addMetric(metrics, "content.proton.resource_usage.disk_usage.total_utilization.max"); + addMetric(metrics, "content.proton.resource_usage.disk_usage.transient.max"); + addMetric(metrics, "content.proton.resource_usage.memory.average"); + addMetric(metrics, "content.proton.resource_usage.memory_usage.total.max"); + addMetric(metrics, "content.proton.resource_usage.memory_usage.total_utilization.max"); + addMetric(metrics, "content.proton.resource_usage.memory_usage.transient.max"); + addMetric(metrics, "content.proton.resource_usage.memory_mappings.max"); + addMetric(metrics, "content.proton.resource_usage.open_file_descriptors.max"); + addMetric(metrics, "content.proton.resource_usage.feeding_blocked.max"); + addMetric(metrics, "content.proton.resource_usage.malloc_arena.max"); + addMetric(metrics, "content.proton.documentdb.attribute.resource_usage.address_space.max"); + addMetric(metrics, "content.proton.documentdb.attribute.resource_usage.feeding_blocked.max"); // CPU util addMetric(metrics, "content.proton.resource_usage.cpu_util.setup", List.of("max", "sum", "count")); @@ -433,74 +433,74 @@ public class VespaMetricSet { addMetric(metrics, "content.proton.resource_usage.cpu_util.other", List.of("max", "sum", "count")); // transaction log - metrics.add(new Metric("content.proton.transactionlog.entries.average")); - metrics.add(new Metric("content.proton.transactionlog.disk_usage.average")); - metrics.add(new Metric("content.proton.transactionlog.replay_time.last")); + addMetric(metrics, "content.proton.transactionlog.entries.average"); + addMetric(metrics, "content.proton.transactionlog.disk_usage.average"); + addMetric(metrics, "content.proton.transactionlog.replay_time.last"); // document store - metrics.add(new Metric("content.proton.documentdb.ready.document_store.disk_usage.average")); - metrics.add(new Metric("content.proton.documentdb.ready.document_store.disk_bloat.average")); - metrics.add(new Metric("content.proton.documentdb.ready.document_store.max_bucket_spread.average")); - metrics.add(new Metric("content.proton.documentdb.ready.document_store.memory_usage.allocated_bytes.average")); - metrics.add(new Metric("content.proton.documentdb.ready.document_store.memory_usage.used_bytes.average")); - metrics.add(new Metric("content.proton.documentdb.ready.document_store.memory_usage.dead_bytes.average")); - metrics.add(new Metric("content.proton.documentdb.ready.document_store.memory_usage.onhold_bytes.average")); - metrics.add(new Metric("content.proton.documentdb.notready.document_store.disk_usage.average")); - metrics.add(new Metric("content.proton.documentdb.notready.document_store.disk_bloat.average")); - metrics.add(new Metric("content.proton.documentdb.notready.document_store.max_bucket_spread.average")); - metrics.add(new Metric("content.proton.documentdb.notready.document_store.memory_usage.allocated_bytes.average")); - metrics.add(new Metric("content.proton.documentdb.notready.document_store.memory_usage.used_bytes.average")); - metrics.add(new Metric("content.proton.documentdb.notready.document_store.memory_usage.dead_bytes.average")); - metrics.add(new Metric("content.proton.documentdb.notready.document_store.memory_usage.onhold_bytes.average")); - metrics.add(new Metric("content.proton.documentdb.removed.document_store.disk_usage.average")); - metrics.add(new Metric("content.proton.documentdb.removed.document_store.disk_bloat.average")); - metrics.add(new Metric("content.proton.documentdb.removed.document_store.max_bucket_spread.average")); - metrics.add(new Metric("content.proton.documentdb.removed.document_store.memory_usage.allocated_bytes.average")); - metrics.add(new Metric("content.proton.documentdb.removed.document_store.memory_usage.used_bytes.average")); - metrics.add(new Metric("content.proton.documentdb.removed.document_store.memory_usage.dead_bytes.average")); - metrics.add(new Metric("content.proton.documentdb.removed.document_store.memory_usage.onhold_bytes.average")); + addMetric(metrics, "content.proton.documentdb.ready.document_store.disk_usage.average"); + addMetric(metrics, "content.proton.documentdb.ready.document_store.disk_bloat.average"); + addMetric(metrics, "content.proton.documentdb.ready.document_store.max_bucket_spread.average"); + addMetric(metrics, "content.proton.documentdb.ready.document_store.memory_usage.allocated_bytes.average"); + addMetric(metrics, "content.proton.documentdb.ready.document_store.memory_usage.used_bytes.average"); + addMetric(metrics, "content.proton.documentdb.ready.document_store.memory_usage.dead_bytes.average"); + addMetric(metrics, "content.proton.documentdb.ready.document_store.memory_usage.onhold_bytes.average"); + addMetric(metrics, "content.proton.documentdb.notready.document_store.disk_usage.average"); + addMetric(metrics, "content.proton.documentdb.notready.document_store.disk_bloat.average"); + addMetric(metrics, "content.proton.documentdb.notready.document_store.max_bucket_spread.average"); + addMetric(metrics, "content.proton.documentdb.notready.document_store.memory_usage.allocated_bytes.average"); + addMetric(metrics, "content.proton.documentdb.notready.document_store.memory_usage.used_bytes.average"); + addMetric(metrics, "content.proton.documentdb.notready.document_store.memory_usage.dead_bytes.average"); + addMetric(metrics, "content.proton.documentdb.notready.document_store.memory_usage.onhold_bytes.average"); + addMetric(metrics, "content.proton.documentdb.removed.document_store.disk_usage.average"); + addMetric(metrics, "content.proton.documentdb.removed.document_store.disk_bloat.average"); + addMetric(metrics, "content.proton.documentdb.removed.document_store.max_bucket_spread.average"); + addMetric(metrics, "content.proton.documentdb.removed.document_store.memory_usage.allocated_bytes.average"); + addMetric(metrics, "content.proton.documentdb.removed.document_store.memory_usage.used_bytes.average"); + addMetric(metrics, "content.proton.documentdb.removed.document_store.memory_usage.dead_bytes.average"); + addMetric(metrics, "content.proton.documentdb.removed.document_store.memory_usage.onhold_bytes.average"); // document store cache - metrics.add(new Metric("content.proton.documentdb.ready.document_store.cache.memory_usage.average")); - metrics.add(new Metric("content.proton.documentdb.ready.document_store.cache.hit_rate.average")); - metrics.add(new Metric("content.proton.documentdb.ready.document_store.cache.lookups.rate")); - metrics.add(new Metric("content.proton.documentdb.ready.document_store.cache.invalidations.rate")); - metrics.add(new Metric("content.proton.documentdb.notready.document_store.cache.memory_usage.average")); - metrics.add(new Metric("content.proton.documentdb.notready.document_store.cache.hit_rate.average")); - metrics.add(new Metric("content.proton.documentdb.notready.document_store.cache.lookups.rate")); - metrics.add(new Metric("content.proton.documentdb.notready.document_store.cache.invalidations.rate")); + addMetric(metrics, "content.proton.documentdb.ready.document_store.cache.memory_usage.average"); + addMetric(metrics, "content.proton.documentdb.ready.document_store.cache.hit_rate.average"); + addMetric(metrics, "content.proton.documentdb.ready.document_store.cache.lookups.rate"); + addMetric(metrics, "content.proton.documentdb.ready.document_store.cache.invalidations.rate"); + addMetric(metrics, "content.proton.documentdb.notready.document_store.cache.memory_usage.average"); + addMetric(metrics, "content.proton.documentdb.notready.document_store.cache.hit_rate.average"); + addMetric(metrics, "content.proton.documentdb.notready.document_store.cache.lookups.rate"); + addMetric(metrics, "content.proton.documentdb.notready.document_store.cache.invalidations.rate"); // attribute - metrics.add(new Metric("content.proton.documentdb.ready.attribute.memory_usage.allocated_bytes.average")); - metrics.add(new Metric("content.proton.documentdb.ready.attribute.memory_usage.used_bytes.average")); - metrics.add(new Metric("content.proton.documentdb.ready.attribute.memory_usage.dead_bytes.average")); - metrics.add(new Metric("content.proton.documentdb.ready.attribute.memory_usage.onhold_bytes.average")); - metrics.add(new Metric("content.proton.documentdb.notready.attribute.memory_usage.allocated_bytes.average")); - metrics.add(new Metric("content.proton.documentdb.notready.attribute.memory_usage.used_bytes.average")); - metrics.add(new Metric("content.proton.documentdb.notready.attribute.memory_usage.dead_bytes.average")); - metrics.add(new Metric("content.proton.documentdb.notready.attribute.memory_usage.onhold_bytes.average")); + addMetric(metrics, "content.proton.documentdb.ready.attribute.memory_usage.allocated_bytes.average"); + addMetric(metrics, "content.proton.documentdb.ready.attribute.memory_usage.used_bytes.average"); + addMetric(metrics, "content.proton.documentdb.ready.attribute.memory_usage.dead_bytes.average"); + addMetric(metrics, "content.proton.documentdb.ready.attribute.memory_usage.onhold_bytes.average"); + addMetric(metrics, "content.proton.documentdb.notready.attribute.memory_usage.allocated_bytes.average"); + addMetric(metrics, "content.proton.documentdb.notready.attribute.memory_usage.used_bytes.average"); + addMetric(metrics, "content.proton.documentdb.notready.attribute.memory_usage.dead_bytes.average"); + addMetric(metrics, "content.proton.documentdb.notready.attribute.memory_usage.onhold_bytes.average"); // index - metrics.add(new Metric("content.proton.documentdb.index.memory_usage.allocated_bytes.average")); - metrics.add(new Metric("content.proton.documentdb.index.memory_usage.used_bytes.average")); - metrics.add(new Metric("content.proton.documentdb.index.memory_usage.dead_bytes.average")); - metrics.add(new Metric("content.proton.documentdb.index.memory_usage.onhold_bytes.average")); + addMetric(metrics, "content.proton.documentdb.index.memory_usage.allocated_bytes.average"); + addMetric(metrics, "content.proton.documentdb.index.memory_usage.used_bytes.average"); + addMetric(metrics, "content.proton.documentdb.index.memory_usage.dead_bytes.average"); + addMetric(metrics, "content.proton.documentdb.index.memory_usage.onhold_bytes.average"); // matching - metrics.add(new Metric("content.proton.documentdb.matching.queries.rate")); - metrics.add(new Metric("content.proton.documentdb.matching.soft_doomed_queries.rate")); + addMetric(metrics, "content.proton.documentdb.matching.queries.rate"); + addMetric(metrics, "content.proton.documentdb.matching.soft_doomed_queries.rate"); addMetric(metrics, "content.proton.documentdb.matching.query_latency", List.of("max", "sum", "count")); addMetric(metrics, "content.proton.documentdb.matching.query_setup_time", List.of("max", "sum", "count")); addMetric(metrics, "content.proton.documentdb.matching.docs_matched", List.of("rate", "count")); - metrics.add(new Metric("content.proton.documentdb.matching.rank_profile.queries.rate")); - metrics.add(new Metric("content.proton.documentdb.matching.rank_profile.soft_doomed_queries.rate")); + addMetric(metrics, "content.proton.documentdb.matching.rank_profile.queries.rate"); + addMetric(metrics, "content.proton.documentdb.matching.rank_profile.soft_doomed_queries.rate"); addMetric(metrics, "content.proton.documentdb.matching.rank_profile.soft_doom_factor", List.of("min", "max", "sum", "count")); addMetric(metrics, "content.proton.documentdb.matching.rank_profile.query_latency", List.of("max", "sum", "count")); addMetric(metrics, "content.proton.documentdb.matching.rank_profile.query_setup_time", List.of("max", "sum", "count")); addMetric(metrics, "content.proton.documentdb.matching.rank_profile.grouping_time", List.of("max", "sum", "count")); addMetric(metrics, "content.proton.documentdb.matching.rank_profile.rerank_time", List.of("max", "sum", "count")); addMetric(metrics, "content.proton.documentdb.matching.rank_profile.docs_matched", List.of("rate", "count")); - metrics.add(new Metric("content.proton.documentdb.matching.rank_profile.limited_queries.rate")); + addMetric(metrics, "content.proton.documentdb.matching.rank_profile.limited_queries.rate"); // feeding addMetric(metrics, "content.proton.documentdb.feeding.commit.operations", List.of("max", "sum", "count", "rate")); @@ -515,15 +515,15 @@ public class VespaMetricSet { // TODO: For the purpose of this file and likely elsewhere, all but the last aggregate specifier, // TODO: such as 'average' and 'sum' in the metric names below are just confusing and can be mentally // TODO: disregarded when considering metric names. Consider cleaning up for Vespa 9. - metrics.add(new Metric("vds.datastored.alldisks.buckets.average")); - metrics.add(new Metric("vds.datastored.alldisks.docs.average")); - metrics.add(new Metric("vds.datastored.alldisks.bytes.average")); + addMetric(metrics, "vds.datastored.alldisks.buckets.average"); + addMetric(metrics, "vds.datastored.alldisks.docs.average"); + addMetric(metrics, "vds.datastored.alldisks.bytes.average"); addMetric(metrics, "vds.visitor.allthreads.averagevisitorlifetime", List.of("max", "sum", "count")); addMetric(metrics, "vds.visitor.allthreads.averagequeuewait", List.of("max", "sum", "count")); addMetric(metrics, "vds.visitor.allthreads.queuesize", List.of("max", "sum", "count")); - metrics.add(new Metric("vds.visitor.allthreads.completed.rate")); - metrics.add(new Metric("vds.visitor.allthreads.created.rate")); - metrics.add(new Metric("vds.visitor.allthreads.failed.rate")); + addMetric(metrics, "vds.visitor.allthreads.completed.rate"); + addMetric(metrics, "vds.visitor.allthreads.created.rate"); + addMetric(metrics, "vds.visitor.allthreads.failed.rate"); addMetric(metrics, "vds.visitor.allthreads.averagemessagesendtime", List.of("max", "sum", "count")); addMetric(metrics, "vds.visitor.allthreads.averageprocessingtime", List.of("max", "sum", "count")); @@ -539,137 +539,137 @@ public class VespaMetricSet { addMetric(metrics, "vds.filestor.allthreads.mergedatawritelatency", List.of("max", "sum", "count")); addMetric(metrics, "vds.filestor.allthreads.put_latency", List.of("max", "sum", "count")); addMetric(metrics, "vds.filestor.allthreads.remove_latency", List.of("max", "sum", "count")); - metrics.add(new Metric("vds.filestor.allstripes.throttled_rpc_direct_dispatches.rate")); - metrics.add(new Metric("vds.filestor.allstripes.throttled_persistence_thread_polls.rate")); - metrics.add(new Metric("vds.filestor.allstripes.timeouts_waiting_for_throttle_token.rate")); + addMetric(metrics, "vds.filestor.allstripes.throttled_rpc_direct_dispatches.rate"); + addMetric(metrics, "vds.filestor.allstripes.throttled_persistence_thread_polls.rate"); + addMetric(metrics, "vds.filestor.allstripes.timeouts_waiting_for_throttle_token.rate"); - metrics.add(new Metric("vds.filestor.allthreads.put.count.rate")); - metrics.add(new Metric("vds.filestor.allthreads.put.failed.rate")); - metrics.add(new Metric("vds.filestor.allthreads.put.test_and_set_failed.rate")); + addMetric(metrics, "vds.filestor.allthreads.put.count.rate"); + addMetric(metrics, "vds.filestor.allthreads.put.failed.rate"); + addMetric(metrics, "vds.filestor.allthreads.put.test_and_set_failed.rate"); addMetric(metrics, "vds.filestor.allthreads.put.latency", List.of("max", "sum", "count")); addMetric(metrics, "vds.filestor.allthreads.put.request_size", List.of("max", "sum", "count")); - metrics.add(new Metric("vds.filestor.allthreads.remove.count.rate")); - metrics.add(new Metric("vds.filestor.allthreads.remove.failed.rate")); - metrics.add(new Metric("vds.filestor.allthreads.remove.test_and_set_failed.rate")); + addMetric(metrics, "vds.filestor.allthreads.remove.count.rate"); + addMetric(metrics, "vds.filestor.allthreads.remove.failed.rate"); + addMetric(metrics, "vds.filestor.allthreads.remove.test_and_set_failed.rate"); addMetric(metrics, "vds.filestor.allthreads.remove.latency", List.of("max", "sum", "count")); addMetric(metrics, "vds.filestor.allthreads.remove.request_size", List.of("max", "sum", "count")); - metrics.add(new Metric("vds.filestor.allthreads.get.count.rate")); - metrics.add(new Metric("vds.filestor.allthreads.get.failed.rate")); + addMetric(metrics, "vds.filestor.allthreads.get.count.rate"); + addMetric(metrics, "vds.filestor.allthreads.get.failed.rate"); addMetric(metrics, "vds.filestor.allthreads.get.latency", List.of("max", "sum", "count")); addMetric(metrics, "vds.filestor.allthreads.get.request_size", List.of("max", "sum", "count")); - metrics.add(new Metric("vds.filestor.allthreads.update.count.rate")); - metrics.add(new Metric("vds.filestor.allthreads.update.failed.rate")); - metrics.add(new Metric("vds.filestor.allthreads.update.test_and_set_failed.rate")); + addMetric(metrics, "vds.filestor.allthreads.update.count.rate"); + addMetric(metrics, "vds.filestor.allthreads.update.failed.rate"); + addMetric(metrics, "vds.filestor.allthreads.update.test_and_set_failed.rate"); addMetric(metrics, "vds.filestor.allthreads.update.latency", List.of("max", "sum", "count")); addMetric(metrics, "vds.filestor.allthreads.update.request_size", List.of("max", "sum", "count")); addMetric(metrics, "vds.filestor.allthreads.createiterator.latency", List.of("max", "sum", "count")); - metrics.add(new Metric("vds.filestor.allthreads.createiterator.count.rate")); - metrics.add(new Metric("vds.filestor.allthreads.visit.count.rate")); + addMetric(metrics, "vds.filestor.allthreads.createiterator.count.rate"); + addMetric(metrics, "vds.filestor.allthreads.visit.count.rate"); addMetric(metrics, "vds.filestor.allthreads.visit.latency", List.of("max", "sum", "count")); - metrics.add(new Metric("vds.filestor.allthreads.remove_location.count.rate")); + addMetric(metrics, "vds.filestor.allthreads.remove_location.count.rate"); addMetric(metrics, "vds.filestor.allthreads.remove_location.latency", List.of("max", "sum", "count")); - metrics.add(new Metric("vds.filestor.allthreads.splitbuckets.count.rate")); - metrics.add(new Metric("vds.filestor.allthreads.joinbuckets.count.rate")); - metrics.add(new Metric("vds.filestor.allthreads.deletebuckets.count.rate")); - metrics.add(new Metric("vds.filestor.allthreads.deletebuckets.failed.rate")); + addMetric(metrics, "vds.filestor.allthreads.splitbuckets.count.rate"); + addMetric(metrics, "vds.filestor.allthreads.joinbuckets.count.rate"); + addMetric(metrics, "vds.filestor.allthreads.deletebuckets.count.rate"); + addMetric(metrics, "vds.filestor.allthreads.deletebuckets.failed.rate"); addMetric(metrics, "vds.filestor.allthreads.deletebuckets.latency", List.of("max", "sum", "count")); - metrics.add(new Metric("vds.filestor.allthreads.setbucketstates.count.rate")); + addMetric(metrics, "vds.filestor.allthreads.setbucketstates.count.rate"); return metrics; } private static Set<Metric> getDistributorMetrics() { Set<Metric> metrics = new LinkedHashSet<>(); - metrics.add(new Metric("vds.idealstate.buckets_rechecking.average")); - metrics.add(new Metric("vds.idealstate.idealstate_diff.average")); - metrics.add(new Metric("vds.idealstate.buckets_toofewcopies.average")); - metrics.add(new Metric("vds.idealstate.buckets_toomanycopies.average")); - metrics.add(new Metric("vds.idealstate.buckets.average")); - metrics.add(new Metric("vds.idealstate.buckets_notrusted.average")); - metrics.add(new Metric("vds.idealstate.bucket_replicas_moving_out.average")); - metrics.add(new Metric("vds.idealstate.bucket_replicas_copying_out.average")); - metrics.add(new Metric("vds.idealstate.bucket_replicas_copying_in.average")); - metrics.add(new Metric("vds.idealstate.bucket_replicas_syncing.average")); - metrics.add(new Metric("vds.idealstate.max_observed_time_since_last_gc_sec.average")); - metrics.add(new Metric("vds.idealstate.delete_bucket.done_ok.rate")); - metrics.add(new Metric("vds.idealstate.delete_bucket.done_failed.rate")); - metrics.add(new Metric("vds.idealstate.delete_bucket.pending.average")); - metrics.add(new Metric("vds.idealstate.merge_bucket.done_ok.rate")); - metrics.add(new Metric("vds.idealstate.merge_bucket.done_failed.rate")); - metrics.add(new Metric("vds.idealstate.merge_bucket.pending.average")); - metrics.add(new Metric("vds.idealstate.merge_bucket.blocked.rate")); - metrics.add(new Metric("vds.idealstate.merge_bucket.throttled.rate")); - metrics.add(new Metric("vds.idealstate.merge_bucket.source_only_copy_changed.rate")); - metrics.add(new Metric("vds.idealstate.merge_bucket.source_only_copy_delete_blocked.rate")); - metrics.add(new Metric("vds.idealstate.merge_bucket.source_only_copy_delete_failed.rate")); - metrics.add(new Metric("vds.idealstate.split_bucket.done_ok.rate")); - metrics.add(new Metric("vds.idealstate.split_bucket.done_failed.rate")); - metrics.add(new Metric("vds.idealstate.split_bucket.pending.average")); - metrics.add(new Metric("vds.idealstate.join_bucket.done_ok.rate")); - metrics.add(new Metric("vds.idealstate.join_bucket.done_failed.rate")); - metrics.add(new Metric("vds.idealstate.join_bucket.pending.average")); - metrics.add(new Metric("vds.idealstate.garbage_collection.done_ok.rate")); - metrics.add(new Metric("vds.idealstate.garbage_collection.done_failed.rate")); - metrics.add(new Metric("vds.idealstate.garbage_collection.pending.average")); + addMetric(metrics, "vds.idealstate.buckets_rechecking.average"); + addMetric(metrics, "vds.idealstate.idealstate_diff.average"); + addMetric(metrics, "vds.idealstate.buckets_toofewcopies.average"); + addMetric(metrics, "vds.idealstate.buckets_toomanycopies.average"); + addMetric(metrics, "vds.idealstate.buckets.average"); + addMetric(metrics, "vds.idealstate.buckets_notrusted.average"); + addMetric(metrics, "vds.idealstate.bucket_replicas_moving_out.average"); + addMetric(metrics, "vds.idealstate.bucket_replicas_copying_out.average"); + addMetric(metrics, "vds.idealstate.bucket_replicas_copying_in.average"); + addMetric(metrics, "vds.idealstate.bucket_replicas_syncing.average"); + addMetric(metrics, "vds.idealstate.max_observed_time_since_last_gc_sec.average"); + addMetric(metrics, "vds.idealstate.delete_bucket.done_ok.rate"); + addMetric(metrics, "vds.idealstate.delete_bucket.done_failed.rate"); + addMetric(metrics, "vds.idealstate.delete_bucket.pending.average"); + addMetric(metrics, "vds.idealstate.merge_bucket.done_ok.rate"); + addMetric(metrics, "vds.idealstate.merge_bucket.done_failed.rate"); + addMetric(metrics, "vds.idealstate.merge_bucket.pending.average"); + addMetric(metrics, "vds.idealstate.merge_bucket.blocked.rate"); + addMetric(metrics, "vds.idealstate.merge_bucket.throttled.rate"); + addMetric(metrics, "vds.idealstate.merge_bucket.source_only_copy_changed.rate"); + addMetric(metrics, "vds.idealstate.merge_bucket.source_only_copy_delete_blocked.rate"); + addMetric(metrics, "vds.idealstate.merge_bucket.source_only_copy_delete_failed.rate"); + addMetric(metrics, "vds.idealstate.split_bucket.done_ok.rate"); + addMetric(metrics, "vds.idealstate.split_bucket.done_failed.rate"); + addMetric(metrics, "vds.idealstate.split_bucket.pending.average"); + addMetric(metrics, "vds.idealstate.join_bucket.done_ok.rate"); + addMetric(metrics, "vds.idealstate.join_bucket.done_failed.rate"); + addMetric(metrics, "vds.idealstate.join_bucket.pending.average"); + addMetric(metrics, "vds.idealstate.garbage_collection.done_ok.rate"); + addMetric(metrics, "vds.idealstate.garbage_collection.done_failed.rate"); + addMetric(metrics, "vds.idealstate.garbage_collection.pending.average"); addMetric(metrics, "vds.idealstate.garbage_collection.documents_removed", List.of("count", "rate")); addMetric(metrics, "vds.distributor.puts.latency", List.of("max", "sum", "count")); - metrics.add(new Metric("vds.distributor.puts.ok.rate")); - metrics.add(new Metric("vds.distributor.puts.failures.total.rate")); - metrics.add(new Metric("vds.distributor.puts.failures.notfound.rate")); - metrics.add(new Metric("vds.distributor.puts.failures.test_and_set_failed.rate")); - metrics.add(new Metric("vds.distributor.puts.failures.concurrent_mutations.rate")); - metrics.add(new Metric("vds.distributor.puts.failures.notconnected.rate")); - metrics.add(new Metric("vds.distributor.puts.failures.notready.rate")); - metrics.add(new Metric("vds.distributor.puts.failures.wrongdistributor.rate")); - metrics.add(new Metric("vds.distributor.puts.failures.safe_time_not_reached.rate")); - metrics.add(new Metric("vds.distributor.puts.failures.storagefailure.rate")); - metrics.add(new Metric("vds.distributor.puts.failures.timeout.rate")); - metrics.add(new Metric("vds.distributor.puts.failures.busy.rate")); - metrics.add(new Metric("vds.distributor.puts.failures.inconsistent_bucket.rate")); + addMetric(metrics, "vds.distributor.puts.ok.rate"); + addMetric(metrics, "vds.distributor.puts.failures.total.rate"); + addMetric(metrics, "vds.distributor.puts.failures.notfound.rate"); + addMetric(metrics, "vds.distributor.puts.failures.test_and_set_failed.rate"); + addMetric(metrics, "vds.distributor.puts.failures.concurrent_mutations.rate"); + addMetric(metrics, "vds.distributor.puts.failures.notconnected.rate"); + addMetric(metrics, "vds.distributor.puts.failures.notready.rate"); + addMetric(metrics, "vds.distributor.puts.failures.wrongdistributor.rate"); + addMetric(metrics, "vds.distributor.puts.failures.safe_time_not_reached.rate"); + addMetric(metrics, "vds.distributor.puts.failures.storagefailure.rate"); + addMetric(metrics, "vds.distributor.puts.failures.timeout.rate"); + addMetric(metrics, "vds.distributor.puts.failures.busy.rate"); + addMetric(metrics, "vds.distributor.puts.failures.inconsistent_bucket.rate"); addMetric(metrics, "vds.distributor.removes.latency", List.of("max", "sum", "count")); - metrics.add(new Metric("vds.distributor.removes.ok.rate")); - metrics.add(new Metric("vds.distributor.removes.failures.total.rate")); - metrics.add(new Metric("vds.distributor.removes.failures.notfound.rate")); - metrics.add(new Metric("vds.distributor.removes.failures.test_and_set_failed.rate")); - metrics.add(new Metric("vds.distributor.removes.failures.concurrent_mutations.rate")); + addMetric(metrics, "vds.distributor.removes.ok.rate"); + addMetric(metrics, "vds.distributor.removes.failures.total.rate"); + addMetric(metrics, "vds.distributor.removes.failures.notfound.rate"); + addMetric(metrics, "vds.distributor.removes.failures.test_and_set_failed.rate"); + addMetric(metrics, "vds.distributor.removes.failures.concurrent_mutations.rate"); addMetric(metrics, "vds.distributor.updates.latency", List.of("max", "sum", "count")); - metrics.add(new Metric("vds.distributor.updates.ok.rate")); - metrics.add(new Metric("vds.distributor.updates.failures.total.rate")); - metrics.add(new Metric("vds.distributor.updates.failures.notfound.rate")); - metrics.add(new Metric("vds.distributor.updates.failures.test_and_set_failed.rate")); - metrics.add(new Metric("vds.distributor.updates.failures.concurrent_mutations.rate")); - metrics.add(new Metric("vds.distributor.updates.diverging_timestamp_updates.rate")); - metrics.add(new Metric("vds.distributor.removelocations.ok.rate")); - metrics.add(new Metric("vds.distributor.removelocations.failures.total.rate")); + addMetric(metrics, "vds.distributor.updates.ok.rate"); + addMetric(metrics, "vds.distributor.updates.failures.total.rate"); + addMetric(metrics, "vds.distributor.updates.failures.notfound.rate"); + addMetric(metrics, "vds.distributor.updates.failures.test_and_set_failed.rate"); + addMetric(metrics, "vds.distributor.updates.failures.concurrent_mutations.rate"); + addMetric(metrics, "vds.distributor.updates.diverging_timestamp_updates.rate"); + addMetric(metrics, "vds.distributor.removelocations.ok.rate"); + addMetric(metrics, "vds.distributor.removelocations.failures.total.rate"); addMetric(metrics, "vds.distributor.gets.latency", List.of("max", "sum", "count")); - metrics.add(new Metric("vds.distributor.gets.ok.rate")); - metrics.add(new Metric("vds.distributor.gets.failures.total.rate")); - metrics.add(new Metric("vds.distributor.gets.failures.notfound.rate")); + addMetric(metrics, "vds.distributor.gets.ok.rate"); + addMetric(metrics, "vds.distributor.gets.failures.total.rate"); + addMetric(metrics, "vds.distributor.gets.failures.notfound.rate"); addMetric(metrics, "vds.distributor.visitor.latency", List.of("max", "sum", "count")); - metrics.add(new Metric("vds.distributor.visitor.ok.rate")); - metrics.add(new Metric("vds.distributor.visitor.failures.total.rate")); - metrics.add(new Metric("vds.distributor.visitor.failures.notready.rate")); - metrics.add(new Metric("vds.distributor.visitor.failures.notconnected.rate")); - metrics.add(new Metric("vds.distributor.visitor.failures.wrongdistributor.rate")); - metrics.add(new Metric("vds.distributor.visitor.failures.safe_time_not_reached.rate")); - metrics.add(new Metric("vds.distributor.visitor.failures.storagefailure.rate")); - metrics.add(new Metric("vds.distributor.visitor.failures.timeout.rate")); - metrics.add(new Metric("vds.distributor.visitor.failures.busy.rate")); - metrics.add(new Metric("vds.distributor.visitor.failures.inconsistent_bucket.rate")); - metrics.add(new Metric("vds.distributor.visitor.failures.notfound.rate")); - - metrics.add(new Metric("vds.distributor.docsstored.average")); - metrics.add(new Metric("vds.distributor.bytesstored.average")); - - metrics.add(new Metric("vds.bouncer.clock_skew_aborts.count")); + addMetric(metrics, "vds.distributor.visitor.ok.rate"); + addMetric(metrics, "vds.distributor.visitor.failures.total.rate"); + addMetric(metrics, "vds.distributor.visitor.failures.notready.rate"); + addMetric(metrics, "vds.distributor.visitor.failures.notconnected.rate"); + addMetric(metrics, "vds.distributor.visitor.failures.wrongdistributor.rate"); + addMetric(metrics, "vds.distributor.visitor.failures.safe_time_not_reached.rate"); + addMetric(metrics, "vds.distributor.visitor.failures.storagefailure.rate"); + addMetric(metrics, "vds.distributor.visitor.failures.timeout.rate"); + addMetric(metrics, "vds.distributor.visitor.failures.busy.rate"); + addMetric(metrics, "vds.distributor.visitor.failures.inconsistent_bucket.rate"); + addMetric(metrics, "vds.distributor.visitor.failures.notfound.rate"); + + addMetric(metrics, "vds.distributor.docsstored.average"); + addMetric(metrics, "vds.distributor.bytesstored.average"); + + addMetric(metrics, "vds.bouncer.clock_skew_aborts.count"); addMetric(metrics, "vds.mergethrottler.averagequeuewaitingtime", List.of("max", "sum", "count")); addMetric(metrics, "vds.mergethrottler.queuesize", List.of("max", "sum", "count")); addMetric(metrics, "vds.mergethrottler.active_window_size", List.of("max", "sum", "count")); - metrics.add(new Metric("vds.mergethrottler.bounced_due_to_back_pressure.rate")); - metrics.add(new Metric("vds.mergethrottler.locallyexecutedmerges.ok.rate")); - metrics.add(new Metric("vds.mergethrottler.mergechains.ok.rate")); - metrics.add(new Metric("vds.mergethrottler.mergechains.failures.busy.rate")); - metrics.add(new Metric("vds.mergethrottler.mergechains.failures.total.rate")); + addMetric(metrics, "vds.mergethrottler.bounced_due_to_back_pressure.rate"); + addMetric(metrics, "vds.mergethrottler.locallyexecutedmerges.ok.rate"); + addMetric(metrics, "vds.mergethrottler.mergechains.ok.rate"); + addMetric(metrics, "vds.mergethrottler.mergechains.failures.busy.rate"); + addMetric(metrics, "vds.mergethrottler.mergechains.failures.total.rate"); return metrics; } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/configserver/ConfigserverCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/container/configserver/ConfigserverCluster.java index 8f257110a04..4bec9123b77 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/configserver/ConfigserverCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/configserver/ConfigserverCluster.java @@ -86,6 +86,7 @@ public class ConfigserverCluster extends AbstractConfigProducer } builder.dynamicReconfiguration(options.hostedVespa().orElse(false)); + builder.snapshotMethod(options.zooKeeperSnapshotMethod()); } @Override diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/configserver/option/CloudConfigOptions.java b/config-model/src/main/java/com/yahoo/vespa/model/container/configserver/option/CloudConfigOptions.java index f342aa1a2bf..256d39422ff 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/configserver/option/CloudConfigOptions.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/configserver/option/CloudConfigOptions.java @@ -37,4 +37,6 @@ public interface CloudConfigOptions { Optional<String> loadBalancerAddress(); Optional<String> athenzDnsSuffix(); Optional<String> ztsUrl(); + String zooKeeperSnapshotMethod(); + } diff --git a/config-model/src/test/java/com/yahoo/vespa/model/container/configserver/TestOptions.java b/config-model/src/test/java/com/yahoo/vespa/model/container/configserver/TestOptions.java index 95c3262f2f5..f84612fd598 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/container/configserver/TestOptions.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/container/configserver/TestOptions.java @@ -3,6 +3,7 @@ package com.yahoo.vespa.model.container.configserver; import com.yahoo.vespa.model.container.configserver.option.CloudConfigOptions; +import java.util.Objects; import java.util.Optional; /** @@ -17,6 +18,7 @@ public class TestOptions implements CloudConfigOptions { private Optional<String> region = Optional.empty(); private Optional<Boolean> useVespaVersionInRequest = Optional.empty(); private Optional<Boolean> hostedVespa = Optional.empty(); + private String zooKeeperSnapshotMethod = "gz"; @Override public Optional<Integer> rpcPort() { @@ -106,6 +108,9 @@ public class TestOptions implements CloudConfigOptions { return Optional.empty(); } + @Override + public String zooKeeperSnapshotMethod() { return zooKeeperSnapshotMethod; } + public TestOptions configServers(ConfigServer[] configServers) { this.configServers = configServers; return this; @@ -131,4 +136,10 @@ public class TestOptions implements CloudConfigOptions { return this; } + public TestOptions zooKeeperSnapshotMethod(String snapshotMethod) { + Objects.requireNonNull(snapshotMethod); + this.zooKeeperSnapshotMethod = snapshotMethod; + return this; + } + } diff --git a/config-provisioning/src/main/java/com/yahoo/config/provision/ZoneEndpoint.java b/config-provisioning/src/main/java/com/yahoo/config/provision/ZoneEndpoint.java index 09b71c6a982..9f114610e32 100644 --- a/config-provisioning/src/main/java/com/yahoo/config/provision/ZoneEndpoint.java +++ b/config-provisioning/src/main/java/com/yahoo/config/provision/ZoneEndpoint.java @@ -13,6 +13,7 @@ import java.util.Objects; public class ZoneEndpoint { public static final ZoneEndpoint defaultEndpoint = new ZoneEndpoint(true, false, List.of()); + public static final ZoneEndpoint privateEndpoint = new ZoneEndpoint(false, false, List.of()); private final boolean isPublicEndpoint; private final boolean isPrivateEndpoint; diff --git a/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/filedistribution/FileDistributionAndUrlDownload.java b/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/filedistribution/FileDistributionAndUrlDownload.java index 3107f9029d8..9e6a54d8e2c 100644 --- a/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/filedistribution/FileDistributionAndUrlDownload.java +++ b/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/filedistribution/FileDistributionAndUrlDownload.java @@ -6,14 +6,6 @@ import com.yahoo.jrt.Supervisor; import com.yahoo.vespa.filedistribution.FileDistributionConnectionPool; import com.yahoo.vespa.filedistribution.FileDownloader; import java.time.Duration; -import java.util.Arrays; -import java.util.Set; -import java.util.stream.Collectors; - -import static com.yahoo.vespa.filedistribution.FileReferenceData.CompressionType; -import static com.yahoo.vespa.filedistribution.FileReferenceData.CompressionType.gzip; -import static com.yahoo.vespa.filedistribution.FileReferenceData.CompressionType.lz4; -import static com.yahoo.vespa.filedistribution.FileReferenceData.CompressionType.zstd; /** * Keeps track of file distribution and url download rpc servers. @@ -41,18 +33,9 @@ public class FileDistributionAndUrlDownload { private FileDownloader createDownloader(Supervisor supervisor, ConfigSourceSet source) { return new FileDownloader(new FileDistributionConnectionPool(source, supervisor), supervisor, - Duration.ofMinutes(5), - acceptedCompressionTypes()); + Duration.ofMinutes(5)); } - private Set<CompressionType> acceptedCompressionTypes() { - Set<CompressionType> acceptedCompressionTypes = Set.of(gzip, lz4, zstd); - String env = System.getenv("VESPA_FILE_DISTRIBUTION_ACCEPTED_COMPRESSION_TYPES"); - if (env != null && ! env.isEmpty()) { - String[] types = env.split(","); - acceptedCompressionTypes = Arrays.stream(types).map(CompressionType::valueOf).collect(Collectors.toSet()); - } - return acceptedCompressionTypes; - } + } diff --git a/config/src/tests/configretriever/configretriever.cpp b/config/src/tests/configretriever/configretriever.cpp index cc36614d4c5..3e4cd9bf980 100644 --- a/config/src/tests/configretriever/configretriever.cpp +++ b/config/src/tests/configretriever/configretriever.cpp @@ -38,7 +38,7 @@ struct ConfigTestFixture { std::shared_ptr<IConfigContext> context; int idcounter; - ConfigTestFixture(const std::string & id) + explicit ConfigTestFixture(const std::string & id) : configId(id), bootstrapBuilder(), componentConfig(), @@ -141,10 +141,13 @@ public: Slime & getData() { return _data; } + ~FixedPayload() override; private: Slime _data; }; +FixedPayload::~FixedPayload() = default; + } ConfigValue createKeyValueV2(const vespalib::string & key, const vespalib::string & value) diff --git a/config/src/vespa/config/frt/protocol.h b/config/src/vespa/config/frt/protocol.h index f99a4aee8a3..b8d7a5ff7fc 100644 --- a/config/src/vespa/config/frt/protocol.h +++ b/config/src/vespa/config/frt/protocol.h @@ -13,9 +13,7 @@ namespace vespalib { } } -namespace config { - -namespace protocol { +namespace config::protocol { int readProtocolVersion(); int readTraceLevel(); @@ -87,6 +85,3 @@ DecompressedData decompress(const char * buf, uint32_t len, const CompressionTyp } } - -} - diff --git a/config/src/vespa/config/retriever/configsnapshot.cpp b/config/src/vespa/config/retriever/configsnapshot.cpp index 293a90ebbba..8c79aec22f4 100644 --- a/config/src/vespa/config/retriever/configsnapshot.cpp +++ b/config/src/vespa/config/retriever/configsnapshot.cpp @@ -237,10 +237,13 @@ public: } Slime & getData() { return _data; } + ~FixedPayload() override; private: Slime _data; }; +FixedPayload::~FixedPayload() = default; + } std::pair<int64_t, ConfigValue> diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/FileServer.java b/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/FileServer.java index 92e1101e398..57d57d16d2f 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/FileServer.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/FileServer.java @@ -20,8 +20,6 @@ import com.yahoo.vespa.filedistribution.FileReferenceData; import com.yahoo.vespa.filedistribution.FileReferenceDownload; import com.yahoo.vespa.filedistribution.LazyFileReferenceData; import com.yahoo.vespa.filedistribution.LazyTemporaryStorageFileReferenceData; -import com.yahoo.vespa.flags.FlagSource; -import com.yahoo.vespa.flags.Flags; import com.yahoo.yolean.Exceptions; import java.io.File; import java.io.IOException; @@ -29,14 +27,12 @@ import java.nio.file.Files; import java.nio.file.Path; import java.time.Duration; import java.time.Instant; -import java.util.LinkedHashSet; import java.util.List; import java.util.Set; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.logging.Level; import java.util.logging.Logger; -import java.util.stream.Collectors; import static com.yahoo.vespa.config.server.filedistribution.FileDistributionUtil.getOtherConfigServersInCluster; import static com.yahoo.vespa.filedistribution.FileReferenceData.CompressionType; @@ -50,6 +46,7 @@ public class FileServer { // Set this low, to make sure we don't wait for a long time trying to download file private static final Duration timeout = Duration.ofSeconds(10); + private static final List<CompressionType> compressionTypesToServe = compressionTypesAsList(List.of("zstd", "lz4", "gzip")); // In preferred order private final FileDirectory fileDirectory; private final ExecutorService executor; @@ -89,16 +86,8 @@ public class FileServer { @SuppressWarnings("WeakerAccess") // Created by dependency injection @Inject - public FileServer(ConfigserverConfig configserverConfig, FlagSource flagSource, FileDirectory fileDirectory) { - this(createFileDownloader(getOtherConfigServersInCluster(configserverConfig), - compressionTypes(Flags.FILE_DISTRIBUTION_ACCEPTED_COMPRESSION_TYPES.bindTo(flagSource).value())), - compressionTypesAsList(Flags.FILE_DISTRIBUTION_COMPRESSION_TYPES_TO_SERVE.bindTo(flagSource).value()), - fileDirectory); - } - - // For testing only - public FileServer(FileDirectory fileDirectory) { - this(createFileDownloader(List.of(), Set.of(gzip)), List.of(gzip), fileDirectory); + public FileServer(ConfigserverConfig configserverConfig, FileDirectory fileDirectory) { + this(createFileDownloader(getOtherConfigServersInCluster(configserverConfig)), compressionTypesToServe, fileDirectory); } FileServer(FileDownloader fileDownloader, List<CompressionType> compressionTypes, FileDirectory fileDirectory) { @@ -237,21 +226,9 @@ public class FileServer { executor.shutdown(); } - private static FileDownloader createFileDownloader(List<String> configServers, Set<CompressionType> acceptedCompressionTypes) { + private static FileDownloader createFileDownloader(List<String> configServers) { Supervisor supervisor = new Supervisor(new Transport("filedistribution-pool")).setDropEmptyBuffers(true); - - return new FileDownloader(configServers.isEmpty() - ? FileDownloader.emptyConnectionPool() - : createConnectionPool(configServers, supervisor), - supervisor, - timeout, - acceptedCompressionTypes); - } - - private static LinkedHashSet<CompressionType> compressionTypes(List<String> compressionTypes) { - return compressionTypes.stream() - .map(CompressionType::valueOf) - .collect(Collectors.toCollection(LinkedHashSet::new)); + return new FileDownloader(createConnectionPool(configServers, supervisor), supervisor, timeout); } private static List<CompressionType> compressionTypesAsList(List<String> compressionTypes) { diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/ApplicationPackageMaintainer.java b/configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/ApplicationPackageMaintainer.java index 346a462fe3e..2948b82dd96 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/ApplicationPackageMaintainer.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/ApplicationPackageMaintainer.java @@ -17,17 +17,13 @@ import com.yahoo.vespa.filedistribution.FileDistributionConnectionPool; import com.yahoo.vespa.filedistribution.FileDownloader; import com.yahoo.vespa.filedistribution.FileReferenceDownload; import com.yahoo.vespa.flags.FlagSource; -import com.yahoo.vespa.flags.Flags; import java.io.File; import java.time.Duration; import java.util.List; import java.util.Optional; -import java.util.Set; import java.util.logging.Logger; -import java.util.stream.Collectors; import static com.yahoo.vespa.config.server.filedistribution.FileDistributionUtil.fileReferenceExistsOnDisk; -import static com.yahoo.vespa.filedistribution.FileReferenceData.CompressionType; /** * Verifies that all active sessions has an application package on local disk. @@ -54,10 +50,7 @@ public class ApplicationPackageMaintainer extends ConfigServerMaintainer { super(applicationRepository, curator, flagSource, applicationRepository.clock(), interval, false); this.applicationRepository = applicationRepository; this.downloadDirectory = new File(Defaults.getDefaults().underVespaHome(applicationRepository.configserverConfig().fileReferencesDir())); - this.fileDownloader = createFileDownloader(otherConfigServersInCluster, - downloadDirectory, - supervisor, - Flags.FILE_DISTRIBUTION_ACCEPTED_COMPRESSION_TYPES.bindTo(flagSource).value()); + this.fileDownloader = createFileDownloader(otherConfigServersInCluster, downloadDirectory, supervisor); } @Override @@ -100,14 +93,10 @@ public class ApplicationPackageMaintainer extends ConfigServerMaintainer { private static FileDownloader createFileDownloader(List<String> otherConfigServersInCluster, File downloadDirectory, - Supervisor supervisor, - List<String> flagValues) { + Supervisor supervisor) { ConfigSourceSet configSourceSet = new ConfigSourceSet(otherConfigServersInCluster); ConnectionPool connectionPool = new FileDistributionConnectionPool(configSourceSet, supervisor); - Set<CompressionType> acceptedCompressionTypes = flagValues.stream() - .map(CompressionType::valueOf) - .collect(Collectors.toSet()); - return new FileDownloader(connectionPool, supervisor, downloadDirectory, Duration.ofSeconds(300), acceptedCompressionTypes); + return new FileDownloader(connectionPool, supervisor, downloadDirectory, Duration.ofSeconds(300)); } @Override diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/rpc/RpcServer.java b/configserver/src/main/java/com/yahoo/vespa/config/server/rpc/RpcServer.java index a2461706f11..034ac97ebd8 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/rpc/RpcServer.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/rpc/RpcServer.java @@ -587,6 +587,7 @@ public class RpcServer implements Runnable, ConfigActivationListener, TenantList boolean downloadFromOtherSourceIfNotFound = request.parameters().get(1).asInt32() == 0; Set<FileReferenceData.CompressionType> acceptedCompressionTypes = Set.of(CompressionType.gzip); // Newer clients specify accepted compression types in request + // TODO Require acceptedCompressionTypes parameter in Vespa 9 if (request.parameters().size() > 2) acceptedCompressionTypes = Arrays.stream(request.parameters().get(2).asStringArray()) .map(CompressionType::valueOf) diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/filedistribution/FileServerTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/filedistribution/FileServerTest.java index 3c9ea238479..49458acd60b 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/filedistribution/FileServerTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/filedistribution/FileServerTest.java @@ -11,7 +11,6 @@ import com.yahoo.vespa.filedistribution.FileDownloader; import com.yahoo.vespa.filedistribution.FileReferenceCompressor; import com.yahoo.vespa.filedistribution.FileReferenceData; import com.yahoo.vespa.filedistribution.FileReferenceDownload; -import com.yahoo.vespa.flags.InMemoryFlagSource; import org.junit.Before; import org.junit.Rule; import org.junit.Test; @@ -140,10 +139,7 @@ public class FileServerTest { private FileServer createFileServer(ConfigserverConfig.Builder configBuilder) throws IOException { File fileReferencesDir = temporaryFolder.newFolder(); configBuilder.fileReferencesDir(fileReferencesDir.getAbsolutePath()); - InMemoryFlagSource flagSource = new InMemoryFlagSource(); - return new FileServer(new ConfigserverConfig(configBuilder), - flagSource, - new FileDirectory(fileReferencesDir)); + return new FileServer(new ConfigserverConfig(configBuilder), new FileDirectory(fileReferencesDir)); } private static class FileReceiver implements FileServer.Receiver { @@ -168,8 +164,7 @@ public class FileServerTest { new Supervisor(new Transport("mock")).setDropEmptyBuffers(true), downloadDirectory, Duration.ofMillis(100), - Duration.ofMillis(100), - Set.of(gzip)); + Duration.ofMillis(100)); } } diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/rpc/MockRpcServer.java b/configserver/src/test/java/com/yahoo/vespa/config/server/rpc/MockRpcServer.java index 0bf6012a668..c5e307f62f0 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/rpc/MockRpcServer.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/rpc/MockRpcServer.java @@ -37,7 +37,7 @@ public class MockRpcServer extends RpcServer { null, Metrics.createTestMetrics(), new HostRegistry(), - new FileServer(new FileDirectory(tempDir)), + new FileServer(createConfig(port), new FileDirectory(tempDir)), new NoopRpcAuthorizer(), new RpcRequestHandlerProvider()); } diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/rpc/RpcTester.java b/configserver/src/test/java/com/yahoo/vespa/config/server/rpc/RpcTester.java index e0b7909f032..b29edd480ad 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/rpc/RpcTester.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/rpc/RpcTester.java @@ -117,7 +117,7 @@ public class RpcTester implements AutoCloseable { flagSource)), Metrics.createTestMetrics(), hostRegistry, - new FileServer(new FileDirectory(temporaryFolder.newFolder())), + new FileServer(configserverConfig, new FileDirectory(temporaryFolder.newFolder())), new NoopRpcAuthorizer(), new RpcRequestHandlerProvider()); rpcServer.setUpGetConfigHandlers(); diff --git a/container-core/src/main/java/com/yahoo/jdisc/http/server/jetty/ResponseMetricAggregator.java b/container-core/src/main/java/com/yahoo/jdisc/http/server/jetty/ResponseMetricAggregator.java index 5569919c39a..48823599608 100644 --- a/container-core/src/main/java/com/yahoo/jdisc/http/server/jetty/ResponseMetricAggregator.java +++ b/container-core/src/main/java/com/yahoo/jdisc/http/server/jetty/ResponseMetricAggregator.java @@ -113,6 +113,7 @@ class ResponseMetricAggregator extends AbstractLifeCycle implements HttpChannel. static Dimensions of(Request req, Collection<String> monitoringHandlerPaths, Collection<String> searchHandlerPaths) { String requestType = requestType(req, monitoringHandlerPaths, searchHandlerPaths); + // note: some request members may not be populated for invalid requests, e.g. invalid request-line. return new Dimensions(protocol(req), scheme(req), method(req), requestType, statusCode(req)); } @@ -127,42 +128,31 @@ class ResponseMetricAggregator extends AbstractLifeCycle implements HttpChannel. } private static String protocol(Request req) { - switch (req.getProtocol()) { - case "HTTP/1": - case "HTTP/1.0": - case "HTTP/1.1": - return "http1"; - case "HTTP/2": - case "HTTP/2.0": - return "http2"; - default: - return "other"; - } + var protocol = req.getProtocol(); + if (protocol == null) return "none"; + return switch (protocol) { + case "HTTP/1", "HTTP/1.0", "HTTP/1.1" -> "http1"; + case "HTTP/2", "HTTP/2.0" -> "http2"; + default -> "other"; + }; } private static String scheme(Request req) { - switch (req.getScheme()) { - case "http": - case "https": - return req.getScheme(); - default: - return "other"; - } + var scheme = req.getScheme(); + if (scheme == null) return "none"; + return switch (scheme) { + case "http", "https" -> scheme; + default -> "other"; + }; } private static String method(Request req) { - switch (req.getMethod()) { - case "GET": - case "PATCH": - case "POST": - case "PUT": - case "DELETE": - case "OPTIONS": - case "HEAD": - return req.getMethod(); - default: - return "other"; - } + var method = req.getMethod(); + if (method == null) return "none"; + return switch (method) { + case "GET", "PATCH", "POST", "PUT", "DELETE", "OPTIONS", "HEAD" -> method; + default -> "other"; + }; } private static String requestType(Request req, Collection<String> monitoringHandlerPaths, @@ -171,13 +161,16 @@ class ResponseMetricAggregator extends AbstractLifeCycle implements HttpChannel. if (requestType != null) return requestType.name().toLowerCase(); // Deduce from path and method: String path = req.getRequestURI(); + if (path == null) return "none"; for (String monitoringHandlerPath : monitoringHandlerPaths) { if (path.startsWith(monitoringHandlerPath)) return "monitoring"; } for (String searchHandlerPath : searchHandlerPaths) { if (path.startsWith(searchHandlerPath)) return "read"; } - if ("GET".equals(req.getMethod())) return "read"; + var method = req.getMethod(); + if (method == null) return "none"; + else if ("GET".equals(method)) return "read"; else return "write"; } diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/dns/MockVpcEndpointService.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/dns/MockVpcEndpointService.java index f101339ed06..563b343dab5 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/dns/MockVpcEndpointService.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/dns/MockVpcEndpointService.java @@ -3,32 +3,52 @@ package com.yahoo.vespa.hosted.controller.api.integration.dns; import ai.vespa.http.DomainName; import com.yahoo.config.provision.CloudAccount; import com.yahoo.vespa.hosted.controller.api.identifiers.ClusterId; +import com.yahoo.vespa.hosted.controller.api.integration.dns.Record.Type; +import java.time.Clock; import java.util.List; +import java.util.Map; import java.util.Optional; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.atomic.AtomicBoolean; /** * @author jonmv */ public class MockVpcEndpointService implements VpcEndpointService { - public interface Stub extends VpcEndpointService { - @Override default List<VpcEndpoint> getConnections(ClusterId clusterId, Optional<CloudAccount> account) { - return List.of(new VpcEndpoint("endpoint-1", "available")); - } - } + public final AtomicBoolean enabled = new AtomicBoolean(); + public final Map<RecordName, State> outcomes = new ConcurrentHashMap<>(); + + private final Clock clock; + private final NameService nameService; - public static final Stub empty = (name, cluster, account) -> Optional.empty(); + public MockVpcEndpointService(Clock clock, NameService nameService) { + this.clock = clock; + this.nameService = nameService; + } - public Stub delegate = empty; + @Override + public synchronized Optional<DnsChallenge> setPrivateDns(DomainName privateDnsName, ClusterId clusterId, Optional<CloudAccount> account) { + DnsChallenge challenge = new DnsChallenge(RecordName.from("challenge--" + privateDnsName.value()), + RecordData.from(account.map(CloudAccount::value).orElse("system")), + clusterId, + "service-id", + account, + clock.instant(), + State.pending); + return Optional.ofNullable(enabled.get() && nameService.findRecords(Type.TXT, challenge.name()).isEmpty() ? challenge : null); + } @Override - public Optional<DnsChallenge> setPrivateDns(DomainName privateDnsName, ClusterId clusterId, Optional<CloudAccount> account) { - return delegate.setPrivateDns(privateDnsName, clusterId, account); + public synchronized State process(DnsChallenge challenge) { + if (outcomes.containsKey(challenge.name())) return outcomes.get(challenge.name()); + if (nameService.findRecords(Type.TXT, challenge.name()).isEmpty()) throw new RuntimeException("No TXT record found for " + challenge.name()); + return State.done; } @Override - public List<VpcEndpoint> getConnections(ClusterId cluster, Optional<CloudAccount> account) { + public synchronized List<VpcEndpoint> getConnections(ClusterId cluster, Optional<CloudAccount> account) { return List.of(new VpcEndpoint("endpoint-1", "available")); } diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/dns/VpcEndpointService.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/dns/VpcEndpointService.java index 5069a429b27..74459792987 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/dns/VpcEndpointService.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/dns/VpcEndpointService.java @@ -4,20 +4,45 @@ import ai.vespa.http.DomainName; import com.yahoo.config.provision.CloudAccount; import com.yahoo.vespa.hosted.controller.api.identifiers.ClusterId; +import java.time.Instant; import java.util.List; import java.util.Optional; +import static java.util.Objects.requireNonNull; + /** * @author jonmv */ public interface VpcEndpointService { - /** Create a TXT record with this name and token, then run the trigger, to pass this challenge. */ - record DnsChallenge(RecordName name, RecordData data, Runnable trigger) { } + /** Create a TXT record with this name and token, and then complete the challenge. */ + record DnsChallenge(RecordName name, RecordData data, ClusterId clusterId, String serviceId, + Optional<CloudAccount> account, Instant createdAt, State state) { + + public DnsChallenge { + requireNonNull(name, "name must be non-null"); + requireNonNull(data, "data must be non-null"); + requireNonNull(clusterId, "clusterId must be non-null"); + requireNonNull(serviceId, "serviceId must be non-null"); + requireNonNull(account, "account must be non-null"); + requireNonNull(createdAt, "createdAt must be non-null"); + requireNonNull(state, "state must be non-null"); + } + + public DnsChallenge withState(State state) { + return new DnsChallenge(name, data, clusterId, serviceId, account, createdAt, state); + } + + } + + enum State { pending, ready, running, done } /** Sets the private DNS name for any VPC endpoint for the given cluster, potentially guarded by a challenge. */ Optional<DnsChallenge> setPrivateDns(DomainName privateDnsName, ClusterId clusterId, Optional<CloudAccount> account); + /** Attempts to complete the challenge, and returns the updated challenge state. */ + State process(DnsChallenge challenge); + /** A connection made to an endpoint service. */ record VpcEndpoint(String endpointId, String state) { } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java index 14f2b38f24a..9721396cd54 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java @@ -358,8 +358,12 @@ public class InternalStepRunner implements StepRunner { controller.jobController().locked(id, lockedRun -> lockedRun.withSummary(null)); Availability availability = endpointsAvailable(id.application(), id.type().zone(), logger); if (availability.status() == Status.available) { + if (controller.routing().policies().processDnsChallenges(new DeploymentId(id.application(), id.type().zone()))) { logger.log("Installation succeeded!"); return Optional.of(running); + } + logger.log("Waiting for DNS challenges for private endpoints to be processed"); + return Optional.empty(); } logger.log(availability.message()); if (availability.status() == Status.endpointsUnavailable && timedOut(id, deployment.get(), timeouts.endpoint())) { diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/CuratorDb.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/CuratorDb.java index be58370285a..f4980073d6c 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/CuratorDb.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/CuratorDb.java @@ -6,6 +6,7 @@ import com.yahoo.component.Version; import com.yahoo.component.annotation.Inject; import com.yahoo.concurrent.UncheckedTimeoutException; import com.yahoo.config.provision.ApplicationId; +import com.yahoo.config.provision.ClusterSpec; import com.yahoo.config.provision.HostName; import com.yahoo.config.provision.TenantName; import com.yahoo.config.provision.zone.ZoneId; @@ -15,12 +16,14 @@ import com.yahoo.slime.SlimeUtils; import com.yahoo.transaction.Mutex; import com.yahoo.vespa.curator.Curator; import com.yahoo.vespa.hosted.controller.Application; +import com.yahoo.vespa.hosted.controller.api.identifiers.ClusterId; import com.yahoo.vespa.hosted.controller.api.identifiers.ControllerVersion; import com.yahoo.vespa.hosted.controller.api.identifiers.DeploymentId; import com.yahoo.vespa.hosted.controller.api.integration.archive.ArchiveBucket; import com.yahoo.vespa.hosted.controller.api.integration.certificates.EndpointCertificateMetadata; import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType; import com.yahoo.vespa.hosted.controller.api.integration.deployment.RunId; +import com.yahoo.vespa.hosted.controller.api.integration.dns.VpcEndpointService.DnsChallenge; import com.yahoo.vespa.hosted.controller.api.integration.vcmr.VespaChangeRequest; import com.yahoo.vespa.hosted.controller.application.TenantAndApplicationId; import com.yahoo.vespa.hosted.controller.auditlog.AuditLog; @@ -90,6 +93,7 @@ public class CuratorDb { private static final Path jobRoot = root.append("jobs"); private static final Path controllerRoot = root.append("controllers"); private static final Path routingPoliciesRoot = root.append("routingPolicies"); + private static final Path dnsChallengesRoot = root.append("dnsChallenges"); private static final Path zoneRoutingPoliciesRoot = root.append("zoneRoutingPolicies"); private static final Path endpointCertificateRoot = root.append("applicationCertificates"); private static final Path archiveBucketsRoot = root.append("archiveBuckets"); @@ -114,6 +118,7 @@ public class CuratorDb { private final RunSerializer runSerializer = new RunSerializer(); private final RetriggerEntrySerializer retriggerEntrySerializer = new RetriggerEntrySerializer(); private final NotificationsSerializer notificationsSerializer = new NotificationsSerializer(); + private final DnsChallengeSerializer dnsChallengeSerializer = new DnsChallengeSerializer(); private final Curator curator; private final Duration tryLockTimeout; @@ -559,6 +564,35 @@ public class CuratorDb { .orElseGet(() -> new ZoneRoutingPolicy(zone, RoutingStatus.DEFAULT)); } + public void writeDnsChallenge(DnsChallenge challenge) { + curator.set(dnsChallengePath(challenge.clusterId()), dnsChallengeSerializer.toJson(challenge)); + } + + public void deleteDnsChallenge(ClusterId id) { + curator.delete(dnsChallengePath(id)); + } + + public List<DnsChallenge> readDnsChallenges(DeploymentId id) { + return curator.getChildren(dnsChallengePath(id)).stream() + .map(cluster -> readDnsChallenge(new ClusterId(id, ClusterSpec.Id.from(cluster)))) + .toList(); + } + + private DnsChallenge readDnsChallenge(ClusterId clusterId) { + return curator.getData(dnsChallengePath(clusterId)) + .map(bytes -> dnsChallengeSerializer.fromJson(bytes, clusterId)) + .orElseThrow(() -> new IllegalArgumentException("no DNS challenge for " + clusterId)); + } + + private static Path dnsChallengePath(DeploymentId id) { + return dnsChallengesRoot.append(id.applicationId().serializedForm()) + .append(id.zoneId().value()); + } + + private static Path dnsChallengePath(ClusterId id) { + return dnsChallengePath(id.deploymentId()).append(id.clusterId().value()); + } + // -------------- Application endpoint certificates ---------------------------- public void writeEndpointCertificateMetadata(ApplicationId applicationId, EndpointCertificateMetadata endpointCertificateMetadata) { diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/DnsChallengeSerializer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/DnsChallengeSerializer.java new file mode 100644 index 00000000000..2518fe48508 --- /dev/null +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/DnsChallengeSerializer.java @@ -0,0 +1,75 @@ +package com.yahoo.vespa.hosted.controller.persistence; + +import com.yahoo.config.provision.ApplicationId; +import com.yahoo.config.provision.CloudAccount; +import com.yahoo.config.provision.ClusterSpec; +import com.yahoo.config.provision.zone.ZoneId; +import com.yahoo.slime.Cursor; +import com.yahoo.slime.Slime; +import com.yahoo.slime.SlimeUtils; +import com.yahoo.vespa.hosted.controller.api.identifiers.ClusterId; +import com.yahoo.vespa.hosted.controller.api.identifiers.DeploymentId; +import com.yahoo.vespa.hosted.controller.api.integration.dns.RecordData; +import com.yahoo.vespa.hosted.controller.api.integration.dns.RecordName; +import com.yahoo.vespa.hosted.controller.api.integration.dns.VpcEndpointService.DnsChallenge; +import com.yahoo.vespa.hosted.controller.api.integration.dns.VpcEndpointService.State; + +import java.time.Instant; + +import static com.yahoo.yolean.Exceptions.uncheck; + +/** + * @author jonmv + */ +class DnsChallengeSerializer { + + private static final String nameField = "name"; + private static final String dataField = "data"; + private static final String serviceIdField = "serviceId"; + private static final String accountField = "account"; + private static final String createdAtField = "createdAt"; + private static final String stateField = "state"; + + DnsChallenge fromJson(byte[] json, ClusterId clusterId) { + Cursor object = SlimeUtils.jsonToSlime(json).get(); + return new DnsChallenge(RecordName.from(object.field(nameField).asString()), + RecordData.from(object.field(dataField).asString()), + clusterId, + object.field(serviceIdField).asString(), + SlimeUtils.optionalString(object.field(accountField)).map(CloudAccount::from), + Instant.ofEpochMilli(object.field(createdAtField).asLong()), + toState(object.field(stateField).asString())); + } + + byte[] toJson(DnsChallenge challenge) { + Slime slime = new Slime(); + Cursor object = slime.setObject(); + object.setString(nameField, challenge.name().name()); + object.setString(dataField, challenge.data().data()); + object.setString(serviceIdField, challenge.serviceId()); + challenge.account().ifPresent(account -> object.setString(accountField, account.value())); + object.setLong(createdAtField, challenge.createdAt().toEpochMilli()); + object.setString(stateField, toString(challenge.state())); + return uncheck(() -> SlimeUtils.toJsonBytes(slime)); + } + + private static State toState(String value) { + return switch (value) { + case "pending" -> State.pending; + case "ready" -> State.ready; + case "running" -> State.running; + case "done" -> State.done; + default -> throw new IllegalArgumentException("invalid serialized state: " + value); + }; + } + + private static String toString(State state) { + return switch (state) { + case pending -> "pending"; + case ready -> "ready"; + case running -> "running"; + case done -> "done"; + }; + } + +} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/RoutingPolicies.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/RoutingPolicies.java index 1c4916b9bed..61c71e964f6 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/RoutingPolicies.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/RoutingPolicies.java @@ -5,6 +5,7 @@ import ai.vespa.http.DomainName; import com.yahoo.concurrent.UncheckedTimeoutException; import com.yahoo.config.application.api.DeploymentSpec; import com.yahoo.config.provision.ApplicationId; +import com.yahoo.config.provision.CloudAccount; import com.yahoo.config.provision.zone.RoutingMethod; import com.yahoo.config.provision.zone.ZoneId; import com.yahoo.transaction.Mutex; @@ -20,6 +21,8 @@ import com.yahoo.vespa.hosted.controller.api.integration.dns.Record; import com.yahoo.vespa.hosted.controller.api.integration.dns.Record.Type; import com.yahoo.vespa.hosted.controller.api.integration.dns.RecordData; import com.yahoo.vespa.hosted.controller.api.integration.dns.RecordName; +import com.yahoo.vespa.hosted.controller.api.integration.dns.VpcEndpointService.DnsChallenge; +import com.yahoo.vespa.hosted.controller.api.integration.dns.VpcEndpointService.State; import com.yahoo.vespa.hosted.controller.api.integration.dns.WeightedAliasTarget; import com.yahoo.vespa.hosted.controller.api.integration.dns.WeightedDirectTarget; import com.yahoo.vespa.hosted.controller.application.Endpoint; @@ -27,12 +30,14 @@ import com.yahoo.vespa.hosted.controller.application.EndpointId; import com.yahoo.vespa.hosted.controller.application.EndpointList; import com.yahoo.vespa.hosted.controller.application.TenantAndApplicationId; import com.yahoo.vespa.hosted.controller.dns.NameServiceForwarder; +import com.yahoo.vespa.hosted.controller.dns.NameServiceQueue; import com.yahoo.vespa.hosted.controller.dns.NameServiceQueue.Priority; import com.yahoo.vespa.hosted.controller.dns.NameServiceRequest; import com.yahoo.vespa.hosted.controller.persistence.CuratorDb; import com.yahoo.yolean.UncheckedInterruptedException; import java.time.Instant; +import java.util.ArrayList; import java.util.Collection; import java.util.Collections; import java.util.HashSet; @@ -394,28 +399,51 @@ public class RoutingPolicies { new ClusterId(deploymentId, endpoint.cluster()), loadBalancer.cloudAccount()) .ifPresent(challenge -> { - try { + try (Mutex lock = db.lockNameServiceQueue()) { nameServiceForwarderIn(deploymentId.zoneId()).createTxt(challenge.name(), List.of(challenge.data()), Priority.high, ownerOf(deploymentId)); - Instant doom = controller.clock().instant().plusSeconds(30); - while (controller.clock().instant().isBefore(doom)) { - try (Mutex lock = controller.curator().lockNameServiceQueue()) { - if (controller.curator().readNameServiceQueue().requests().stream() - .noneMatch(request -> request.name().equals(challenge.name()))) { - try { challenge.trigger().run(); } - finally { nameServiceForwarderIn(deploymentId.zoneId()).removeRecords(Type.TXT, challenge.name(), Priority.normal, ownerOf(deploymentId)); } - return; - } - } - Thread.sleep(100); - } - throw new UncheckedTimeoutException("timed out waiting for DNS challenge to be processed"); - } - catch (InterruptedException e) { - throw new UncheckedInterruptedException("interrupted waiting for DNS challenge to be processed", e, true); + db.writeDnsChallenge(challenge); } }); } + /** Returns true iff. the given deployment has no incomplete DNS challenges, or throws (and cleans up) on errors. */ + public boolean processDnsChallenges(DeploymentId deploymentId) { + try (Mutex lock = db.lockNameServiceQueue()) { + List<DnsChallenge> challenges = new ArrayList<>(db.readDnsChallenges(deploymentId)); + Set<RecordName> pendingRequests = controller.curator().readNameServiceQueue().requests().stream() + .map(NameServiceRequest::name) + .collect(Collectors.toSet()); + try { + challenges.removeIf(challenge -> { + if (challenge.state() == State.pending) { + if (pendingRequests.contains(challenge.name())) return false; + challenge = challenge.withState(State.ready); + } + State state = controller.serviceRegistry().vpcEndpointService().process(challenge); + if (state == State.done) { + removeDnsChallenge(challenge); + return true; + } + else { + db.writeDnsChallenge(challenge.withState(state)); + return false; + } + }); + return challenges.isEmpty(); + } + catch (RuntimeException e) { + challenges.forEach(this::removeDnsChallenge); + throw e; + } + } + } + + private void removeDnsChallenge(DnsChallenge challenge) { + nameServiceForwarderIn(challenge.clusterId().deploymentId().zoneId()) + .removeRecords(Type.TXT, challenge.name(), Priority.normal, ownerOf(challenge.clusterId().deploymentId())); + db.deleteDnsChallenge(challenge.clusterId()); + } + /** * Remove policies and zone DNS records unreferenced by given load balancers * diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentContext.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentContext.java index 95c22480c0f..14835a822e6 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentContext.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentContext.java @@ -251,7 +251,7 @@ public class DeploymentContext { flushDnsUpdates(Integer.MAX_VALUE); assertEquals(List.of(), tester.controller().curator().readNameServiceQueue().requests(), - "All name service requests dispatched"); + "All name service requests dispatched"); return this; } @@ -590,6 +590,9 @@ public class DeploymentContext { tester.cloud().set(Status.SUCCESS); runner.advance(currentRun(job)); assertEquals(succeeded, jobs.run(id).stepStatuses().get(Step.endStagingSetup)); + + if ( ! deferDnsUpdates) + flushDnsUpdates(); } } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ConfigServerMock.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ConfigServerMock.java index 6e6ecef4e66..b8954ff6e73 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ConfigServerMock.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ConfigServerMock.java @@ -22,6 +22,7 @@ import com.yahoo.config.provision.NodeType; import com.yahoo.config.provision.ZoneEndpoint.AllowedUrn; import com.yahoo.config.provision.ZoneEndpoint.AccessType; import com.yahoo.config.provision.zone.ZoneId; +import com.yahoo.rdl.UUID; import com.yahoo.vespa.flags.json.FlagData; import com.yahoo.vespa.hosted.controller.api.application.v4.model.ClusterMetrics; import com.yahoo.vespa.hosted.controller.api.application.v4.model.DeploymentData; @@ -69,8 +70,8 @@ import java.util.LinkedHashSet; import java.util.List; import java.util.Map; import java.util.Optional; +import java.util.Random; import java.util.Set; -import java.util.UUID; import java.util.function.Consumer; import java.util.function.Supplier; import java.util.logging.Level; @@ -385,8 +386,7 @@ public class ConfigServerMock extends AbstractComponent implements ConfigServer /** Add any of given loadBalancers that do not already exist to the load balancers in zone */ public void putLoadBalancers(ZoneId zone, List<LoadBalancer> loadBalancers) { - this.loadBalancers.putIfAbsent(zone, new LinkedHashSet<>()); - this.loadBalancers.get(zone).addAll(loadBalancers); + this.loadBalancers.computeIfAbsent(zone, __ -> new LinkedHashSet<>()).addAll(loadBalancers); } public void removeLoadBalancers(ApplicationId application, ZoneId zone) { @@ -420,7 +420,7 @@ public class ConfigServerMock extends AbstractComponent implements ConfigServer deployment.cloudAccount().ifPresent(account -> this.cloudAccounts.put(id, account)); if (!deferLoadBalancerProvisioning.contains(id.zoneId().environment())) { - putLoadBalancers(id.zoneId(), List.of(new LoadBalancer(UUID.randomUUID().toString(), + putLoadBalancers(id.zoneId(), List.of(new LoadBalancer(id.dottedString() + "." + cluster, id.applicationId(), cluster, Optional.of(HostName.of("lb-0--" + id.applicationId().toFullString() + "--" + id.zoneId().toString())), diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ServiceRegistryMock.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ServiceRegistryMock.java index 0ba8866c990..be257daa211 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ServiceRegistryMock.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ServiceRegistryMock.java @@ -67,7 +67,7 @@ public class ServiceRegistryMock extends AbstractComponent implements ServiceReg private final ZoneRegistryMock zoneRegistryMock; private final ConfigServerMock configServerMock; private final MemoryNameService memoryNameService = new MemoryNameService(); - private final MockVpcEndpointService vpcEndpointService = new MockVpcEndpointService(); + private final MockVpcEndpointService vpcEndpointService = new MockVpcEndpointService(clock, memoryNameService); private final MockMailer mockMailer = new MockMailer(); private final EndpointCertificateMock endpointCertificateMock = new EndpointCertificateMock(clock); private final EndpointCertificateValidatorMock endpointCertificateValidatorMock = new EndpointCertificateValidatorMock(); diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/DnsChallengeSerializerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/DnsChallengeSerializerTest.java new file mode 100644 index 00000000000..674cc29e91c --- /dev/null +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/DnsChallengeSerializerTest.java @@ -0,0 +1,45 @@ +package com.yahoo.vespa.hosted.controller.persistence; + +import com.yahoo.config.provision.ApplicationId; +import com.yahoo.config.provision.CloudAccount; +import com.yahoo.config.provision.ClusterSpec; +import com.yahoo.config.provision.zone.ZoneId; +import com.yahoo.vespa.hosted.controller.api.identifiers.ClusterId; +import com.yahoo.vespa.hosted.controller.api.identifiers.DeploymentId; +import com.yahoo.vespa.hosted.controller.api.integration.dns.RecordData; +import com.yahoo.vespa.hosted.controller.api.integration.dns.RecordName; +import com.yahoo.vespa.hosted.controller.api.integration.dns.VpcEndpointService.DnsChallenge; +import com.yahoo.vespa.hosted.controller.api.integration.dns.VpcEndpointService.State; +import org.junit.jupiter.api.Test; + +import java.time.Instant; +import java.util.Optional; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +/** + * @author jonmv + */ +class DnsChallengeSerializerTest { + + private final DnsChallengeSerializer serializer = new DnsChallengeSerializer(); + private final ClusterId clusterId = new ClusterId(new DeploymentId(ApplicationId.defaultId(), + ZoneId.defaultId()), + ClusterSpec.Id.from("default")); + private final DnsChallenge challenge = new DnsChallenge(RecordName.from("name.tld"), + RecordData.from("1234"), + clusterId, + "deadbeef", + Optional.of(CloudAccount.from("123321123321")), + Instant.ofEpochMilli(123), + State.pending); + + @Test + void testSerialization() { + DnsChallenge deserialized = serializer.fromJson(serializer.toJson(challenge), clusterId); + assertEquals(challenge, deserialized); + for (State state : State.values()) + assertEquals(challenge.withState(state), serializer.fromJson(serializer.toJson(challenge.withState(state)), clusterId)); + } + +} diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/routing/RoutingPoliciesTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/routing/RoutingPoliciesTest.java index 2932860efaa..94cffb94184 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/routing/RoutingPoliciesTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/routing/RoutingPoliciesTest.java @@ -26,6 +26,7 @@ import com.yahoo.vespa.hosted.controller.api.integration.dns.Record.Type; import com.yahoo.vespa.hosted.controller.api.integration.dns.RecordData; import com.yahoo.vespa.hosted.controller.api.integration.dns.RecordName; import com.yahoo.vespa.hosted.controller.api.integration.dns.VpcEndpointService.DnsChallenge; +import com.yahoo.vespa.hosted.controller.api.integration.dns.VpcEndpointService.State; import com.yahoo.vespa.hosted.controller.application.Endpoint; import com.yahoo.vespa.hosted.controller.application.EndpointId; import com.yahoo.vespa.hosted.controller.application.EndpointList; @@ -57,6 +58,7 @@ import java.util.stream.Collectors; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertSame; +import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; /** @@ -515,39 +517,33 @@ public class RoutingPoliciesTest { void private_dns_for_vpc_endpoint() { // Challenge answered for endpoint RoutingPoliciesTester tester = new RoutingPoliciesTester(); - Map<RecordName, RecordData> challenges = new ConcurrentHashMap<>(); - tester.tester.controllerTester().serviceRegistry().vpcEndpointService().delegate = (name, cluster, account) -> { - RecordName recordName = RecordName.from("challenge--" + name.value()); - if (challenges.containsKey(recordName)) return Optional.empty(); - RecordData recordData = RecordData.from(account.map(CloudAccount::value).orElse("system")); - return Optional.of(new DnsChallenge(recordName, recordData, () -> challenges.put(recordName, recordData))); - }; + tester.tester.controllerTester().serviceRegistry().vpcEndpointService().enabled.set(true); DeploymentContext app = tester.newDeploymentContext("t", "a", "default"); ApplicationPackage appPackage = applicationPackageBuilder().region(zone3.region()).build(); app.submit(appPackage); - AtomicBoolean done = new AtomicBoolean(); - new Thread(() -> { - while ( ! done.get()) { - app.flushDnsUpdates(Integer.MAX_VALUE); - try { Thread.sleep(10); } catch (InterruptedException e) { break; } - } - }).start(); app.deploy(); - done.set(true); - - assertEquals(Map.of( RecordName.from("challenge--a.t.aws-us-east-1a.vespa.oath.cloud"), - RecordData.from("system"), - RecordName.from("challenge--a.t.us-east-1.test.vespa.oath.cloud"), - RecordData.from("system"), - RecordName.from("challenge--a.t.us-east-3.staging.vespa.oath.cloud"), - RecordData.from("system")), - challenges); + + // TXT records are cleaned up as we go—the last challenge is the last to go here, and we must flush it ourselves. + assertEquals(Set.of("a.t.aws-us-east-1a.vespa.oath.cloud", + "challenge--a.t.aws-us-east-1a.vespa.oath.cloud"), + tester.recordNames()); + app.flushDnsUpdates(); assertEquals(Set.of(new Record(Type.CNAME, RecordName.from("a.t.aws-us-east-1a.vespa.oath.cloud"), RecordData.from("lb-0--t.a.default--prod.aws-us-east-1a."))), tester.controllerTester().nameService().records()); + + + tester.tester.controllerTester().serviceRegistry().vpcEndpointService().outcomes + .put(RecordName.from("challenge--a.t.aws-us-east-1a.vespa.oath.cloud"), State.running); + + // Deployment fails because challenge is not answered (immediately). + assertEquals("Status of run 2 of production-aws-us-east-1a for t.a ==> expected: <succeeded> but was: <unfinished>", + assertThrows(AssertionError.class, + () -> app.submit(appPackage).deploy()) + .getMessage()); } @Test diff --git a/document/src/vespa/document/datatype/annotationreferencedatatype.cpp b/document/src/vespa/document/datatype/annotationreferencedatatype.cpp index 6983676a42b..066944a4e77 100644 --- a/document/src/vespa/document/datatype/annotationreferencedatatype.cpp +++ b/document/src/vespa/document/datatype/annotationreferencedatatype.cpp @@ -15,6 +15,8 @@ AnnotationReferenceDataType::AnnotationReferenceDataType(const AnnotationType &t _type(&type) { } +AnnotationReferenceDataType::~AnnotationReferenceDataType() = default; + const AnnotationType & AnnotationReferenceDataType::getAnnotationType() const { assert(_type); diff --git a/document/src/vespa/document/datatype/annotationreferencedatatype.h b/document/src/vespa/document/datatype/annotationreferencedatatype.h index bf926253e4c..213e022e036 100644 --- a/document/src/vespa/document/datatype/annotationreferencedatatype.h +++ b/document/src/vespa/document/datatype/annotationreferencedatatype.h @@ -16,6 +16,7 @@ public: AnnotationReferenceDataType(const AnnotationType &type, int id); AnnotationReferenceDataType(const AnnotationReferenceDataType &) = delete; AnnotationReferenceDataType & operator=(const AnnotationReferenceDataType &) = delete; + ~AnnotationReferenceDataType() override; const AnnotationType &getAnnotationType() const; void print(std::ostream &out, bool verbose, const std::string &indent) const override; std::unique_ptr<FieldValue> createFieldValue() const override; diff --git a/document/src/vespa/document/datatype/arraydatatype.cpp b/document/src/vespa/document/datatype/arraydatatype.cpp index 68c4e87ae5e..2097385559e 100644 --- a/document/src/vespa/document/datatype/arraydatatype.cpp +++ b/document/src/vespa/document/datatype/arraydatatype.cpp @@ -17,6 +17,8 @@ ArrayDataType::ArrayDataType(const DataType& nestedType) { } +ArrayDataType::~ArrayDataType() = default; + FieldValue::UP ArrayDataType::createFieldValue() const { diff --git a/document/src/vespa/document/datatype/arraydatatype.h b/document/src/vespa/document/datatype/arraydatatype.h index a984fee0c0d..579505432a3 100644 --- a/document/src/vespa/document/datatype/arraydatatype.h +++ b/document/src/vespa/document/datatype/arraydatatype.h @@ -16,15 +16,16 @@ public: explicit ArrayDataType(const DataType &nestedType); ArrayDataType(const ArrayDataType &o) = delete; ArrayDataType &operator=(const ArrayDataType &rhs) = delete; + ~ArrayDataType() override; ArrayDataType(const DataType &nestedType, int32_t id); - std::unique_ptr<FieldValue> createFieldValue() const override; + [[nodiscard]] std::unique_ptr<FieldValue> createFieldValue() const override; void print(std::ostream&, bool verbose, const std::string& indent) const override; - bool equals(const DataType& other) const noexcept override; + [[nodiscard]] bool equals(const DataType& other) const noexcept override; void onBuildFieldPath(FieldPath & path, vespalib::stringref remainFieldName) const override; - bool isArray() const noexcept override { return true; } + [[nodiscard]] bool isArray() const noexcept override { return true; } }; } // document diff --git a/document/src/vespa/document/repo/documenttyperepo.cpp b/document/src/vespa/document/repo/documenttyperepo.cpp index d7435c979a0..c6d30232654 100644 --- a/document/src/vespa/document/repo/documenttyperepo.cpp +++ b/document/src/vespa/document/repo/documenttyperepo.cpp @@ -42,7 +42,7 @@ class DocumentTypeMap : public DocumentTypeMapT { public: using DocumentTypeMapT::DocumentTypeMapT; - DataTypeRepo * findRepo(int32_t doc_type_id) const { + [[nodiscard]] DataTypeRepo * findRepo(int32_t doc_type_id) const { auto iter = find(doc_type_id); if (iter == end()) { return nullptr; @@ -66,27 +66,33 @@ class Repo { hash_map<string, const DataType *> _name_map; public: - ~Repo() {} - - void inherit(const Repo &parent); - bool addDataType(const DataType &type); - template <typename T> const DataType * addDataType(unique_ptr<T> type); - - const DataType &addTensorType(const string &spec); - const DataType *lookup(int32_t id) const; - const DataType *lookup(stringref name) const; - const DataType &findOrThrow(int32_t id) const; - const DataType &findOrThrowOrCreate(int32_t id, const string &detailedType); + Repo() noexcept; + ~Repo(); + + void inherit(const Repo &parent) __attribute__((noinline)); + bool addDataType(const DataType &type) __attribute__((noinline)); + template <typename T> const DataType * addDataType(unique_ptr<T> type) __attribute__((noinline)); + + const DataType &addTensorType(const string &spec) __attribute__((noinline)); + const DataType *lookup(int32_t id) const __attribute__((noinline)); + const DataType *lookup(stringref name) const __attribute__((noinline)); + const DataType &findOrThrow(int32_t id) const __attribute__((noinline)); + const DataType &findOrThrowOrCreate(int32_t id, const string &detailedType) __attribute__((noinline)); }; -void Repo::inherit(const Repo &parent) { +Repo::Repo() noexcept = default; +Repo::~Repo() = default; + +void +Repo::inherit(const Repo &parent) { _id_map.insert(parent._id_map.begin(), parent._id_map.end()); _tensorTypes.insert(parent._tensorTypes.begin(), parent._tensorTypes.end()); _name_map.insert(parent._name_map.begin(), parent._name_map.end()); } // Returns true if a reference to type is stored. -bool Repo::addDataType(const DataType &type) { +bool +Repo::addDataType(const DataType &type) { const DataType *& data_type = _id_map[type.getId()]; if (data_type) { if (data_type->equals(type) && (data_type->getName() == type.getName())) { @@ -109,7 +115,8 @@ bool Repo::addDataType(const DataType &type) { } template <typename T> -const DataType* Repo::addDataType(unique_ptr<T> type) { +const DataType* +Repo::addDataType(unique_ptr<T> type) { int id = type->getId(); if (addDataType(*type)) { _owned_types.emplace_back(std::move(type)); @@ -306,6 +313,7 @@ void addStruct(int32_t id, const Datatype::Sstruct &s, Repo &repo) { } } +void addArray(int32_t id, const Datatype::Array &a, Repo &repo) __attribute__((noinline)); void addArray(int32_t id, const Datatype::Array &a, Repo &repo) { const DataType &nested = repo.findOrThrow(a.element.id); repo.addDataType(std::make_unique<ArrayDataType>(nested, id)); @@ -330,6 +338,7 @@ void addAnnotationRef(int32_t id, const Datatype::Annotationref &a, Repo &r, con r.addDataType(std::make_unique<AnnotationReferenceDataType>(*type, id)); } +void addDataType(const Datatype &type, Repo &repo, const AnnotationTypeRepo &a_repo) __attribute__((noinline)); void addDataType(const Datatype &type, Repo &repo, const AnnotationTypeRepo &a_repo) { switch (type.type) { case Datatype::Type::STRUCT: @@ -567,38 +576,8 @@ private: MadeTypes _made_types; std::set<int> _needed_idx_set; - void apply() { - findNeeded(); - for (const CDocType & docT : _input) { - auto [iter,succ] = _doc_types_in_progress.emplace(docT.idx, - DocTypeInProgress(docT, _output)); - LOG_ASSERT(succ); - auto & dtInP = iter->second; - createSimpleTypes(dtInP); - createEmptyStructs(dtInP); - initializeDocTypeAndInheritAnnotations(dtInP); - createEmptyAnnotationTypes(dtInP); - } - for (auto & [id, dtInP] : _doc_types_in_progress) { - createReferenceTypes(dtInP); - } - createComplexTypes(); - fillStructs(); - for (auto & [id, dtInP] : _doc_types_in_progress) { - fillDocument(dtInP); - fillAnnotationTypes(dtInP); - } - for (const auto & docT : _input) { - for (const auto & structT : docT.structtype) { - performStructInherit(structT.idx); - } - } - } - - void madeType(const DataType *t, int idx) { - _made_types[idx] = t; - _needed_idx_set.erase(idx); - } + void apply() __attribute__((noinline)); + void madeType(const DataType *t, int idx) __attribute__((noinline)); void createSimpleTypes(DocTypeInProgress & dtInP) { for (const auto & primT : dtInP.cfg.primitivetype) { @@ -734,58 +713,8 @@ private: } } - void createComplexTypes() { - while (_needed_idx_set.size() > 0) { - size_t missing_cnt = _needed_idx_set.size(); - for (const auto & docT : _input) { - auto iter = _doc_types_in_progress.find(docT.idx); - LOG_ASSERT(iter != _doc_types_in_progress.end()); - auto & dtInP = iter->second; - createComplexTypesForDocType(dtInP.cfg, dtInP.repo()); - } - if (_needed_idx_set.size() == missing_cnt) { - for (int idx : _needed_idx_set) { - LOG(error, "no progress, datatype [idx %d] still missing", idx); - } - throw IllegalArgumentException("no progress"); - } - LOG(debug, "retry complex types, %zd missing", _needed_idx_set.size()); - } - } - - void createComplexTypesForDocType(const CDocType & docT, Repo& repo) { - for (const auto & arrT : docT.arraytype) { - if (_made_types[arrT.idx] != nullptr) { - continue; // OK already - } - if (const DataType * nested = _made_types[arrT.elementtype]) { - auto at = std::make_unique<ArrayDataType>(*nested, arrT.internalid); - madeType(repo.addDataType(std::move(at)), arrT.idx); - } - } - for (const auto & mapT : docT.maptype) { - if (_made_types[mapT.idx] != nullptr) { - continue; // OK already - } - const DataType * kt = _made_types[mapT.keytype]; - const DataType * vt = _made_types[mapT.valuetype]; - if (kt && vt) { - auto mt = std::make_unique<MapDataType>(*kt, *vt, mapT.internalid); - madeType(repo.addDataType(std::move(mt)), mapT.idx); - } - } - for (const auto & wsetT : docT.wsettype) { - if (_made_types[wsetT.idx] != nullptr) { - continue; // OK already - } - if (const DataType * nested = _made_types[wsetT.elementtype]) { - auto wt = std::make_unique<WeightedSetDataType>(*nested, - wsetT.createifnonexistent, wsetT.removeifzero, - wsetT.internalid); - madeType(repo.addDataType(std::move(wt)), wsetT.idx); - } - } - } + void createComplexTypes() __attribute__((noinline)); + void createComplexTypesForDocType(const CDocType & docT, Repo& repo) __attribute__((noinline)); void fillStructs() { for (auto & [idx, in_progress] : _structs_in_progress) { @@ -970,17 +899,108 @@ private: } public: - ApplyNewDoctypeConfig(const DocumenttypesConfig::DoctypeVector & input, - DocumentTypeMap & output) - : _input(input), _output(output) - { - apply(); - } + ApplyNewDoctypeConfig(const DocumenttypesConfig::DoctypeVector & input, DocumentTypeMap & output); ~ApplyNewDoctypeConfig(); }; +ApplyNewDoctypeConfig::ApplyNewDoctypeConfig(const DocumenttypesConfig::DoctypeVector & input, DocumentTypeMap & output) + : _input(input), + _output(output) +{ + apply(); +} ApplyNewDoctypeConfig::~ApplyNewDoctypeConfig() = default; +void +ApplyNewDoctypeConfig::madeType(const DataType *t, int idx) { + _made_types[idx] = t; + _needed_idx_set.erase(idx); +} + +void +ApplyNewDoctypeConfig::apply() { + findNeeded(); + for (const CDocType & docT : _input) { + auto [iter,succ] = _doc_types_in_progress.emplace(docT.idx, + DocTypeInProgress(docT, _output)); + LOG_ASSERT(succ); + auto & dtInP = iter->second; + createSimpleTypes(dtInP); + createEmptyStructs(dtInP); + initializeDocTypeAndInheritAnnotations(dtInP); + createEmptyAnnotationTypes(dtInP); + } + for (auto & [id, dtInP] : _doc_types_in_progress) { + createReferenceTypes(dtInP); + } + createComplexTypes(); + fillStructs(); + for (auto & [id, dtInP] : _doc_types_in_progress) { + fillDocument(dtInP); + fillAnnotationTypes(dtInP); + } + for (const auto & docT : _input) { + for (const auto & structT : docT.structtype) { + performStructInherit(structT.idx); + } + } +} + +void +ApplyNewDoctypeConfig::createComplexTypes() { + while (_needed_idx_set.size() > 0) { + size_t missing_cnt = _needed_idx_set.size(); + for (const auto & docT : _input) { + auto iter = _doc_types_in_progress.find(docT.idx); + LOG_ASSERT(iter != _doc_types_in_progress.end()); + auto & dtInP = iter->second; + createComplexTypesForDocType(dtInP.cfg, dtInP.repo()); + } + if (_needed_idx_set.size() == missing_cnt) { + for (int idx : _needed_idx_set) { + LOG(error, "no progress, datatype [idx %d] still missing", idx); + } + throw IllegalArgumentException("no progress"); + } + LOG(debug, "retry complex types, %zd missing", _needed_idx_set.size()); + } +} + +void +ApplyNewDoctypeConfig::createComplexTypesForDocType(const CDocType & docT, Repo& repo) { + for (const auto & arrT : docT.arraytype) { + if (_made_types[arrT.idx] != nullptr) { + continue; // OK already + } + if (const DataType * nested = _made_types[arrT.elementtype]) { + auto at = std::make_unique<ArrayDataType>(*nested, arrT.internalid); + madeType(repo.addDataType(std::move(at)), arrT.idx); + } + } + for (const auto & mapT : docT.maptype) { + if (_made_types[mapT.idx] != nullptr) { + continue; // OK already + } + const DataType * kt = _made_types[mapT.keytype]; + const DataType * vt = _made_types[mapT.valuetype]; + if (kt && vt) { + auto mt = std::make_unique<MapDataType>(*kt, *vt, mapT.internalid); + madeType(repo.addDataType(std::move(mt)), mapT.idx); + } + } + for (const auto & wsetT : docT.wsettype) { + if (_made_types[wsetT.idx] != nullptr) { + continue; // OK already + } + if (const DataType * nested = _made_types[wsetT.elementtype]) { + auto wt = std::make_unique<WeightedSetDataType>(*nested, + wsetT.createifnonexistent, wsetT.removeifzero, + wsetT.internalid); + madeType(repo.addDataType(std::move(wt)), wsetT.idx); + } + } +} + void configureDocTypes(const DocumenttypesConfig::DoctypeVector &t, DocumentTypeMap &type_map) { LOG(debug, "applying new doc type config"); ApplyNewDoctypeConfig(t, type_map); @@ -1022,8 +1042,7 @@ DocumentTypeRepo::DocumentTypeRepo(const DocumenttypesConfig &config) : } } -DocumentTypeRepo::~DocumentTypeRepo() { -} +DocumentTypeRepo::~DocumentTypeRepo() = default; DataTypeRepo *DocumentTypeRepo::findRepo(int32_t doc_type_id) const { return _doc_types->findRepo(doc_type_id); diff --git a/document/src/vespa/document/update/fieldpathupdate.cpp b/document/src/vespa/document/update/fieldpathupdate.cpp index a8a42f69215..fa4cad5fa8c 100644 --- a/document/src/vespa/document/update/fieldpathupdate.cpp +++ b/document/src/vespa/document/update/fieldpathupdate.cpp @@ -38,7 +38,7 @@ parseDocumentSelection(vespalib::stringref query, const DocumentTypeRepo& repo) } // namespace -FieldPathUpdate::FieldPathUpdate(FieldPathUpdateType type) +FieldPathUpdate::FieldPathUpdate(FieldPathUpdateType type) noexcept : _type(type), _originalFieldPath(), _originalWhereClause() diff --git a/document/src/vespa/document/update/fieldpathupdate.h b/document/src/vespa/document/update/fieldpathupdate.h index fb205b67a60..7074b9edf4c 100644 --- a/document/src/vespa/document/update/fieldpathupdate.h +++ b/document/src/vespa/document/update/fieldpathupdate.h @@ -61,7 +61,7 @@ public: protected: /** To be used for deserialization */ - FieldPathUpdate(FieldPathUpdateType type); + FieldPathUpdate(FieldPathUpdateType type) noexcept; FieldPathUpdate(const FieldPathUpdate &); FieldPathUpdate & operator =(const FieldPathUpdate &); diff --git a/document/src/vespa/document/update/removefieldpathupdate.cpp b/document/src/vespa/document/update/removefieldpathupdate.cpp index 1b94039240b..a2909a2a3e1 100644 --- a/document/src/vespa/document/update/removefieldpathupdate.cpp +++ b/document/src/vespa/document/update/removefieldpathupdate.cpp @@ -8,11 +8,13 @@ namespace document { using namespace fieldvalue; -RemoveFieldPathUpdate::RemoveFieldPathUpdate() +RemoveFieldPathUpdate::RemoveFieldPathUpdate() noexcept : FieldPathUpdate(Remove) { } +RemoveFieldPathUpdate::~RemoveFieldPathUpdate() = default; + RemoveFieldPathUpdate::RemoveFieldPathUpdate(stringref fieldPath, stringref whereClause) : FieldPathUpdate(Remove, fieldPath, whereClause) { @@ -36,7 +38,7 @@ namespace { class RemoveIteratorHandler : public IteratorHandler { public: - RemoveIteratorHandler() {} + RemoveIteratorHandler() = default; ModificationStatus doModify(FieldValue &) override { return ModificationStatus::REMOVED; diff --git a/document/src/vespa/document/update/removefieldpathupdate.h b/document/src/vespa/document/update/removefieldpathupdate.h index df71fcd717c..888effb029d 100644 --- a/document/src/vespa/document/update/removefieldpathupdate.h +++ b/document/src/vespa/document/update/removefieldpathupdate.h @@ -9,12 +9,13 @@ class RemoveFieldPathUpdate final : public FieldPathUpdate { public: /** For deserialization */ - RemoveFieldPathUpdate(); + RemoveFieldPathUpdate() noexcept; RemoveFieldPathUpdate(RemoveFieldPathUpdate &&) noexcept = default; RemoveFieldPathUpdate & operator =(RemoveFieldPathUpdate &&) noexcept = default; RemoveFieldPathUpdate(const RemoveFieldPathUpdate &) = delete; RemoveFieldPathUpdate & operator =(const RemoveFieldPathUpdate &) = delete; RemoveFieldPathUpdate(stringref fieldPath, stringref whereClause = stringref()); + ~RemoveFieldPathUpdate() override; void print(std::ostream& out, bool verbose, const std::string& indent) const override; diff --git a/documentapi/src/tests/policies/policies_test.cpp b/documentapi/src/tests/policies/policies_test.cpp index 19960d148df..33823681765 100644 --- a/documentapi/src/tests/policies/policies_test.cpp +++ b/documentapi/src/tests/policies/policies_test.cpp @@ -50,18 +50,18 @@ private: const DataType *_docType; private: - bool trySelect(TestFrame &frame, uint32_t numSelects, const std::vector<string> &expected); - void setupExternPolicy(TestFrame &frame, mbus::Slobrok &slobrok, const string &pattern, int32_t numEntries = -1); - ContentPolicy &setupContentPolicy(TestFrame &frame, const string ¶m, - const string &pattern = "", int32_t numEntries = -1); + static bool trySelect(TestFrame &frame, uint32_t numSelects, const std::vector<string> &expected); + static void setupExternPolicy(TestFrame &frame, mbus::Slobrok &slobrok, const string &pattern, int32_t numEntries = -1); + static ContentPolicy &setupContentPolicy(TestFrame &frame, const string ¶m, + const string &pattern = "", int32_t numEntries = -1); bool isErrorPolicy(const string &name, const string ¶m); - void assertMirrorReady(const IMirrorAPI &mirror); - void assertMirrorContains(const IMirrorAPI &mirror, const string &pattern, uint32_t numEntries); + static void assertMirrorReady(const IMirrorAPI &mirror); + static void assertMirrorContains(const IMirrorAPI &mirror, const string &pattern, uint32_t numEntries); mbus::Message::UP newPutDocumentMessage(const string &documentId); public: Test(); - ~Test(); + ~Test() override; int Main() override; void testAND(); void testDocumentRouteSelector(); @@ -70,7 +70,7 @@ public: void get_document_messages_are_sent_to_the_route_handling_the_given_document_type(); void testExternSend(); void testExternMultipleSlobroks(); - void testLoadBalancer(); + static void testLoadBalancer(); void testLocalService(); void testLocalServiceCache(); void testProtocol(); @@ -101,8 +101,7 @@ int Test::Main() { TEST_INIT(_argv[0]); - _repo.reset(new DocumentTypeRepo(readDocumenttypesConfig( - TEST_PATH("../../../test/cfg/testdoctypes.cfg")))); + _repo = std::make_shared<DocumentTypeRepo>(readDocumenttypesConfig(TEST_PATH("../../../test/cfg/testdoctypes.cfg"))); _docType = _repo->getDocumentType("testdoc"); testProtocol(); TEST_FLUSH(); @@ -140,7 +139,7 @@ Test::Main() { void Test::testProtocol() { - mbus::IProtocol::SP protocol(new DocumentProtocol(_repo)); + auto protocol = std::make_shared<DocumentProtocol>(_repo); mbus::IRoutingPolicy::UP policy = protocol->createPolicy("AND", ""); ASSERT_TRUE(dynamic_cast<ANDPolicy*>(policy.get()) != nullptr); @@ -218,9 +217,9 @@ Test::requireThatExternPolicySelectsFromExternSlobrok() mbus::Slobrok slobrok; std::vector<mbus::TestServer*> servers; for (uint32_t i = 0; i < 10; ++i) { - mbus::TestServer *server = new mbus::TestServer( + auto *server = new mbus::TestServer( mbus::Identity(make_string("docproc/cluster.default/%d", i)), mbus::RoutingSpec(), slobrok, - mbus::IProtocol::SP(new DocumentProtocol(_repo))); + std::make_shared<DocumentProtocol>(_repo)); servers.push_back(server); server->net.registerSession("chain.default"); } @@ -231,12 +230,12 @@ Test::requireThatExternPolicySelectsFromExternSlobrok() ASSERT_TRUE(frame.select(leaf, 1)); lst.insert(leaf[0]->getRoute().toString()); - leaf[0]->handleReply(mbus::Reply::UP(new mbus::EmptyReply())); + leaf[0]->handleReply(std::make_unique<mbus::EmptyReply>()); ASSERT_TRUE(frame.getReceptor().getReply(TIMEOUT)); } EXPECT_EQUAL(servers.size(), lst.size()); - for (uint32_t i = 0; i < servers.size(); ++i) { - delete servers[i]; + for (auto & server : servers) { + delete server; } } @@ -247,7 +246,7 @@ Test::requireThatExternPolicyMergesOneReplyAsProtocol() frame.setMessage(newPutDocumentMessage("id:ns:testdoc::")); mbus::Slobrok slobrok; mbus::TestServer server(mbus::Identity("docproc/cluster.default/0"), mbus::RoutingSpec(), slobrok, - mbus::IProtocol::SP(new DocumentProtocol(_repo))); + std::make_shared<DocumentProtocol>(_repo)); server.net.registerSession("chain.default"); setupExternPolicy(frame, slobrok, "docproc/cluster.default/0/chain.default", 1); EXPECT_TRUE(frame.testMergeOneReply(server.net.getConnectionSpec() + "/chain.default")); @@ -256,23 +255,18 @@ Test::requireThatExternPolicyMergesOneReplyAsProtocol() mbus::Message::UP Test::newPutDocumentMessage(const string &documentId) { - Document::SP doc(new Document(*_docType, DocumentId(documentId))); - return make_unique<PutDocumentMessage>(doc); + return make_unique<PutDocumentMessage>(std::make_shared<Document>(*_docType, DocumentId(documentId))); } void -Test::setupExternPolicy(TestFrame &frame, mbus::Slobrok &slobrok, const string &pattern, - int32_t numEntries) +Test::setupExternPolicy(TestFrame &frame, mbus::Slobrok &slobrok, const string &pattern, int32_t numEntries) { string param = vespalib::make_string("tcp/localhost:%d;%s", slobrok.port(), pattern.c_str()); frame.setHop(mbus::HopSpec("test", vespalib::make_string("[Extern:%s]", param.c_str()))); mbus::MessageBus &mbus = frame.getMessageBus(); const mbus::HopBlueprint *hop = mbus.getRoutingTable(DocumentProtocol::NAME)->getHop("test"); - const mbus::PolicyDirective dir = static_cast<mbus::PolicyDirective&>(*hop->getDirective(0)); - ExternPolicy &policy = static_cast<ExternPolicy&>(*mbus.getRoutingPolicy( - DocumentProtocol::NAME, - dir.getName(), - dir.getParam())); + const mbus::PolicyDirective &dir = dynamic_cast<const mbus::PolicyDirective&>(*hop->getDirective(0)); + ExternPolicy &policy = dynamic_cast<ExternPolicy&>(*mbus.getRoutingPolicy(DocumentProtocol::NAME, dir.getName(), dir.getParam())); assertMirrorReady(*policy.getMirror()); if (numEntries >= 0) { assertMirrorContains(*policy.getMirror(), pattern, numEntries); @@ -316,7 +310,7 @@ Test::testExternSend() mbus::Slobrok slobrok; mbus::TestServer itr(mbus::Identity("itr"), mbus::RoutingSpec() .addTable(mbus::RoutingTableSpec(DocumentProtocol::NAME) - .addRoute(mbus::RouteSpec("default").addHop("dst")) + .addRoute(std::move(mbus::RouteSpec("default").addHop("dst"))) .addHop(mbus::HopSpec("dst", "dst/session"))), slobrok, std::make_shared<DocumentProtocol>(_repo)); mbus::Receptor ir; @@ -410,7 +404,7 @@ Test::testLocalService() ASSERT_TRUE(frame.select(leaf, 1)); lst.insert(leaf[0]->getRoute().toString()); - leaf[0]->handleReply(mbus::Reply::UP(new mbus::EmptyReply())); + leaf[0]->handleReply(std::make_unique<mbus::EmptyReply>()); ASSERT_TRUE(frame.getReceptor().getReply(TIMEOUT)); } EXPECT_EQUAL(10u, lst.size()); @@ -423,7 +417,7 @@ Test::testLocalService() ASSERT_TRUE(frame.select(leaf, 1)); lst.insert(leaf[0]->getRoute().toString()); - leaf[0]->handleReply(mbus::Reply::UP(new mbus::EmptyReply())); + leaf[0]->handleReply(std::make_unique<mbus::EmptyReply>()); ASSERT_TRUE(frame.getReceptor().getReply(TIMEOUT)); } EXPECT_EQUAL(1u, lst.size()); @@ -444,13 +438,13 @@ Test::testLocalServiceCache() TestFrame barFrame(fooFrame); mbus::HopSpec barHop("test", "docproc/cluster.default/[LocalService]/chain.bar"); - barFrame.setMessage(mbus::Message::UP(new GetDocumentMessage(DocumentId("id:ns:testdoc::bar")))); + barFrame.setMessage(std::make_unique<GetDocumentMessage>(DocumentId("id:ns:testdoc::bar"))); barFrame.setHop(barHop); fooFrame.getMessageBus().setupRouting( mbus::RoutingSpec().addTable(mbus::RoutingTableSpec(DocumentProtocol::NAME) - .addHop(fooHop) - .addHop(barHop))); + .addHop(std::move(fooHop)) + .addHop(std::move(barHop)))); fooFrame.getNetwork().registerSession("0/chain.foo"); fooFrame.getNetwork().registerSession("0/chain.bar"); @@ -464,8 +458,8 @@ Test::testLocalServiceCache() ASSERT_TRUE(barFrame.select(barSelected, 1)); EXPECT_EQUAL("docproc/cluster.default/0/chain.bar", barSelected[0]->getRoute().getHop(0).toString()); - barSelected[0]->handleReply(mbus::Reply::UP(new mbus::EmptyReply())); - fooSelected[0]->handleReply(mbus::Reply::UP(new mbus::EmptyReply())); + barSelected[0]->handleReply(std::make_unique<mbus::EmptyReply>()); + fooSelected[0]->handleReply(std::make_unique<mbus::EmptyReply>()); ASSERT_TRUE(barFrame.getReceptor().getReply(TIMEOUT)); ASSERT_TRUE(fooFrame.getReceptor().getReply(TIMEOUT)); @@ -515,19 +509,19 @@ Test::testRoundRobinCache() TestFrame fooFrame(_repo, "docproc/cluster.default"); mbus::HopSpec fooHop("foo", "[RoundRobin]"); fooHop.addRecipient("docproc/cluster.default/0/chain.foo"); - fooFrame.setMessage(mbus::Message::UP(new GetDocumentMessage(DocumentId("id:ns:testdoc::foo")))); + fooFrame.setMessage(std::make_unique<GetDocumentMessage>(DocumentId("id:ns:testdoc::foo"))); fooFrame.setHop(fooHop); TestFrame barFrame(fooFrame); mbus::HopSpec barHop("bar", "[RoundRobin]"); barHop.addRecipient("docproc/cluster.default/0/chain.bar"); - barFrame.setMessage(mbus::Message::UP(new GetDocumentMessage(DocumentId("id:ns:testdoc::bar")))); + barFrame.setMessage(std::make_unique<GetDocumentMessage>(DocumentId("id:ns:testdoc::bar"))); barFrame.setHop(barHop); fooFrame.getMessageBus().setupRouting( mbus::RoutingSpec().addTable(mbus::RoutingTableSpec(DocumentProtocol::NAME) - .addHop(fooHop) - .addHop(barHop))); + .addHop(std::move(fooHop)) + .addHop(std::move(barHop)))); fooFrame.getNetwork().registerSession("0/chain.foo"); fooFrame.getNetwork().registerSession("0/chain.bar"); @@ -541,8 +535,8 @@ Test::testRoundRobinCache() ASSERT_TRUE(barFrame.select(barSelected, 1)); EXPECT_EQUAL("docproc/cluster.default/0/chain.bar", barSelected[0]->getRoute().getHop(0).toString()); - barSelected[0]->handleReply(mbus::Reply::UP(new mbus::EmptyReply())); - fooSelected[0]->handleReply(mbus::Reply::UP(new mbus::EmptyReply())); + barSelected[0]->handleReply(std::make_unique<mbus::EmptyReply>()); + fooSelected[0]->handleReply(std::make_unique<mbus::EmptyReply>()); ASSERT_TRUE(barFrame.getReceptor().getReply(TIMEOUT)); ASSERT_TRUE(fooFrame.getReceptor().getReply(TIMEOUT)); @@ -568,16 +562,16 @@ Test::multipleGetRepliesAreMergedToFoundDocument() for (uint32_t i = 0, len = selected.size(); i < len; ++i) { Document::SP doc; if (i == 0) { - doc.reset(new Document(*_docType, DocumentId("id:ns:testdoc::yarn"))); + doc = std::make_shared<Document>(*_docType, DocumentId("id:ns:testdoc::yarn")); doc->setLastModified(123456ULL); } - mbus::Reply::UP reply(new GetDocumentReply(std::move(doc))); + auto reply = std::make_unique<GetDocumentReply>(std::move(doc)); selected[i]->handleReply(std::move(reply)); } mbus::Reply::UP reply = frame.getReceptor().getReply(TIMEOUT); EXPECT_TRUE(reply); EXPECT_EQUAL(static_cast<uint32_t>(DocumentProtocol::REPLY_GETDOCUMENT), reply->getType()); - EXPECT_EQUAL(123456ULL, static_cast<GetDocumentReply&>(*reply).getLastModified()); + EXPECT_EQUAL(123456ULL, dynamic_cast<GetDocumentReply&>(*reply).getLastModified()); } void @@ -621,7 +615,7 @@ Test::testDocumentRouteSelector() frame.setMessage(std::move(put)); EXPECT_TRUE(frame.testSelect( StringList().add("foo"))); - frame.setMessage(mbus::Message::UP(new RemoveDocumentMessage(DocumentId("id:ns:testdoc::")))); + frame.setMessage(std::make_unique<RemoveDocumentMessage>(DocumentId("id:ns:testdoc::"))); EXPECT_TRUE(frame.testSelect(StringList().add("foo"))); frame.setMessage(make_unique<UpdateDocumentMessage>( @@ -674,7 +668,7 @@ createDocumentRouteSelectorConfigWithTwoRoutes() } std::unique_ptr<TestFrame> -createFrameWithTwoRoutes(std::shared_ptr<const DocumentTypeRepo> repo) +createFrameWithTwoRoutes(const std::shared_ptr<const DocumentTypeRepo> & repo) { auto result = std::make_unique<TestFrame>(repo); result->setHop(mbus::HopSpec("test", createDocumentRouteSelectorConfigWithTwoRoutes()) @@ -683,13 +677,13 @@ createFrameWithTwoRoutes(std::shared_ptr<const DocumentTypeRepo> repo) } std::unique_ptr<RemoveDocumentMessage> -makeRemove(vespalib::string docId) +makeRemove(vespalib::stringref docId) { return std::make_unique<RemoveDocumentMessage>(DocumentId(docId)); } std::unique_ptr<GetDocumentMessage> -makeGet(vespalib::string docId) +makeGet(vespalib::stringref docId) { return std::make_unique<GetDocumentMessage>(DocumentId(docId)); } @@ -742,9 +736,9 @@ void Test::testLoadBalancer() { LoadBalancer lb("foo", ""); IMirrorAPI::SpecList entries; - entries.push_back(IMirrorAPI::Spec("foo/0/default", "tcp/bar:1")); - entries.push_back(IMirrorAPI::Spec("foo/1/default", "tcp/bar:2")); - entries.push_back(IMirrorAPI::Spec("foo/2/default", "tcp/bar:3")); + entries.emplace_back("foo/0/default", "tcp/bar:1"); + entries.emplace_back("foo/1/default", "tcp/bar:2"); + entries.emplace_back("foo/2/default", "tcp/bar:3"); for (int i = 0; i < 99; i++) { std::pair<string, int> recipient = lb.getRecipient(entries); @@ -796,10 +790,10 @@ Test::requireThatContentPolicyIsRandomWithoutState() mbus::Slobrok slobrok; std::vector<mbus::TestServer*> servers; for (uint32_t i = 0; i < 5; ++i) { - mbus::TestServer *srv = new mbus::TestServer( + auto *srv = new mbus::TestServer( mbus::Identity(vespalib::make_string("storage/cluster.mycluster/distributor/%d", i)), mbus::RoutingSpec(), slobrok, - mbus::IProtocol::SP(new DocumentProtocol(_repo))); + std::make_shared<DocumentProtocol>(_repo)); servers.push_back(srv); srv->net.registerSession("default"); } @@ -816,11 +810,11 @@ Test::requireThatContentPolicyIsRandomWithoutState() std::vector<mbus::RoutingNode*> leaf; ASSERT_TRUE(frame.select(leaf, 1)); lst.insert(leaf[0]->getRoute().toString()); - leaf[0]->handleReply(mbus::Reply::UP(new mbus::EmptyReply())); + leaf[0]->handleReply(std::make_unique<mbus::EmptyReply>()); } EXPECT_EQUAL(servers.size(), lst.size()); - for (uint32_t i = 0; i < servers.size(); ++i) { - delete servers[i]; + for (auto & server : servers) { + delete server; } } @@ -831,9 +825,8 @@ Test::setupContentPolicy(TestFrame &frame, const string ¶m, frame.setHop(mbus::HopSpec("test", vespalib::make_string("[Content:%s]", param.c_str()))); mbus::MessageBus &mbus = frame.getMessageBus(); const mbus::HopBlueprint *hop = mbus.getRoutingTable(DocumentProtocol::NAME)->getHop("test"); - const mbus::PolicyDirective dir = static_cast<mbus::PolicyDirective&>(*hop->getDirective(0)); - ContentPolicy &policy = static_cast<ContentPolicy&>(*mbus.getRoutingPolicy(DocumentProtocol::NAME, - dir.getName(), dir.getParam())); + const mbus::PolicyDirective & dir = dynamic_cast<const mbus::PolicyDirective&>(*hop->getDirective(0)); + ContentPolicy &policy = dynamic_cast<ContentPolicy&>(*mbus.getRoutingPolicy(DocumentProtocol::NAME, dir.getName(), dir.getParam())); policy.initSynchronous(); assertMirrorReady(*policy.getMirror()); if (numEntries >= 0) { @@ -851,7 +844,7 @@ Test::requireThatContentPolicyIsTargetedWithState() mbus::Slobrok slobrok; std::vector<mbus::TestServer*> servers; for (uint32_t i = 0; i < 5; ++i) { - mbus::TestServer *srv = new mbus::TestServer( + auto *srv = new mbus::TestServer( mbus::Identity(vespalib::make_string("storage/cluster.mycluster/distributor/%d", i)), mbus::RoutingSpec(), slobrok, make_shared<DocumentProtocol>(_repo)); @@ -861,14 +854,12 @@ Test::requireThatContentPolicyIsTargetedWithState() string param = vespalib::make_string( "cluster=mycluster;slobroks=tcp/localhost:%d;clusterconfigid=%s;syncinit", slobrok.port(), getDefaultDistributionConfig(2, 5).c_str()); - ContentPolicy &policy = setupContentPolicy( - frame, param, - "storage/cluster.mycluster/distributor/*/default", 5); + ContentPolicy &policy = setupContentPolicy(frame, param, "storage/cluster.mycluster/distributor/*/default", 5); ASSERT_TRUE(policy.getSystemState() == nullptr); { std::vector<mbus::RoutingNode*> leaf; ASSERT_TRUE(frame.select(leaf, 1)); - leaf[0]->handleReply(mbus::Reply::UP(new WrongDistributionReply("distributor:5 storage:5"))); + leaf[0]->handleReply(std::make_unique<WrongDistributionReply>("distributor:5 storage:5")); ASSERT_TRUE(policy.getSystemState() != nullptr); EXPECT_EQUAL(policy.getSystemState()->toString(), "distributor:5 storage:5"); } @@ -877,11 +868,11 @@ Test::requireThatContentPolicyIsTargetedWithState() std::vector<mbus::RoutingNode*> leaf; ASSERT_TRUE(frame.select(leaf, 1)); lst.insert(leaf[0]->getRoute().toString()); - leaf[0]->handleReply(mbus::Reply::UP(new mbus::EmptyReply())); + leaf[0]->handleReply(std::make_unique<mbus::EmptyReply>()); } EXPECT_EQUAL(1u, lst.size()); - for (uint32_t i = 0; i < servers.size(); ++i) { - delete servers[i]; + for (auto & server : servers) { + delete server; } } @@ -907,7 +898,7 @@ Test::requireThatContentPolicyCombinesSystemAndSlobrokState() { std::vector<mbus::RoutingNode*> leaf; ASSERT_TRUE(frame.select(leaf, 1)); - leaf[0]->handleReply(mbus::Reply::UP(new WrongDistributionReply("distributor:99 storage:99"))); + leaf[0]->handleReply(std::make_unique<WrongDistributionReply>("distributor:99 storage:99")); ASSERT_TRUE(policy.getSystemState() != nullptr); EXPECT_EQUAL(policy.getSystemState()->toString(), "distributor:99 storage:99"); } @@ -933,7 +924,7 @@ Test::testSubsetService() std::vector<mbus::RoutingNode*> leaf; ASSERT_TRUE(frame.select(leaf, 1)); lst.insert(leaf[0]->getRoute().toString()); - leaf[0]->handleReply(mbus::Reply::UP(new mbus::EmptyReply())); + leaf[0]->handleReply(std::make_unique<mbus::EmptyReply>()); ASSERT_TRUE(frame.getReceptor().getReply(TIMEOUT)); } ASSERT_TRUE(lst.size() > 1); // must have requeried @@ -952,7 +943,7 @@ Test::testSubsetService() } prev = next; - leaf[0]->handleReply(mbus::Reply::UP(new mbus::EmptyReply())); + leaf[0]->handleReply(std::make_unique<mbus::EmptyReply>()); ASSERT_TRUE(frame.getReceptor().getReply(TIMEOUT)); } @@ -967,7 +958,7 @@ Test::testSubsetService() frame.getNetwork().unregisterSession(route.substr(frame.getIdentity().length() + 1)); ASSERT_TRUE(frame.waitSlobrok("docproc/cluster.default/*/chain.default", 10 - i)); - mbus::Reply::UP reply(new mbus::EmptyReply()); + auto reply = std::make_unique<mbus::EmptyReply>(); reply->addError(mbus::Error(mbus::ErrorCode::NO_ADDRESS_FOR_SERVICE, route)); leaf[0]->handleReply(std::move(reply)); ASSERT_TRUE(frame.getReceptor().getReply(TIMEOUT)); @@ -984,18 +975,18 @@ Test::testSubsetServiceCache() { TestFrame fooFrame(_repo, "docproc/cluster.default"); mbus::HopSpec fooHop("foo", "docproc/cluster.default/[SubsetService:2]/chain.foo"); - fooFrame.setMessage(mbus::Message::UP(new GetDocumentMessage(DocumentId("id:ns:testdoc::foo")))); + fooFrame.setMessage(std::make_unique<GetDocumentMessage>(DocumentId("id:ns:testdoc::foo"))); fooFrame.setHop(fooHop); TestFrame barFrame(fooFrame); mbus::HopSpec barHop("bar", "docproc/cluster.default/[SubsetService:2]/chain.bar"); - barFrame.setMessage(mbus::Message::UP(new GetDocumentMessage(DocumentId("id:ns:testdoc::bar")))); + barFrame.setMessage(std::make_unique<GetDocumentMessage>(DocumentId("id:ns:testdoc::bar"))); barFrame.setHop(barHop); fooFrame.getMessageBus().setupRouting( mbus::RoutingSpec().addTable(mbus::RoutingTableSpec(DocumentProtocol::NAME) - .addHop(fooHop) - .addHop(barHop))); + .addHop(std::move(fooHop)) + .addHop(std::move(barHop)))); fooFrame.getNetwork().registerSession("0/chain.foo"); fooFrame.getNetwork().registerSession("0/chain.bar"); @@ -1009,8 +1000,8 @@ Test::testSubsetServiceCache() ASSERT_TRUE(barFrame.select(barSelected, 1)); EXPECT_EQUAL("docproc/cluster.default/0/chain.bar", barSelected[0]->getRoute().getHop(0).toString()); - barSelected[0]->handleReply(mbus::Reply::UP(new mbus::EmptyReply())); - fooSelected[0]->handleReply(mbus::Reply::UP(new mbus::EmptyReply())); + barSelected[0]->handleReply(std::make_unique<mbus::EmptyReply>()); + fooSelected[0]->handleReply(std::make_unique<mbus::EmptyReply>()); ASSERT_TRUE(barFrame.getReceptor().getReply(TIMEOUT)); ASSERT_TRUE(fooFrame.getReceptor().getReply(TIMEOUT)); @@ -1024,7 +1015,7 @@ Test::trySelect(TestFrame &frame, uint32_t numSelects, const std::vector<string> if (!expected.empty()) { frame.select(leaf, 1); lst.insert(leaf[0]->getRoute().toString()); - leaf[0]->handleReply(mbus::Reply::UP(new mbus::EmptyReply())); + leaf[0]->handleReply(std::make_unique<mbus::EmptyReply>()); } else { frame.select(leaf, 0); } @@ -1037,7 +1028,7 @@ Test::trySelect(TestFrame &frame, uint32_t numSelects, const std::vector<string> LOG(error, "Expected %d recipients, got %d.", (uint32_t)expected.size(), (uint32_t)lst.size()); return false; } - std::set<string>::iterator it = lst.begin(); + auto it = lst.begin(); for (uint32_t i = 0; i < expected.size(); ++i, ++it) { if (*it != expected[i]) { LOG(error, "Expected '%s', got '%s'.", expected[i].c_str(), it->c_str()); diff --git a/documentapi/src/tests/policies/testframe.cpp b/documentapi/src/tests/policies/testframe.cpp index cb33ddd649b..973297a50d1 100644 --- a/documentapi/src/tests/policies/testframe.cpp +++ b/documentapi/src/tests/policies/testframe.cpp @@ -22,7 +22,7 @@ private: string _address; public: - MyServiceAddress(const string &address) : _address(address) {} + explicit MyServiceAddress(const string &address) : _address(address) {} const string &getAddress() { return _address; } }; @@ -32,7 +32,7 @@ private: std::vector<mbus::RoutingNode*> _nodes; public: - MyNetwork(const mbus::RPCNetworkParams ¶ms) : + explicit MyNetwork(const mbus::RPCNetworkParams ¶ms) : mbus::RPCNetwork(params), _nodes() { } @@ -86,7 +86,7 @@ void TestFrame::setHop(const mbus::HopSpec &hop) { _hop = hop; - _mbus->setupRouting(mbus::RoutingSpec().addTable(mbus::RoutingTableSpec(DocumentProtocol::NAME).addHop(_hop))); + _mbus->setupRouting(mbus::RoutingSpec().addTable(mbus::RoutingTableSpec(DocumentProtocol::NAME).addHop(mbus::HopSpec(_hop)))); } bool @@ -111,8 +111,8 @@ TestFrame::testSelect(const std::vector<string> &expected) std::vector<mbus::RoutingNode*> selected; if (!select(selected, expected.size())) { LOG(error, "Failed to select recipients."); - for (size_t i = 0; i < selected.size(); ++i) { - LOG(error, "Selected: %s", selected[i]->getRoute().toString().c_str()); + for (auto & i : selected) { + LOG(error, "Selected: %s", i->getRoute().toString().c_str()); } return false; } @@ -154,7 +154,7 @@ TestFrame::testMerge(const ReplyMap &replies, for (mbus::RoutingNode* node : selected) { string route = node->getRoute().toString(); - ReplyMap::const_iterator mip = replies.find(route); + auto mip = replies.find(route); if (mip == replies.end()) { LOG(error, "Recipient '%s' not expected.", route.c_str()); return false; diff --git a/documentapi/src/tests/policies/testframe.h b/documentapi/src/tests/policies/testframe.h index 3b2af060a45..a609e4b5b5c 100644 --- a/documentapi/src/tests/policies/testframe.h +++ b/documentapi/src/tests/policies/testframe.h @@ -31,8 +31,8 @@ public: * * @param identity The identity to use for the server. */ - TestFrame(const std::shared_ptr<const document::DocumentTypeRepo> &repo, - const string &ident = "anonymous"); + explicit TestFrame(const std::shared_ptr<const document::DocumentTypeRepo> &repo, + const string &ident = "anonymous"); TestFrame &operator=(const TestFrame &) = delete; diff --git a/documentapi/src/vespa/documentapi/messagebus/messages/getbucketlistreply.cpp b/documentapi/src/vespa/documentapi/messagebus/messages/getbucketlistreply.cpp index cbac3e2c4d0..fb15b8b47ab 100644 --- a/documentapi/src/vespa/documentapi/messagebus/messages/getbucketlistreply.cpp +++ b/documentapi/src/vespa/documentapi/messagebus/messages/getbucketlistreply.cpp @@ -23,12 +23,12 @@ GetBucketListReply::BucketInfo::operator==(const GetBucketListReply::BucketInfo return _bucket == rhs._bucket && _bucketInformation == rhs._bucketInformation; } -GetBucketListReply::GetBucketListReply() : +GetBucketListReply::GetBucketListReply() noexcept : DocumentReply(DocumentProtocol::REPLY_GETBUCKETLIST), _buckets() -{ - // empty -} +{ } + +GetBucketListReply::~GetBucketListReply() = default; std::ostream & operator<<(std::ostream &out, const GetBucketListReply::BucketInfo &info) diff --git a/documentapi/src/vespa/documentapi/messagebus/messages/getbucketlistreply.h b/documentapi/src/vespa/documentapi/messagebus/messages/getbucketlistreply.h index 4b8b8179857..a78ef5c3a61 100644 --- a/documentapi/src/vespa/documentapi/messagebus/messages/getbucketlistreply.h +++ b/documentapi/src/vespa/documentapi/messagebus/messages/getbucketlistreply.h @@ -23,10 +23,8 @@ private: std::vector<BucketInfo> _buckets; public: - /** - * Constructs a new reply with no content. - */ - GetBucketListReply(); + GetBucketListReply() noexcept; + ~GetBucketListReply() override; /** * Returns the bucket state contained in this. @@ -40,9 +38,9 @@ public: * * @return The state object. */ - const std::vector<BucketInfo> &getBuckets() const { return _buckets; } + [[nodiscard]] const std::vector<BucketInfo> &getBuckets() const { return _buckets; } - string toString() const override { return "getbucketlistreply"; } + [[nodiscard]] string toString() const override { return "getbucketlistreply"; } }; std::ostream & operator<<(std::ostream &out, const GetBucketListReply::BucketInfo &info); diff --git a/documentapi/src/vespa/documentapi/messagebus/messages/queryresultmessage.cpp b/documentapi/src/vespa/documentapi/messagebus/messages/queryresultmessage.cpp index f79b552a89d..4948fd7bc4d 100644 --- a/documentapi/src/vespa/documentapi/messagebus/messages/queryresultmessage.cpp +++ b/documentapi/src/vespa/documentapi/messagebus/messages/queryresultmessage.cpp @@ -6,9 +6,9 @@ namespace documentapi { QueryResultMessage::QueryResultMessage() = default; -QueryResultMessage::QueryResultMessage(const vdslib::SearchResult & result, const vdslib::DocumentSummary & summary) : +QueryResultMessage::QueryResultMessage(vdslib::SearchResult && result, const vdslib::DocumentSummary & summary) : VisitorMessage(), - _searchResult(result), + _searchResult(std::move(result)), _summary(summary) {} @@ -17,7 +17,7 @@ QueryResultMessage::~QueryResultMessage() = default; DocumentReply::UP QueryResultMessage::doCreateReply() const { - return DocumentReply::UP(new VisitorReply(DocumentProtocol::REPLY_QUERYRESULT)); + return std::make_unique<VisitorReply>(DocumentProtocol::REPLY_QUERYRESULT); } uint32_t diff --git a/documentapi/src/vespa/documentapi/messagebus/messages/queryresultmessage.h b/documentapi/src/vespa/documentapi/messagebus/messages/queryresultmessage.h index 561fc909e86..eb839bdf21f 100644 --- a/documentapi/src/vespa/documentapi/messagebus/messages/queryresultmessage.h +++ b/documentapi/src/vespa/documentapi/messagebus/messages/queryresultmessage.h @@ -32,7 +32,7 @@ public: * * @param result The result to set. */ - QueryResultMessage(const vdslib::SearchResult & result, const vdslib::DocumentSummary & summary); + QueryResultMessage(vdslib::SearchResult && result, const vdslib::DocumentSummary & summary); uint32_t getApproxSize() const override; uint32_t getType() const override; diff --git a/documentapi/src/vespa/documentapi/messagebus/messages/searchresultmessage.cpp b/documentapi/src/vespa/documentapi/messagebus/messages/searchresultmessage.cpp index b71adec745a..8e25e70d749 100644 --- a/documentapi/src/vespa/documentapi/messagebus/messages/searchresultmessage.cpp +++ b/documentapi/src/vespa/documentapi/messagebus/messages/searchresultmessage.cpp @@ -13,9 +13,9 @@ SearchResultMessage::SearchResultMessage() : // empty } -SearchResultMessage::SearchResultMessage(const SearchResult &result) : +SearchResultMessage::SearchResultMessage(SearchResult &&result) : VisitorMessage(), - SearchResult(result) + SearchResult(std::move(result)) { // empty } @@ -23,7 +23,7 @@ SearchResultMessage::SearchResultMessage(const SearchResult &result) : DocumentReply::UP SearchResultMessage::doCreateReply() const { - return DocumentReply::UP(new VisitorReply(DocumentProtocol::REPLY_SEARCHRESULT)); + return std::make_unique<VisitorReply>(DocumentProtocol::REPLY_SEARCHRESULT); } uint32_t diff --git a/documentapi/src/vespa/documentapi/messagebus/messages/searchresultmessage.h b/documentapi/src/vespa/documentapi/messagebus/messages/searchresultmessage.h index a464aa6218b..d22f8197534 100644 --- a/documentapi/src/vespa/documentapi/messagebus/messages/searchresultmessage.h +++ b/documentapi/src/vespa/documentapi/messagebus/messages/searchresultmessage.h @@ -12,23 +12,11 @@ protected: DocumentReply::UP doCreateReply() const override; public: - /** - * Convenience typedefs. - */ using UP = std::unique_ptr<SearchResultMessage>; using SP = std::shared_ptr<SearchResultMessage>; - /** - * Constructs a new search result message for deserialization. - */ SearchResultMessage(); - - /** - * Constructs a new search result message for the given search result. - * - * @param result The result to set. - */ - SearchResultMessage(const vdslib::SearchResult &result); + SearchResultMessage(vdslib::SearchResult &&result); uint32_t getApproxSize() const override; uint32_t getType() const override; @@ -36,4 +24,3 @@ public: }; } - diff --git a/documentapi/src/vespa/documentapi/messagebus/messages/visitor.cpp b/documentapi/src/vespa/documentapi/messagebus/messages/visitor.cpp index 1717b687322..d7bca6cab7c 100644 --- a/documentapi/src/vespa/documentapi/messagebus/messages/visitor.cpp +++ b/documentapi/src/vespa/documentapi/messagebus/messages/visitor.cpp @@ -168,14 +168,12 @@ DocumentListMessage::Entry::serialize(vespalib::GrowableByteBuffer& buf) const buf.putByte(_removeEntry ? 1 : 0); } -DocumentListMessage::DocumentListMessage() = default; - -DocumentListMessage::DocumentListMessage(document::BucketId bid) : +DocumentListMessage::DocumentListMessage() noexcept = default; +DocumentListMessage::~DocumentListMessage() = default; +DocumentListMessage::DocumentListMessage(document::BucketId bid) noexcept : _bucketId(bid), _documents() -{ - // empty -} +{ } DocumentReply::UP DocumentListMessage::doCreateReply() const diff --git a/documentapi/src/vespa/documentapi/messagebus/messages/visitor.h b/documentapi/src/vespa/documentapi/messagebus/messages/visitor.h index 234c4ce1812..00667d006fe 100644 --- a/documentapi/src/vespa/documentapi/messagebus/messages/visitor.h +++ b/documentapi/src/vespa/documentapi/messagebus/messages/visitor.h @@ -265,8 +265,9 @@ protected: DocumentReply::UP doCreateReply() const override; public: - DocumentListMessage(); - DocumentListMessage(document::BucketId bid); + DocumentListMessage() noexcept; + DocumentListMessage(document::BucketId bid) noexcept; + ~DocumentListMessage() override; const document::BucketId& getBucketId() const { return _bucketId; }; void setBucketId(const document::BucketId& id) { _bucketId = id; }; diff --git a/eval/src/vespa/eval/eval/test/gen_spec.cpp b/eval/src/vespa/eval/eval/test/gen_spec.cpp index 988ff035bbb..80cf91ed7e9 100644 --- a/eval/src/vespa/eval/eval/test/gen_spec.cpp +++ b/eval/src/vespa/eval/eval/test/gen_spec.cpp @@ -47,6 +47,22 @@ Sequence Seq(const std::vector<double> &seq) { //----------------------------------------------------------------------------- +DimSpec::DimSpec(const vespalib::string &name, size_t size) noexcept + : _name(name), _size(size), _dict() +{ + assert(_size); +} +DimSpec::DimSpec(const vespalib::string &name, std::vector<vespalib::string> dict) noexcept + : _name(name), _size(), _dict(std::move(dict)) +{ + assert(!_size); +} + +DimSpec::DimSpec(DimSpec &&) noexcept = default; +DimSpec::DimSpec(const DimSpec &) = default; +DimSpec & DimSpec::operator=(DimSpec &&) noexcept = default; +DimSpec & DimSpec::operator=(const DimSpec &) = default; + DimSpec::~DimSpec() = default; std::vector<vespalib::string> @@ -120,9 +136,9 @@ GenSpec::from_desc(const vespalib::string &desc) return {std::move(dim_list)}; } -GenSpec::GenSpec(GenSpec &&other) = default; +GenSpec::GenSpec(GenSpec &&other) noexcept = default; GenSpec::GenSpec(const GenSpec &other) = default; -GenSpec &GenSpec::operator=(GenSpec &&other) = default; +GenSpec &GenSpec::operator=(GenSpec &&other) noexcept = default; GenSpec &GenSpec::operator=(const GenSpec &other) = default; GenSpec::~GenSpec() = default; diff --git a/eval/src/vespa/eval/eval/test/gen_spec.h b/eval/src/vespa/eval/eval/test/gen_spec.h index 30893eb90a9..8a2a6ea12ec 100644 --- a/eval/src/vespa/eval/eval/test/gen_spec.h +++ b/eval/src/vespa/eval/eval/test/gen_spec.h @@ -56,16 +56,12 @@ private: size_t _size; std::vector<vespalib::string> _dict; public: - DimSpec(const vespalib::string &name, size_t size) noexcept - : _name(name), _size(size), _dict() - { - assert(_size); - } - DimSpec(const vespalib::string &name, std::vector<vespalib::string> dict) noexcept - : _name(name), _size(), _dict(std::move(dict)) - { - assert(!_size); - } + DimSpec(const vespalib::string &name, size_t size) noexcept; + DimSpec(const vespalib::string &name, std::vector<vespalib::string> dict) noexcept; + DimSpec(DimSpec &&) noexcept; + DimSpec & operator=(DimSpec &&) noexcept; + DimSpec & operator=(const DimSpec &); + DimSpec(const DimSpec &); ~DimSpec(); static std::vector<vespalib::string> make_dict(size_t size, size_t stride, const vespalib::string &prefix); ValueType::Dimension type() const { @@ -103,9 +99,9 @@ private: seq_t _seq; public: - GenSpec() : _dims(), _cells(CellType::DOUBLE), _seq(N()) {} - GenSpec(double bias) : _dims(), _cells(CellType::DOUBLE), _seq(N(bias)) {} - GenSpec(std::vector<DimSpec> dims_in) : _dims(std::move(dims_in)), _cells(CellType::DOUBLE), _seq(N()) {} + GenSpec() noexcept : _dims(), _cells(CellType::DOUBLE), _seq(N()) {} + GenSpec(double bias) noexcept : _dims(), _cells(CellType::DOUBLE), _seq(N(bias)) {} + GenSpec(std::vector<DimSpec> dims_in) noexcept : _dims(std::move(dims_in)), _cells(CellType::DOUBLE), _seq(N()) {} // Make a GenSpec object from a textual description // (dimension names must be single characters a-z) @@ -114,9 +110,9 @@ public: // 'a2_1b3_2c5_1' -> GenSpec().map("a", 2).map("b", 3, 2).map("c", 5); static GenSpec from_desc(const vespalib::string &desc); - GenSpec(GenSpec &&other); + GenSpec(GenSpec &&other) noexcept; GenSpec(const GenSpec &other); - GenSpec &operator=(GenSpec &&other); + GenSpec &operator=(GenSpec &&other) noexcept; GenSpec &operator=(const GenSpec &other); ~GenSpec(); const std::vector<DimSpec> &dims() const { return _dims; } diff --git a/eval/src/vespa/eval/gp/gp.cpp b/eval/src/vespa/eval/gp/gp.cpp index ca58907e320..f4c8796af23 100644 --- a/eval/src/vespa/eval/gp/gp.cpp +++ b/eval/src/vespa/eval/gp/gp.cpp @@ -28,6 +28,17 @@ Program::Ref map(const std::map<Program::Ref,Program::Ref> &ref_map, Program::Re } // namespace vespalib::gp::<unnamed> +Program::Program(Program &&) noexcept = default; +Program & Program::operator=(Program &&) noexcept = default; +Program::Program(const Program &) = default; +Program::~Program() = default; + +Program::Program(const OpRepo &repo, size_t in_cnt, size_t out_cnt, size_t alt_cnt, size_t gen) + : _repo(repo), _stats(gen), _waste(0.0), + _in_cnt(in_cnt), _out_cnt(out_cnt), _alt_cnt(alt_cnt), + _program(), _frozen(0), _bound() +{} + void Program::assert_valid(Ref ref, size_t limit) const { diff --git a/eval/src/vespa/eval/gp/gp.h b/eval/src/vespa/eval/gp/gp.h index cb5cb0d442e..ec27fa70d44 100644 --- a/eval/src/vespa/eval/gp/gp.h +++ b/eval/src/vespa/eval/gp/gp.h @@ -164,16 +164,13 @@ struct Program : public Sim { size_t rnd_op(Random &rnd) { return rnd.get(0, _repo.max_op()); } Ref rnd_ref(Random &rnd, size_t limit) { return Ref::rnd(rnd, _in_cnt, limit); } - Program(Program &&) = default; - Program &operator=(Program &&) = default; - Program(const Program &) = default; + Program(Program &&) noexcept; + Program &operator=(Program &&) noexcept; + Program(const Program &); Program &operator=(const Program &) = delete; - ~Program() {} + ~Program() override; - Program(const OpRepo &repo, size_t in_cnt, size_t out_cnt, size_t alt_cnt, size_t gen) - : _repo(repo), _stats(gen), _waste(0.0), - _in_cnt(in_cnt), _out_cnt(out_cnt), _alt_cnt(alt_cnt), - _program(), _frozen(0), _bound() {} + Program(const OpRepo &repo, size_t in_cnt, size_t out_cnt, size_t alt_cnt, size_t gen); Ref add_op(size_t code, Ref lhs, Ref rhs); Ref add_forward(Ref ref); diff --git a/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/FileDownloader.java b/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/FileDownloader.java index 63ae8faacfe..68f969df3af 100644 --- a/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/FileDownloader.java +++ b/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/FileDownloader.java @@ -10,7 +10,6 @@ import java.io.File; import java.time.Duration; import java.util.Map; import java.util.Optional; -import java.util.Set; import java.util.concurrent.CompletableFuture; import java.util.concurrent.ExecutionException; import java.util.concurrent.Future; @@ -19,8 +18,6 @@ import java.util.concurrent.TimeoutException; import java.util.logging.Level; import java.util.logging.Logger; -import static com.yahoo.vespa.filedistribution.FileReferenceData.CompressionType; - /** * Handles downloads of files (file references only for now) * @@ -40,20 +37,19 @@ public class FileDownloader implements AutoCloseable { private final FileReferenceDownloader fileReferenceDownloader; private final Downloads downloads = new Downloads(); - public FileDownloader(ConnectionPool connectionPool, Supervisor supervisor, Duration timeout, Set<CompressionType> acceptedCompressionTypes) { - this(connectionPool, supervisor, defaultDownloadDirectory, timeout, defaultSleepBetweenRetries, acceptedCompressionTypes); + public FileDownloader(ConnectionPool connectionPool, Supervisor supervisor, Duration timeout) { + this(connectionPool, supervisor, defaultDownloadDirectory, timeout, defaultSleepBetweenRetries); } - public FileDownloader(ConnectionPool connectionPool, Supervisor supervisor, File downloadDirectory, Duration timeout, Set<CompressionType> acceptedCompressionTypes) { - this(connectionPool, supervisor, downloadDirectory, timeout, defaultSleepBetweenRetries, acceptedCompressionTypes); + public FileDownloader(ConnectionPool connectionPool, Supervisor supervisor, File downloadDirectory, Duration timeout) { + this(connectionPool, supervisor, downloadDirectory, timeout, defaultSleepBetweenRetries); } public FileDownloader(ConnectionPool connectionPool, Supervisor supervisor, File downloadDirectory, Duration timeout, - Duration sleepBetweenRetries, - Set<CompressionType> acceptedCompressionTypes) { + Duration sleepBetweenRetries) { this.connectionPool = connectionPool; this.supervisor = supervisor; this.downloadDirectory = downloadDirectory; @@ -64,8 +60,7 @@ public class FileDownloader implements AutoCloseable { downloads, timeout, sleepBetweenRetries, - downloadDirectory, - acceptedCompressionTypes); + downloadDirectory); if (forceDownload) log.log(Level.INFO, "Force download of file references (download even if file reference exists on disk)"); } diff --git a/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/FileReferenceDownloader.java b/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/FileReferenceDownloader.java index f14de0b9807..0966561ce5d 100644 --- a/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/FileReferenceDownloader.java +++ b/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/FileReferenceDownloader.java @@ -23,6 +23,9 @@ import java.util.logging.Level; import java.util.logging.Logger; import static com.yahoo.vespa.filedistribution.FileReferenceData.CompressionType; +import static com.yahoo.vespa.filedistribution.FileReferenceData.CompressionType.gzip; +import static com.yahoo.vespa.filedistribution.FileReferenceData.CompressionType.lz4; +import static com.yahoo.vespa.filedistribution.FileReferenceData.CompressionType.zstd; /** * Downloads file reference from config server and keeps track of files being downloaded @@ -31,7 +34,8 @@ import static com.yahoo.vespa.filedistribution.FileReferenceData.CompressionType */ public class FileReferenceDownloader { - private final static Logger log = Logger.getLogger(FileReferenceDownloader.class.getName()); + private static final Logger log = Logger.getLogger(FileReferenceDownloader.class.getName()); + private static final Set<CompressionType> defaultAcceptedCompressionTypes = Set.of(gzip, lz4, zstd); private final ExecutorService downloadExecutor = Executors.newFixedThreadPool(Math.max(8, Runtime.getRuntime().availableProcessors()), @@ -42,14 +46,12 @@ public class FileReferenceDownloader { private final Duration sleepBetweenRetries; private final Duration rpcTimeout; private final File downloadDirectory; - private final Set<CompressionType> acceptedCompressionTypes; FileReferenceDownloader(ConnectionPool connectionPool, Downloads downloads, Duration timeout, Duration sleepBetweenRetries, - File downloadDirectory, - Set<CompressionType> acceptedCompressionTypes) { + File downloadDirectory) { this.connectionPool = connectionPool; this.downloads = downloads; this.downloadTimeout = timeout; @@ -57,7 +59,6 @@ public class FileReferenceDownloader { this.downloadDirectory = downloadDirectory; String timeoutString = System.getenv("VESPA_CONFIGPROXY_FILEDOWNLOAD_RPC_TIMEOUT"); this.rpcTimeout = Duration.ofSeconds(timeoutString == null ? 30 : Integer.parseInt(timeoutString)); - this.acceptedCompressionTypes = requireNonEmpty(acceptedCompressionTypes); } private void waitUntilDownloadStarted(FileReferenceDownload fileReferenceDownload) { @@ -139,8 +140,8 @@ public class FileReferenceDownloader { Request request = new Request("filedistribution.serveFile"); request.parameters().add(new StringValue(fileReferenceDownload.fileReference().value())); request.parameters().add(new Int32Value(fileReferenceDownload.downloadFromOtherSourceIfNotFound() ? 0 : 1)); - String[] temp = new String[acceptedCompressionTypes.size()]; - acceptedCompressionTypes.stream().map(Enum::name).toList().toArray(temp); + String[] temp = new String[defaultAcceptedCompressionTypes.size()]; + defaultAcceptedCompressionTypes.stream().map(Enum::name).toList().toArray(temp); request.parameters().add(new StringArray(temp)); return request; } diff --git a/filedistribution/src/test/java/com/yahoo/vespa/filedistribution/FileDownloaderTest.java b/filedistribution/src/test/java/com/yahoo/vespa/filedistribution/FileDownloaderTest.java index ffef06e6367..d593a783064 100644 --- a/filedistribution/src/test/java/com/yahoo/vespa/filedistribution/FileDownloaderTest.java +++ b/filedistribution/src/test/java/com/yahoo/vespa/filedistribution/FileDownloaderTest.java @@ -25,7 +25,6 @@ import java.nio.file.Path; import java.time.Duration; import java.util.Arrays; import java.util.Optional; -import java.util.Set; import java.util.concurrent.ExecutionException; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; @@ -41,7 +40,6 @@ import static org.junit.Assert.fail; public class FileDownloaderTest { private static final Duration sleepBetweenRetries = Duration.ofMillis(10); - private static final Set<FileReferenceData.CompressionType> acceptedCompressionTypes = Set.of(gzip); private MockConnection connection; private FileDownloader fileDownloader; @@ -265,16 +263,6 @@ public class FileDownloaderTest { assertEquals("content", IOUtils.readFile(downloadedFile)); } - @Test - public void testCompressionTypes() { - try { - createDownloader(connection, Duration.ofSeconds(1), Set.of()); - fail("expected to fail when set is empty"); - } catch (IllegalArgumentException e) { - // ignore - } - } - private void writeFileReference(File dir, String fileReferenceString, String fileName) throws IOException { File fileReferenceDir = new File(dir, fileReferenceString); fileReferenceDir.mkdir(); @@ -314,11 +302,7 @@ public class FileDownloaderTest { } private FileDownloader createDownloader(MockConnection connection, Duration timeout) { - return createDownloader(connection, timeout, acceptedCompressionTypes); - } - - private FileDownloader createDownloader(MockConnection connection, Duration timeout, Set<FileReferenceData.CompressionType> acceptedCompressionTypes) { - return new FileDownloader(connection, supervisor, downloadDir, timeout, sleepBetweenRetries, acceptedCompressionTypes); + return new FileDownloader(connection, supervisor, downloadDir, timeout, sleepBetweenRetries); } private static class MockConnection implements ConnectionPool, com.yahoo.vespa.config.Connection { diff --git a/flags/src/main/java/com/yahoo/vespa/flags/Flags.java b/flags/src/main/java/com/yahoo/vespa/flags/Flags.java index 00c0bb89431..93cf9c7c564 100644 --- a/flags/src/main/java/com/yahoo/vespa/flags/Flags.java +++ b/flags/src/main/java/com/yahoo/vespa/flags/Flags.java @@ -217,7 +217,7 @@ public class Flags { public static final UnboundIntFlag MAX_ACTIVATION_INHIBITED_OUT_OF_SYNC_GROUPS = defineIntFlag( "max-activation-inhibited-out-of-sync-groups", 0, - List.of("vekterli"), "2021-02-19", "2023-02-01", + List.of("vekterli"), "2021-02-19", "2023-05-01", "Allows replicas in up to N content groups to not be activated " + "for query visibility if they are out of sync with a majority of other replicas", "Takes effect at redeployment", @@ -225,7 +225,7 @@ public class Flags { public static final UnboundDoubleFlag MIN_NODE_RATIO_PER_GROUP = defineDoubleFlag( "min-node-ratio-per-group", 0.0, - List.of("geirst", "vekterli"), "2021-07-16", "2023-02-01", + List.of("geirst", "vekterli"), "2021-07-16", "2023-05-01", "Minimum ratio of nodes that have to be available (i.e. not Down) in any hierarchic content cluster group for the group to be Up", "Takes effect at redeployment", ZONE_ID, APPLICATION_ID); @@ -332,7 +332,7 @@ public class Flags { public static final UnboundBooleanFlag RESTRICT_DATA_PLANE_BINDINGS = defineFeatureFlag( "restrict-data-plane-bindings", false, - List.of("mortent"), "2022-09-08", "2023-02-01", + List.of("mortent"), "2022-09-08", "2023-05-01", "Use restricted data plane bindings", "Takes effect at redeployment", APPLICATION_ID); @@ -366,11 +366,17 @@ public class Flags { public static final UnboundStringFlag CORE_ENCRYPTION_PUBLIC_KEY_ID = defineStringFlag( "core-encryption-public-key-id", "", - List.of("vekterli"), "2022-11-03", "2023-02-01", + List.of("vekterli"), "2022-11-03", "2023-05-01", "Specifies which public key to use for core dump encryption.", "Takes effect on the next tick.", ZONE_ID, NODE_TYPE, HOSTNAME); - + + public static final UnboundStringFlag ZOOKEEPER_SNAPSHOT_METHOD = defineStringFlag( + "zookeeper-snapshot-method", "gz", + List.of("mpolden"), "2023-02-01", "2023-05-01", + "ZooKeeper snapshot compression method. Valid values are '', 'gz' and 'snappy'", + "Takes effect on node restart"); + /** WARNING: public for testing: All flags should be defined in {@link Flags}. */ public static UnboundBooleanFlag defineFeatureFlag(String flagId, boolean defaultValue, List<String> owners, String createdAt, String expiresAt, String description, diff --git a/fnet/src/tests/frt/rpc/invoke.cpp b/fnet/src/tests/frt/rpc/invoke.cpp index e69513aa980..38f260dd202 100644 --- a/fnet/src/tests/frt/rpc/invoke.cpp +++ b/fnet/src/tests/frt/rpc/invoke.cpp @@ -90,69 +90,11 @@ private: vespalib::Stash _echo_stash; FRT_Values _echo_args; - EchoTest(const EchoTest &); - EchoTest &operator=(const EchoTest &); - public: - EchoTest(FRT_Supervisor *supervisor) - : _echo_stash(), - _echo_args(_echo_stash) - { - FRT_ReflectionBuilder rb(supervisor); - rb.DefineMethod("echo", "*", "*", - FRT_METHOD(EchoTest::RPC_Echo), this); - - FRT_Values *args = &_echo_args; - args->EnsureFree(16); - - args->AddInt8(8); - uint8_t *pt_int8 = args->AddInt8Array(3); - pt_int8[0] = 1; - pt_int8[1] = 2; - pt_int8[2] = 3; - - args->AddInt16(16); - uint16_t *pt_int16 = args->AddInt16Array(3); - pt_int16[0] = 2; - pt_int16[1] = 4; - pt_int16[2] = 6; - - args->AddInt32(32); - uint32_t *pt_int32 = args->AddInt32Array(3); - pt_int32[0] = 4; - pt_int32[1] = 8; - pt_int32[2] = 12; - - args->AddInt64(64); - uint64_t *pt_int64 = args->AddInt64Array(3); - pt_int64[0] = 8; - pt_int64[1] = 16; - pt_int64[2] = 24; - - args->AddFloat(32.5); - float *pt_float = args->AddFloatArray(3); - pt_float[0] = 0.25; - pt_float[1] = 0.5; - pt_float[2] = 0.75; - - args->AddDouble(64.5); - double *pt_double = args->AddDoubleArray(3); - pt_double[0] = 0.1; - pt_double[1] = 0.2; - pt_double[2] = 0.3; - - args->AddString("string"); - FRT_StringValue *pt_string = args->AddStringArray(3); - args->SetString(&pt_string[0], "str1"); - args->SetString(&pt_string[1], "str2"); - args->SetString(&pt_string[2], "str3"); - - args->AddData("data", 4); - FRT_DataValue *pt_data = args->AddDataArray(3); - args->SetData(&pt_data[0], "dat1", 4); - args->SetData(&pt_data[1], "dat2", 4); - args->SetData(&pt_data[2], "dat3", 4); - } + EchoTest(const EchoTest &) = delete; + EchoTest &operator=(const EchoTest &) = delete; + EchoTest(FRT_Supervisor *supervisor); + ~EchoTest() override; bool prepare_params(FRT_RPCRequest &req) { @@ -178,6 +120,66 @@ public: } }; +EchoTest::~EchoTest() = default; + +EchoTest::EchoTest(FRT_Supervisor *supervisor) + : _echo_stash(), + _echo_args(_echo_stash) +{ + FRT_ReflectionBuilder rb(supervisor); + rb.DefineMethod("echo", "*", "*", FRT_METHOD(EchoTest::RPC_Echo), this); + + FRT_Values *args = &_echo_args; + args->EnsureFree(16); + + args->AddInt8(8); + uint8_t *pt_int8 = args->AddInt8Array(3); + pt_int8[0] = 1; + pt_int8[1] = 2; + pt_int8[2] = 3; + + args->AddInt16(16); + uint16_t *pt_int16 = args->AddInt16Array(3); + pt_int16[0] = 2; + pt_int16[1] = 4; + pt_int16[2] = 6; + + args->AddInt32(32); + uint32_t *pt_int32 = args->AddInt32Array(3); + pt_int32[0] = 4; + pt_int32[1] = 8; + pt_int32[2] = 12; + + args->AddInt64(64); + uint64_t *pt_int64 = args->AddInt64Array(3); + pt_int64[0] = 8; + pt_int64[1] = 16; + pt_int64[2] = 24; + + args->AddFloat(32.5); + float *pt_float = args->AddFloatArray(3); + pt_float[0] = 0.25; + pt_float[1] = 0.5; + pt_float[2] = 0.75; + + args->AddDouble(64.5); + double *pt_double = args->AddDoubleArray(3); + pt_double[0] = 0.1; + pt_double[1] = 0.2; + pt_double[2] = 0.3; + + args->AddString("string"); + FRT_StringValue *pt_string = args->AddStringArray(3); + args->SetString(&pt_string[0], "str1"); + args->SetString(&pt_string[1], "str2"); + args->SetString(&pt_string[2], "str3"); + + args->AddData("data", 4); + FRT_DataValue *pt_data = args->AddDataArray(3); + args->SetData(&pt_data[0], "dat1", 4); + args->SetData(&pt_data[1], "dat2", 4); + args->SetData(&pt_data[2], "dat3", 4); +} //------------------------------------------------------------- struct MyAccessFilter : FRT_RequestAccessFilter { diff --git a/jrt_test/src/tests/rpc-error/test-errors.cpp b/jrt_test/src/tests/rpc-error/test-errors.cpp index 86d699e530e..e64c2abfff6 100644 --- a/jrt_test/src/tests/rpc-error/test-errors.cpp +++ b/jrt_test/src/tests/rpc-error/test-errors.cpp @@ -12,6 +12,8 @@ private: FRT_Target *target; public: + TestErrors(); + ~TestErrors() override; void init(const char *spec) { client = & server.supervisor(); target = client->GetTarget(spec); @@ -31,6 +33,8 @@ public: int Main() override; }; +TestErrors::TestErrors() = default; +TestErrors::~TestErrors() = default; void TestErrors::testNoError() diff --git a/logd/src/tests/rpc_forwarder/rpc_forwarder_test.cpp b/logd/src/tests/rpc_forwarder/rpc_forwarder_test.cpp index f5349face36..6fa56dfe5c4 100644 --- a/logd/src/tests/rpc_forwarder/rpc_forwarder_test.cpp +++ b/logd/src/tests/rpc_forwarder/rpc_forwarder_test.cpp @@ -42,19 +42,8 @@ struct RpcServer : public FRT_Invokable { bool reply_with_proto_response; public: - RpcServer() - : server(), - request_count(0), - messages(), - reply_with_error(false), - reply_with_proto_response(true) - { - FRT_ReflectionBuilder builder(&server.supervisor()); - builder.DefineMethod("vespa.logserver.archiveLogMessages", "bix", "bix", - FRT_METHOD(RpcServer::rpc_archive_log_messages), this); - server.supervisor().Listen(0); - } - ~RpcServer() = default; + RpcServer(); + ~RpcServer() override; int get_listen_port() { return server.supervisor().GetListenPort(); } @@ -81,6 +70,20 @@ public: } }; +RpcServer::RpcServer() + : server(), + request_count(0), + messages(), + reply_with_error(false), + reply_with_proto_response(true) +{ + FRT_ReflectionBuilder builder(&server.supervisor()); + builder.DefineMethod("vespa.logserver.archiveLogMessages", "bix", "bix", + FRT_METHOD(RpcServer::rpc_archive_log_messages), this); + server.supervisor().Listen(0); +} +RpcServer::~RpcServer() = default; + std::string make_log_line(const std::string& level, const std::string& payload) { diff --git a/logd/src/tests/watcher/watcher_test.cpp b/logd/src/tests/watcher/watcher_test.cpp index 55ba720c0cb..fa8c2ffe932 100644 --- a/logd/src/tests/watcher/watcher_test.cpp +++ b/logd/src/tests/watcher/watcher_test.cpp @@ -61,14 +61,8 @@ struct DummyForwarder : public Forwarder { std::mutex lock; std::condition_variable cond; std::vector<std::string> lines; - DummyForwarder() - : Forwarder(), - lock(), - cond(), - lines() - { - } - ~DummyForwarder() override = default; + DummyForwarder(); + ~DummyForwarder() override; void forwardLine(std::string_view log_line) override { std::lock_guard guard(lock); lines.emplace_back(log_line); @@ -87,6 +81,14 @@ struct DummyForwarder : public Forwarder { } }; +DummyForwarder::DummyForwarder() + : Forwarder(), + lock(), + cond(), + lines() +{ } +DummyForwarder::~DummyForwarder() = default; + struct WatcherFixture { DummyForwarder fwd; diff --git a/messagebus/src/tests/advancedrouting/advancedrouting.cpp b/messagebus/src/tests/advancedrouting/advancedrouting.cpp index fbe6c8f77bd..1711122df8a 100644 --- a/messagebus/src/tests/advancedrouting/advancedrouting.cpp +++ b/messagebus/src/tests/advancedrouting/advancedrouting.cpp @@ -123,7 +123,7 @@ Test::testAdvanced(TestData &data) data._srcServer.mb.setupRouting(RoutingSpec().addTable(RoutingTableSpec(SimpleProtocol::NAME) .addHop(HopSpec("bar", "dst/bar")) .addHop(HopSpec("baz", "dst/baz")) - .addRoute(RouteSpec("baz").addHop("baz")))); + .addRoute(std::move(RouteSpec("baz").addHop("baz"))))); string route = vespalib::make_string("[Custom:%s,bar,route:baz,dst/cox,?dst/unknown]", data._fooSession->getConnectionSpec().c_str()); EXPECT_TRUE(data._srcSession->send(createMessage("msg"), Route::parse(route)).isAccepted()); diff --git a/messagebus/src/tests/context/context.cpp b/messagebus/src/tests/context/context.cpp index 85b92cc6fa0..e91946aab05 100644 --- a/messagebus/src/tests/context/context.cpp +++ b/messagebus/src/tests/context/context.cpp @@ -18,7 +18,7 @@ struct Handler : public IMessageHandler { DestinationSession::UP session; - Handler(MessageBus &mb) : session() { + explicit Handler(MessageBus &mb) : session() { session = mb.createDestinationSession("session", true, *this); } ~Handler() override { @@ -33,7 +33,7 @@ RoutingSpec getRouting() { return RoutingSpec() .addTable(RoutingTableSpec("Simple") .addHop(HopSpec("test", "test/session")) - .addRoute(RouteSpec("test").addHop("test"))); + .addRoute(std::move(RouteSpec("test").addHop("test")))); } TEST_SETUP(Test); diff --git a/messagebus/src/tests/error/error.cpp b/messagebus/src/tests/error/error.cpp index da7966621fa..d97440f12e0 100644 --- a/messagebus/src/tests/error/error.cpp +++ b/messagebus/src/tests/error/error.cpp @@ -4,14 +4,11 @@ #include <vespa/messagebus/emptyreply.h> #include <vespa/messagebus/errorcode.h> #include <vespa/messagebus/intermediatesession.h> -#include <vespa/messagebus/messagebus.h> #include <vespa/messagebus/routing/routingspec.h> #include <vespa/messagebus/sourcesession.h> #include <vespa/messagebus/sourcesessionparams.h> #include <vespa/messagebus/testlib/receptor.h> #include <vespa/messagebus/testlib/simplemessage.h> -#include <vespa/messagebus/testlib/simplereply.h> -#include <vespa/messagebus/testlib/simpleprotocol.h> #include <vespa/messagebus/testlib/slobrok.h> #include <vespa/messagebus/testlib/testserver.h> #include <vespa/vespalib/testkit/testapp.h> @@ -23,7 +20,7 @@ RoutingSpec getRouting() { .addTable(RoutingTableSpec("Simple") .addHop(HopSpec("pxy", "test/pxy/session")) .addHop(HopSpec("dst", "test/dst/session")) - .addRoute(RouteSpec("test").addHop("pxy").addHop("dst"))); + .addRoute(std::move(RouteSpec("test").addHop("pxy").addHop("dst")))); } TEST("error_test") { diff --git a/messagebus/src/tests/messagebus/messagebus.cpp b/messagebus/src/tests/messagebus/messagebus.cpp index 1abf5ae8ab2..ba466c4210e 100644 --- a/messagebus/src/tests/messagebus/messagebus.cpp +++ b/messagebus/src/tests/messagebus/messagebus.cpp @@ -30,14 +30,14 @@ struct Base { return RoutingSpec() .addTable(RoutingTableSpec("Simple") .addHop(HopSpec("DocProc", "docproc/*/session")) - .addHop(HopSpec("Search", "search/[All]/[Hash]/session") + .addHop(std::move(HopSpec("Search", "search/[All]/[Hash]/session") .addRecipient("search/r.0/c.0/session") .addRecipient("search/r.0/c.1/session") .addRecipient("search/r.1/c.0/session") - .addRecipient("search/r.1/c.1/session")) - .addRoute(RouteSpec("Index").addHop("DocProc").addHop("Search")) - .addRoute(RouteSpec("DocProc").addHop("DocProc")) - .addRoute(RouteSpec("Search").addHop("Search"))); + .addRecipient("search/r.1/c.1/session"))) + .addRoute(std::move(RouteSpec("Index").addHop("DocProc").addHop("Search"))) + .addRoute(std::move(RouteSpec("DocProc").addHop("DocProc"))) + .addRoute(std::move(RouteSpec("Search").addHop("Search")))); } bool waitQueueSize(uint32_t size) { for (uint32_t i = 0; i < 1000; ++i) { @@ -71,8 +71,11 @@ struct Server : public Base { { // empty } + ~Server() override; }; +Server::~Server() = default; + struct DocProc : public Server { using UP = std::unique_ptr<DocProc>; IntermediateSession::UP session; diff --git a/messagebus/src/tests/messageordering/messageordering.cpp b/messagebus/src/tests/messageordering/messageordering.cpp index e82e58d1730..d83ef59be28 100644 --- a/messagebus/src/tests/messageordering/messageordering.cpp +++ b/messagebus/src/tests/messageordering/messageordering.cpp @@ -23,7 +23,7 @@ getRouting() return RoutingSpec() .addTable(RoutingTableSpec("Simple") .addHop(HopSpec("dst", "test/dst/session")) - .addRoute(RouteSpec("test").addHop("dst"))); + .addRoute(std::move(RouteSpec("test").addHop("dst")))); } class MultiReceptor : public IMessageHandler @@ -113,7 +113,7 @@ VerifyReplyReceptor::handleReply(Reply::UP reply) LOG(warning, "%s", ss.str().c_str()); } else { vespalib::string expected(vespalib::make_string("%d", _replyCount)); - auto & simpleReply(static_cast<SimpleReply&>(*reply)); + auto & simpleReply(dynamic_cast<SimpleReply&>(*reply)); if (simpleReply.getValue() != expected) { std::stringstream ss; ss << "Received out-of-sequence reply! Expected " diff --git a/messagebus/src/tests/routing/routing.cpp b/messagebus/src/tests/routing/routing.cpp index acdcf55fe52..2c4056b868d 100644 --- a/messagebus/src/tests/routing/routing.cpp +++ b/messagebus/src/tests/routing/routing.cpp @@ -804,7 +804,7 @@ void Test::testRecognizeRouteDirective(TestData &data) { data._srcServer.mb.setupRouting(RoutingSpec().addTable(RoutingTableSpec(SimpleProtocol::NAME) - .addRoute(RouteSpec("dst").addHop("dst/session")) + .addRoute(std::move(RouteSpec("dst").addHop("dst/session"))) .addHop(HopSpec("dir", "route:dst")))); EXPECT_TRUE(data._srcSession->send(createMessage("msg"), Route::parse("dir")).isAccepted()); Message::UP msg = data._dstHandler.getMessage(RECEPTOR_TIMEOUT); @@ -819,7 +819,7 @@ void Test::testRecognizeRouteName(TestData &data) { data._srcServer.mb.setupRouting(RoutingSpec().addTable(RoutingTableSpec(SimpleProtocol::NAME) - .addRoute(RouteSpec("dst").addHop("dst/session")))); + .addRoute(std::move(RouteSpec("dst").addHop("dst/session"))))); EXPECT_TRUE(data._srcSession->send(createMessage("msg"), Route::parse("dst")).isAccepted()); Message::UP msg = data._dstHandler.getMessage(RECEPTOR_TIMEOUT); ASSERT_TRUE(msg); @@ -846,7 +846,7 @@ void Test::testRouteResolutionOverflow(TestData &data) { data._srcServer.mb.setupRouting(RoutingSpec().addTable(RoutingTableSpec(SimpleProtocol::NAME) - .addRoute(RouteSpec("foo").addHop("route:foo")))); + .addRoute(std::move(RouteSpec("foo").addHop("route:foo"))))); EXPECT_TRUE(data._srcSession->send(createMessage("msg"), "foo").isAccepted()); Reply::UP reply = data._srcHandler.getReply(RECEPTOR_TIMEOUT); ASSERT_TRUE(reply); @@ -858,7 +858,7 @@ void Test::testInsertRoute(TestData &data) { data._srcServer.mb.setupRouting(RoutingSpec().addTable(RoutingTableSpec(SimpleProtocol::NAME) - .addRoute(RouteSpec("foo").addHop("dst/session").addHop("bar")))); + .addRoute(std::move(RouteSpec("foo").addHop("dst/session").addHop("bar"))))); EXPECT_TRUE(data._srcSession->send(createMessage("msg"), Route::parse("route:foo baz")).isAccepted()); Message::UP msg = data._dstHandler.getMessage(RECEPTOR_TIMEOUT); ASSERT_TRUE(msg); @@ -1302,7 +1302,7 @@ void Test::testHopBlueprintIgnoresReply(TestData &data) { data._srcServer.mb.setupRouting(RoutingSpec().addTable(RoutingTableSpec(SimpleProtocol::NAME) - .addHop(HopSpec("foo", "dst/session").setIgnoreResult(true)))); + .addHop(std::move(HopSpec("foo", "dst/session").setIgnoreResult(true))))); EXPECT_TRUE(data._srcSession->send(createMessage("msg"), Route::parse("foo")).isAccepted()); Message::UP msg = data._dstHandler.getMessage(RECEPTOR_TIMEOUT); ASSERT_TRUE(msg); @@ -1419,7 +1419,7 @@ Test::requireThatIgnoreFlagPersistsThroughHopLookup(TestData &data) void Test::requireThatIgnoreFlagPersistsThroughRouteLookup(TestData &data) { - setupRouting(data, RoutingTableSpec(SimpleProtocol::NAME).addRoute(RouteSpec("foo").addHop("dst/unknown"))); + setupRouting(data, RoutingTableSpec(SimpleProtocol::NAME).addRoute(std::move(RouteSpec("foo").addHop("dst/unknown")))); ASSERT_TRUE(testSend(data, "?foo")); ASSERT_TRUE(testTrace(data, StringList().add("Ignoring errors in reply."))); } diff --git a/messagebus/src/tests/routingcontext/routingcontext.cpp b/messagebus/src/tests/routingcontext/routingcontext.cpp index aba50156412..c21725a482e 100644 --- a/messagebus/src/tests/routingcontext/routingcontext.cpp +++ b/messagebus/src/tests/routingcontext/routingcontext.cpp @@ -59,7 +59,7 @@ private: CustomPolicyFactory &_factory; public: - CustomPolicy(CustomPolicyFactory &factory); + explicit CustomPolicy(CustomPolicyFactory &factory); void select(RoutingContext &ctx) override; void merge(RoutingContext &ctx) override; }; @@ -73,21 +73,19 @@ CustomPolicy::CustomPolicy(CustomPolicyFactory &factory) : void CustomPolicy::select(RoutingContext &ctx) { - Reply::UP reply(new EmptyReply()); + auto reply = std::make_unique<EmptyReply>(); reply->getTrace().setLevel(9); const std::vector<Route> &all = ctx.getAllRecipients(); if (_factory._expectedAll.size() == all.size()) { ctx.trace(1, make_string("Got %d expected recipients.", (uint32_t)all.size())); - for (std::vector<Route>::const_iterator it = all.begin(); - it != all.end(); ++it) - { - if (find(_factory._expectedAll.begin(), _factory._expectedAll.end(), it->toString()) != _factory._expectedAll.end()) { - ctx.trace(1, make_string("Got expected recipient '%s'.", it->toString().c_str())); + for (const auto & it : all) { + if (find(_factory._expectedAll.begin(), _factory._expectedAll.end(), it.toString()) != _factory._expectedAll.end()) { + ctx.trace(1, make_string("Got expected recipient '%s'.", it.toString().c_str())); } else { reply->addError(Error(ErrorCode::APP_FATAL_ERROR, make_string("Matched recipient '%s' not expected.", - it->toString().c_str()))); + it.toString().c_str()))); } } } else { @@ -115,15 +113,12 @@ CustomPolicy::select(RoutingContext &ctx) ctx.getMatchedRecipients(matched); if (_factory._expectedMatched.size() == matched.size()) { ctx.trace(1, make_string("Got %d expected recipients.", (uint32_t)matched.size())); - for (std::vector<Route>::iterator it = matched.begin(); - it != matched.end(); ++it) - { - if (find(_factory._expectedMatched.begin(), _factory._expectedMatched.end(), it->toString()) != _factory._expectedMatched.end()) { - ctx.trace(1, make_string("Got matched recipient '%s'.", it->toString().c_str())); + for (auto & it : matched) { + if (find(_factory._expectedMatched.begin(), _factory._expectedMatched.end(), it.toString()) != _factory._expectedMatched.end()) { + ctx.trace(1, make_string("Got matched recipient '%s'.", it.toString().c_str())); } else { reply->addError(Error(ErrorCode::APP_FATAL_ERROR, - make_string("Matched recipient '%s' not expected.", - it->toString().c_str()))); + make_string("Matched recipient '%s' not expected.", it.toString().c_str()))); } } } else { @@ -134,10 +129,8 @@ CustomPolicy::select(RoutingContext &ctx) } if (!reply->hasErrors() && _factory._forward) { - for (std::vector<Route>::iterator it = matched.begin(); - it != matched.end(); ++it) - { - ctx.addChild(*it); + for (auto & it : matched) { + ctx.addChild(it); } } else { ctx.setReply(std::move(reply)); @@ -258,7 +251,7 @@ TestData::start() Message::UP Test::createMessage(const string &msg) { - Message::UP ret(new SimpleMessage(msg)); + auto ret = std::make_unique<SimpleMessage>(msg); ret->getTrace().setLevel(9); return ret; } @@ -289,22 +282,22 @@ void Test::testSingleDirective(TestData &data) { IProtocol::SP protocol(new SimpleProtocol()); - SimpleProtocol &simple = static_cast<SimpleProtocol&>(*protocol); + auto &simple = dynamic_cast<SimpleProtocol&>(*protocol); simple.addPolicyFactory("Custom", SimpleProtocol::IPolicyFactory::SP(new CustomPolicyFactory( false, StringList().add("foo").add("bar").add("baz/cox"), StringList().add("foo").add("bar")))); data._srcServer.mb.putProtocol(protocol); data._srcServer.mb.setupRouting(RoutingSpec().addTable(RoutingTableSpec(SimpleProtocol::NAME) - .addRoute(RouteSpec("myroute").addHop("myhop")) - .addHop(HopSpec("myhop", "[Custom]") + .addRoute(std::move(RouteSpec("myroute").addHop("myhop"))) + .addHop(std::move(HopSpec("myhop", "[Custom]") .addRecipient("foo") .addRecipient("bar") - .addRecipient("baz/cox")))); + .addRecipient("baz/cox"))))); for (uint32_t i = 0; i < 2; ++i) { EXPECT_TRUE(data._srcSession->send(createMessage("msg"), "myroute").isAccepted()); Reply::UP reply = data._srcHandler.getReply(); - ASSERT_TRUE(reply.get() != NULL); + ASSERT_TRUE(reply); printf("%s", reply->getTrace().toString().c_str()); EXPECT_TRUE(!reply->hasErrors()); } @@ -314,24 +307,24 @@ void Test::testMoreDirectives(TestData &data) { IProtocol::SP protocol(new SimpleProtocol()); - SimpleProtocol &simple = static_cast<SimpleProtocol&>(*protocol); + auto &simple = dynamic_cast<SimpleProtocol&>(*protocol); simple.addPolicyFactory("Custom", SimpleProtocol::IPolicyFactory::SP(new CustomPolicyFactory( false, StringList().add("foo").add("foo/bar").add("foo/bar0/baz").add("foo/bar1/baz").add("foo/bar/baz/cox"), StringList().add("foo/bar0/baz").add("foo/bar1/baz")))); data._srcServer.mb.putProtocol(protocol); data._srcServer.mb.setupRouting(RoutingSpec().addTable(RoutingTableSpec(SimpleProtocol::NAME) - .addRoute(RouteSpec("myroute").addHop("myhop")) - .addHop(HopSpec("myhop", "foo/[Custom]/baz") + .addRoute(std::move(RouteSpec("myroute").addHop("myhop"))) + .addHop(std::move(HopSpec("myhop", "foo/[Custom]/baz") .addRecipient("foo") .addRecipient("foo/bar") .addRecipient("foo/bar0/baz") .addRecipient("foo/bar1/baz") - .addRecipient("foo/bar/baz/cox")))); + .addRecipient("foo/bar/baz/cox"))))); for (uint32_t i = 0; i < 2; ++i) { EXPECT_TRUE(data._srcSession->send(createMessage("msg"), "myroute").isAccepted()); Reply::UP reply = data._srcHandler.getReply(); - ASSERT_TRUE(reply.get() != NULL); + ASSERT_TRUE(reply); printf("%s", reply->getTrace().toString().c_str()); EXPECT_TRUE(!reply->hasErrors()); } @@ -341,7 +334,7 @@ void Test::testRecipientsRemain(TestData &data) { IProtocol::SP protocol(new SimpleProtocol()); - SimpleProtocol &simple = static_cast<SimpleProtocol&>(*protocol); + auto &simple = dynamic_cast<SimpleProtocol&>(*protocol); simple.addPolicyFactory("First", SimpleProtocol::IPolicyFactory::SP(new CustomPolicyFactory( true, StringList().add("foo/bar"), @@ -352,13 +345,13 @@ Test::testRecipientsRemain(TestData &data) StringList().add("foo/bar")))); data._srcServer.mb.putProtocol(protocol); data._srcServer.mb.setupRouting(RoutingSpec().addTable(RoutingTableSpec(SimpleProtocol::NAME) - .addRoute(RouteSpec("myroute").addHop("myhop")) - .addHop(HopSpec("myhop", "[First]/[Second]") - .addRecipient("foo/bar")))); + .addRoute(std::move(RouteSpec("myroute").addHop("myhop"))) + .addHop(std::move(HopSpec("myhop", "[First]/[Second]") + .addRecipient("foo/bar"))))); for (uint32_t i = 0; i < 2; ++i) { EXPECT_TRUE(data._srcSession->send(createMessage("msg"), "myroute").isAccepted()); Reply::UP reply = data._srcHandler.getReply(); - ASSERT_TRUE(reply.get() != NULL); + ASSERT_TRUE(reply); printf("%s", reply->getTrace().toString().c_str()); EXPECT_TRUE(!reply->hasErrors()); } @@ -367,25 +360,22 @@ Test::testRecipientsRemain(TestData &data) void Test::testConstRoute(TestData &data) { - IProtocol::SP protocol(new SimpleProtocol()); - SimpleProtocol &simple = static_cast<SimpleProtocol&>(*protocol); + auto protocol = std::make_shared<SimpleProtocol>(); + auto &simple = dynamic_cast<SimpleProtocol&>(*protocol); simple.addPolicyFactory("DocumentRouteSelector", - SimpleProtocol::IPolicyFactory::SP(new CustomPolicyFactory( - true, - StringList().add("dst"), - StringList().add("dst")))); + std::make_unique<CustomPolicyFactory>(true, StringList().add("dst"), StringList().add("dst"))); data._srcServer.mb.putProtocol(protocol); data._srcServer.mb.setupRouting(RoutingSpec().addTable(RoutingTableSpec(SimpleProtocol::NAME) - .addRoute(RouteSpec("default").addHop("indexing")) - .addHop(HopSpec("indexing", "[DocumentRouteSelector]").addRecipient("dst")) + .addRoute(std::move(RouteSpec("default").addHop("indexing"))) + .addHop(std::move(HopSpec("indexing", "[DocumentRouteSelector]").addRecipient("dst"))) .addHop(HopSpec("dst", "dst/session")))); for (uint32_t i = 0; i < 2; ++i) { EXPECT_TRUE(data._srcSession->send(createMessage("msg"), Route::parse("route:default")).isAccepted()); Message::UP msg = data._dstHandler.getMessage(TIMEOUT); - ASSERT_TRUE(msg.get() != NULL); + ASSERT_TRUE(msg); data._dstSession->acknowledge(std::move(msg)); Reply::UP reply = data._srcHandler.getReply(); - ASSERT_TRUE(reply.get() != NULL); + ASSERT_TRUE(reply); printf("%s", reply->getTrace().toString().c_str()); EXPECT_TRUE(!reply->hasErrors()); } diff --git a/messagebus/src/tests/routingspec/routingspec.cpp b/messagebus/src/tests/routingspec/routingspec.cpp index c3e6754ee7d..e7f4dec4f48 100644 --- a/messagebus/src/tests/routingspec/routingspec.cpp +++ b/messagebus/src/tests/routingspec/routingspec.cpp @@ -94,31 +94,31 @@ Test::testConstructors() { RoutingSpec spec = RoutingSpec() .addTable(RoutingTableSpec("foo") - .addHop(HopSpec("foo-h1", "foo-h1-sel") + .addHop(std::move(HopSpec("foo-h1", "foo-h1-sel") .addRecipient("foo-h1-r1") - .addRecipient("foo-h1-r2")) - .addHop(HopSpec("foo-h2", "foo-h2-sel") + .addRecipient("foo-h1-r2"))) + .addHop(std::move(HopSpec("foo-h2", "foo-h2-sel") .addRecipient("foo-h2-r1") - .addRecipient("foo-h2-r2")) - .addRoute(RouteSpec("foo-r1") + .addRecipient("foo-h2-r2"))) + .addRoute(std::move(RouteSpec("foo-r1") .addHop("foo-h1") - .addHop("foo-h2")) - .addRoute(RouteSpec("foo-r2") + .addHop("foo-h2"))) + .addRoute(std::move(RouteSpec("foo-r2") .addHop("foo-h2") - .addHop("foo-h1"))) + .addHop("foo-h1")))) .addTable(RoutingTableSpec("bar") - .addHop(HopSpec("bar-h1", "bar-h1-sel") + .addHop(std::move(HopSpec("bar-h1", "bar-h1-sel") .addRecipient("bar-h1-r1") - .addRecipient("bar-h1-r2")) - .addHop(HopSpec("bar-h2", "bar-h2-sel") + .addRecipient("bar-h1-r2"))) + .addHop(std::move(HopSpec("bar-h2", "bar-h2-sel") .addRecipient("bar-h2-r1") - .addRecipient("bar-h2-r2")) - .addRoute(RouteSpec("bar-r1") + .addRecipient("bar-h2-r2"))) + .addRoute(std::move(RouteSpec("bar-r1") .addHop("bar-h1") - .addHop("bar-h2")) - .addRoute(RouteSpec("bar-r2") + .addHop("bar-h2"))) + .addRoute(std::move(RouteSpec("bar-r2") .addHop("bar-h2") - .addHop("bar-h1"))); + .addHop("bar-h1")))); EXPECT_TRUE(testRouting(spec)); RoutingSpec specCopy = spec; @@ -184,20 +184,20 @@ Test::testConfigGeneration() .addHop(HopSpec("myhop1", "myselector1"))))); EXPECT_TRUE(testConfig(RoutingSpec().addTable(RoutingTableSpec("mytable1") .addHop(HopSpec("myhop1", "myselector1")) - .addRoute(RouteSpec("myroute1").addHop("myhop1"))))); + .addRoute(std::move(RouteSpec("myroute1").addHop("myhop1")))))); EXPECT_TRUE(testConfig(RoutingSpec().addTable(RoutingTableSpec("mytable1") .addHop(HopSpec("myhop1", "myselector1")) .addHop(HopSpec("myhop2", "myselector2")) - .addRoute(RouteSpec("myroute1").addHop("myhop1")) - .addRoute(RouteSpec("myroute2").addHop("myhop2")) - .addRoute(RouteSpec("myroute12").addHop("myhop1").addHop("myhop2"))))); + .addRoute(std::move(RouteSpec("myroute1").addHop("myhop1"))) + .addRoute(std::move(RouteSpec("myroute2").addHop("myhop2"))) + .addRoute(std::move(RouteSpec("myroute12").addHop("myhop1").addHop("myhop2")))))); EXPECT_TRUE(testConfig(RoutingSpec() .addTable(RoutingTableSpec("mytable1") .addHop(HopSpec("myhop1", "myselector1")) .addHop(HopSpec("myhop2", "myselector2")) - .addRoute(RouteSpec("myroute1").addHop("myhop1")) - .addRoute(RouteSpec("myroute2").addHop("myhop2")) - .addRoute(RouteSpec("myroute12").addHop("myhop1").addHop("myhop2"))) + .addRoute(std::move(RouteSpec("myroute1").addHop("myhop1"))) + .addRoute(std::move(RouteSpec("myroute2").addHop("myhop2"))) + .addRoute(std::move(RouteSpec("myroute12").addHop("myhop1").addHop("myhop2")))) .addTable(RoutingTableSpec("mytable2")))); EXPECT_EQUAL("routingtable[2]\n" @@ -222,11 +222,11 @@ Test::testConfigGeneration() .addTable(RoutingTableSpec("mytable1")) .addTable(RoutingTableSpec("mytable2") .addHop(HopSpec("myhop1", "myselector1")) - .addHop(HopSpec("myhop2", "myselector2").setIgnoreResult(true)) - .addHop(HopSpec("myhop1", "myselector3") + .addHop(std::move(HopSpec("myhop2", "myselector2").setIgnoreResult(true))) + .addHop(std::move(HopSpec("myhop1", "myselector3") .addRecipient("myrecipient1") - .addRecipient("myrecipient2")) - .addRoute(RouteSpec("myroute1").addHop("myhop1"))).toString()); + .addRecipient("myrecipient2"))) + .addRoute(std::move(RouteSpec("myroute1").addHop("myhop1")))).toString()); } bool diff --git a/messagebus/src/tests/simple-roundtrip/simple-roundtrip.cpp b/messagebus/src/tests/simple-roundtrip/simple-roundtrip.cpp index d3736f067c9..911c3d38e5e 100644 --- a/messagebus/src/tests/simple-roundtrip/simple-roundtrip.cpp +++ b/messagebus/src/tests/simple-roundtrip/simple-roundtrip.cpp @@ -18,9 +18,7 @@ RoutingSpec getRouting() { .addTable(RoutingTableSpec("Simple") .addHop(HopSpec("pxy", "test/pxy/session")) .addHop(HopSpec("dst", "test/dst/session")) - .addRoute(RouteSpec("test") - .addHop("pxy") - .addHop("dst"))); + .addRoute(std::move(RouteSpec("test").addHop("pxy").addHop("dst")))); } int @@ -47,47 +45,47 @@ Test::Main() ASSERT_TRUE(pxyNet.waitSlobrok("test/dst/session")); // send message on client - ss->send(SimpleMessage::UP(new SimpleMessage("test message")), "test"); + ss->send(std::make_unique<SimpleMessage>("test message"), "test"); // check message on proxy Message::UP msg = pxy.getMessage(); - ASSERT_TRUE(msg.get() != 0); + ASSERT_TRUE(msg); EXPECT_TRUE(msg->getProtocol() == SimpleProtocol::NAME); EXPECT_TRUE(msg->getType() == SimpleProtocol::MESSAGE); - EXPECT_TRUE(static_cast<SimpleMessage&>(*msg).getValue() == "test message"); + EXPECT_TRUE(dynamic_cast<SimpleMessage&>(*msg).getValue() == "test message"); // forward message on proxy - static_cast<SimpleMessage&>(*msg).setValue("test message pxy"); + dynamic_cast<SimpleMessage&>(*msg).setValue("test message pxy"); is->forward(std::move(msg)); // check message on server msg = dst.getMessage(); - ASSERT_TRUE(msg.get() != 0); + ASSERT_TRUE(msg); EXPECT_TRUE(msg->getProtocol() == SimpleProtocol::NAME); EXPECT_TRUE(msg->getType() == SimpleProtocol::MESSAGE); - EXPECT_TRUE(static_cast<SimpleMessage&>(*msg).getValue() == "test message pxy"); + EXPECT_TRUE(dynamic_cast<SimpleMessage&>(*msg).getValue() == "test message pxy"); // send reply on server - SimpleReply::UP sr(new SimpleReply("test reply")); + auto sr = std::make_unique<SimpleReply>("test reply"); msg->swapState(*sr); ds->reply(Reply::UP(sr.release())); // check reply on proxy Reply::UP reply = pxy.getReply(); - ASSERT_TRUE(reply.get() != 0); + ASSERT_TRUE(reply); EXPECT_TRUE(reply->getProtocol() == SimpleProtocol::NAME); EXPECT_TRUE(reply->getType() == SimpleProtocol::REPLY); - EXPECT_TRUE(static_cast<SimpleReply&>(*reply).getValue() == "test reply"); + EXPECT_TRUE(dynamic_cast<SimpleReply&>(*reply).getValue() == "test reply"); // forward reply on proxy - static_cast<SimpleReply&>(*reply).setValue("test reply pxy"); + dynamic_cast<SimpleReply&>(*reply).setValue("test reply pxy"); is->forward(std::move(reply)); // check reply on client reply = src.getReply(); - ASSERT_TRUE(reply.get() != 0); + ASSERT_TRUE(reply); EXPECT_TRUE(reply->getProtocol() == SimpleProtocol::NAME); EXPECT_TRUE(reply->getType() == SimpleProtocol::REPLY); - EXPECT_TRUE(static_cast<SimpleReply&>(*reply).getValue() == "test reply pxy"); + EXPECT_TRUE(dynamic_cast<SimpleReply&>(*reply).getValue() == "test reply pxy"); TEST_DONE(); } diff --git a/messagebus/src/tests/sourcesession/sourcesession.cpp b/messagebus/src/tests/sourcesession/sourcesession.cpp index 6ebbc68f965..a91c128b0b0 100644 --- a/messagebus/src/tests/sourcesession/sourcesession.cpp +++ b/messagebus/src/tests/sourcesession/sourcesession.cpp @@ -13,7 +13,6 @@ #include <vespa/messagebus/routing/routingspec.h> #include <vespa/messagebus/testlib/simplemessage.h> #include <vespa/messagebus/testlib/simpleprotocol.h> -#include <vespa/messagebus/testlib/simplereply.h> #include <vespa/messagebus/testlib/slobrok.h> #include <vespa/messagebus/testlib/testserver.h> #include <vespa/vespalib/testkit/testapp.h> @@ -29,7 +28,7 @@ struct DelayedHandler : public IMessageHandler DelayedHandler(MessageBus &mb, uint32_t d) : session(), delay(d) { session = mb.createDestinationSession("session", true, *this); } - ~DelayedHandler() { + ~DelayedHandler() override { session.reset(); } void handleMessage(Message::UP msg) override { @@ -45,14 +44,14 @@ RoutingSpec getRouting() { return RoutingSpec() .addTable(RoutingTableSpec("Simple") .addHop(HopSpec("dst", "dst/session")) - .addRoute(RouteSpec("dst").addHop("dst"))); + .addRoute(std::move(RouteSpec("dst").addHop("dst")))); } RoutingSpec getBadRouting() { return RoutingSpec() .addTable(RoutingTableSpec("Simple") .addHop(HopSpec("dst", "dst/session")) - .addRoute(RouteSpec("dst").addHop("dst"))); + .addRoute(std::move(RouteSpec("dst").addHop("dst")))); } bool waitQueueSize(RoutableQueue &queue, uint32_t size) { @@ -183,8 +182,8 @@ Test::testResendConnDown() RPCNetworkParams(slobrok.config())); src.mb.setupRouting(RoutingSpec().addTable(RoutingTableSpec(SimpleProtocol::NAME) .addHop(HopSpec("dst", "dst2/session")) - .addHop(HopSpec("pxy", "[All]").addRecipient("dst")) - .addRoute(RouteSpec("dst").addHop("pxy")))); + .addHop(std::move(HopSpec("pxy", "[All]").addRecipient("dst"))) + .addRoute(std::move(RouteSpec("dst").addHop("pxy"))))); RoutableQueue srcQ; SourceSession::UP ss = src.mb.createSourceSession(srcQ); @@ -243,7 +242,7 @@ Test::testIllegalRoute() while (srcQ.size() > 0) { Routable::UP routable = srcQ.dequeue(); ASSERT_TRUE(routable->isReply()); - Reply::UP r(static_cast<Reply*>(routable.release())); + Reply::UP r(dynamic_cast<Reply*>(routable.release())); EXPECT_EQUAL(1u, r->getNumErrors()); EXPECT_EQUAL((uint32_t)ErrorCode::NO_ADDRESS_FOR_SERVICE, r->getError(0).getCode()); string trace = r->getTrace().toString(); @@ -275,7 +274,7 @@ Test::testNoServices() while (srcQ.size() > 0) { Routable::UP routable = srcQ.dequeue(); ASSERT_TRUE(routable->isReply()); - Reply::UP r(static_cast<Reply*>(routable.release())); + Reply::UP r(dynamic_cast<Reply*>(routable.release())); EXPECT_TRUE(r->getNumErrors() == 1); EXPECT_TRUE(r->getError(0).getCode() == ErrorCode::NO_ADDRESS_FOR_SERVICE); string trace = r->getTrace().toString(); diff --git a/messagebus/src/tests/throttling/throttling.cpp b/messagebus/src/tests/throttling/throttling.cpp index a601845a5fc..3063e3ae8b5 100644 --- a/messagebus/src/tests/throttling/throttling.cpp +++ b/messagebus/src/tests/throttling/throttling.cpp @@ -4,7 +4,6 @@ #include <vespa/messagebus/destinationsession.h> #include <vespa/messagebus/dynamicthrottlepolicy.h> #include <vespa/messagebus/routablequeue.h> -#include <vespa/messagebus/routing/retrytransienterrorspolicy.h> #include <vespa/messagebus/routing/routingspec.h> #include <vespa/messagebus/sourcesession.h> #include <vespa/messagebus/sourcesessionparams.h> @@ -41,7 +40,7 @@ RoutingSpec getRouting() return RoutingSpec() .addTable(RoutingTableSpec("Simple") .addHop(HopSpec("dst", "dst/session")) - .addRoute(RouteSpec("dst").addHop("dst"))); + .addRoute(std::move(RouteSpec("dst").addHop("dst")))); } bool waitQueueSize(RoutableQueue &queue, uint32_t size) diff --git a/messagebus/src/tests/trace-roundtrip/trace-roundtrip.cpp b/messagebus/src/tests/trace-roundtrip/trace-roundtrip.cpp index 17d4071d411..7fbc3b096d6 100644 --- a/messagebus/src/tests/trace-roundtrip/trace-roundtrip.cpp +++ b/messagebus/src/tests/trace-roundtrip/trace-roundtrip.cpp @@ -12,8 +12,6 @@ #include <vespa/messagebus/testlib/receptor.h> #include <vespa/messagebus/sourcesessionparams.h> #include <vespa/messagebus/testlib/simplemessage.h> -#include <vespa/messagebus/testlib/simplereply.h> -#include <vespa/messagebus/testlib/simpleprotocol.h> using namespace mbus; @@ -81,7 +79,7 @@ RoutingSpec getRouting() { .addTable(RoutingTableSpec("Simple") .addHop(HopSpec("pxy", "test/pxy/session")) .addHop(HopSpec("dst", "test/dst/session")) - .addRoute(RouteSpec("test").addHop("pxy").addHop("dst"))); + .addRoute(std::move(RouteSpec("test").addHop("pxy").addHop("dst")))); } int diff --git a/messagebus/src/vespa/messagebus/configagent.cpp b/messagebus/src/vespa/messagebus/configagent.cpp index 0184f0456cf..60c6690f7e3 100644 --- a/messagebus/src/vespa/messagebus/configagent.cpp +++ b/messagebus/src/vespa/messagebus/configagent.cpp @@ -1,8 +1,8 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/messagebus/routing/routingspec.h> #include "configagent.h" #include "iconfighandler.h" +#include <vespa/messagebus/routing/routingspec.h> using namespace config; using namespace messagebus; @@ -18,26 +18,22 @@ ConfigAgent::configure(std::unique_ptr<MessagebusConfig> config) { const MessagebusConfig &cfg(*config); RoutingSpec spec; - using CFG = MessagebusConfig; - for (uint32_t t = 0; t < cfg.routingtable.size(); ++t) { - const CFG::Routingtable &table = cfg.routingtable[t]; + for (const auto & table : cfg.routingtable) { RoutingTableSpec tableSpec(table.protocol); - for (uint32_t h = 0; h < table.hop.size(); ++h) { - const CFG::Routingtable::Hop &hop = table.hop[h]; + for (const auto & hop : table.hop) { HopSpec hopSpec(hop.name, hop.selector); - for (uint32_t i = 0; i < hop.recipient.size(); ++i) { - hopSpec.addRecipient(hop.recipient[i]); + for (const auto & i : hop.recipient) { + hopSpec.addRecipient(i); } hopSpec.setIgnoreResult(hop.ignoreresult); - tableSpec.addHop(hopSpec); + tableSpec.addHop(std::move(hopSpec)); } - for (uint32_t r = 0; r < table.route.size(); ++r) { - const CFG::Routingtable::Route &route = table.route[r]; + for (const auto & route : table.route) { RouteSpec routeSpec(route.name); - for (uint32_t i = 0; i < route.hop.size(); ++i) { - routeSpec.addHop(route.hop[i]); + for (const auto & i : route.hop) { + routeSpec.addHop(i); } - tableSpec.addRoute(routeSpec); + tableSpec.addRoute(std::move(routeSpec)); } spec.addTable(tableSpec); } diff --git a/messagebus/src/vespa/messagebus/iconfighandler.h b/messagebus/src/vespa/messagebus/iconfighandler.h index 91398629ca8..5cfd26b6440 100644 --- a/messagebus/src/vespa/messagebus/iconfighandler.h +++ b/messagebus/src/vespa/messagebus/iconfighandler.h @@ -15,7 +15,7 @@ class RoutingSpec; class IConfigHandler { public: - virtual ~IConfigHandler() {} + virtual ~IConfigHandler() = default; /** * This method will be invoked to initialize or change the routing diff --git a/messagebus/src/vespa/messagebus/network/rpcsendv2.cpp b/messagebus/src/vespa/messagebus/network/rpcsendv2.cpp index c211ff83110..18e29451a0d 100644 --- a/messagebus/src/vespa/messagebus/network/rpcsendv2.cpp +++ b/messagebus/src/vespa/messagebus/network/rpcsendv2.cpp @@ -88,11 +88,12 @@ namespace { class OutputBuf : public vespalib::Output { public: explicit OutputBuf(size_t estimatedSize) : _buf(estimatedSize) { } + ~OutputBuf() override; DataBuffer & getBuf() { return _buf; } private: vespalib::WritableMemory reserve(size_t bytes) override { _buf.ensureFree(bytes); - return vespalib::WritableMemory(_buf.getFree(), _buf.getFreeLen()); + return {_buf.getFree(), _buf.getFreeLen()}; } Output &commit(size_t bytes) override { _buf.moveFreeToData(bytes); @@ -100,6 +101,8 @@ private: } DataBuffer _buf; }; +OutputBuf::~OutputBuf() = default; + } void diff --git a/messagebus/src/vespa/messagebus/routing/hopspec.cpp b/messagebus/src/vespa/messagebus/routing/hopspec.cpp index 0ea996982ed..7c8980b33b1 100644 --- a/messagebus/src/vespa/messagebus/routing/hopspec.cpp +++ b/messagebus/src/vespa/messagebus/routing/hopspec.cpp @@ -14,27 +14,15 @@ HopSpec::HopSpec(const string &name, const string &selector) : _ignoreResult(false) { } -HopSpec::~HopSpec() {} +HopSpec::HopSpec(const HopSpec & rhs) = default; +HopSpec & HopSpec::operator=(const HopSpec & rhs) = default; +HopSpec::HopSpec(HopSpec && rhs) noexcept = default; +HopSpec & HopSpec::operator=(HopSpec && rhs) noexcept = default; +HopSpec::~HopSpec() = default; HopSpec & -HopSpec::addRecipients(const std::vector<string> &recipients) -{ - _recipients.insert(_recipients.end(), recipients.begin(), recipients.end()); - return *this; -} - -string -HopSpec::removeRecipient(uint32_t i) -{ - string ret = _recipients[i]; - _recipients.erase(_recipients.begin() + i); - return ret; -} - -HopSpec & -HopSpec::setIgnoreResult(bool ignoreResult) -{ - _ignoreResult = ignoreResult; +HopSpec::addRecipient(const string &recipient) { + _recipients.push_back(recipient); return *this; } diff --git a/messagebus/src/vespa/messagebus/routing/hopspec.h b/messagebus/src/vespa/messagebus/routing/hopspec.h index 2479537d7cc..fd2e4dfb431 100644 --- a/messagebus/src/vespa/messagebus/routing/hopspec.h +++ b/messagebus/src/vespa/messagebus/routing/hopspec.h @@ -31,6 +31,10 @@ public: * @param selector A string that represents the selector for this hop. */ HopSpec(const string &name, const string &selector); + HopSpec(const HopSpec & rhs); + HopSpec & operator=(const HopSpec & rhs); + HopSpec(HopSpec && rhs) noexcept; + HopSpec & operator=(HopSpec && rhs) noexcept; ~HopSpec(); /** @@ -38,28 +42,21 @@ public: * * @return The name. */ - const string &getName() const { return _name; } + [[nodiscard]] const string &getName() const { return _name; } /** * Returns the string selector that resolves the recipients of this hop. * * @return The selector. */ - const string &getSelector() const { return _selector; } - - /** - * Returns whether or not there are any recipients that the selector can choose from. - * - * @return True if there is at least one recipient. - */ - bool hasRecipients() const { return !_recipients.empty(); } + [[nodiscard]] const string &getSelector() const { return _selector; } /** * Returns the number of recipients that the selector can choose from. * * @return The number of recipients. */ - uint32_t getNumRecipients() const { return _recipients.size(); } + [[nodiscard]] uint32_t getNumRecipients() const { return _recipients.size(); } /** * Returns the recipients at the given index. @@ -67,7 +64,7 @@ public: * @param i The index of the recipient to return. * @return The recipient at the given index. */ - const string &getRecipient(uint32_t i) const { return _recipients[i]; } + [[nodiscard]] const string &getRecipient(uint32_t i) const { return _recipients[i]; } /** * Adds the given recipient to this. @@ -75,46 +72,14 @@ public: * @param recipient The recipient to add. * @return This, to allow chaining. */ - HopSpec &addRecipient(const string &recipient) { _recipients.push_back(recipient); return *this; } - - /** - * Adds the given recipients to this. - * - * @param recipients The recipients to add. - * @return This, to allow chaining. - */ - HopSpec &addRecipients(const std::vector<string> &recipients); - - /** - * Sets the recipient at the given index. - * - * @param i The index at which to set the recipient. - * @param recipient The recipient to set. - * @return This, to allow chaining. - */ - HopSpec &setRecipient(uint32_t i, const string &recipient) { _recipients[i] = recipient; return *this; } - - /** - * Removes the recipient at the given index. - * - * @param i The index of the recipient to remove. - * @return The removed recipient. - */ - string removeRecipient(uint32_t i); - - /** - * Clears the list of recipients that the selector may choose from. - * - * @return This, to allow chaining. - */ - HopSpec &clearRecipients() { _recipients.clear(); return *this; } + HopSpec &addRecipient(const string &recipient); /** * Returns whether or not to ignore the result when routing through this hop. * * @return True to ignore the result. */ - bool getIgnoreResult() const { return _ignoreResult; } + [[nodiscard]] bool getIgnoreResult() const { return _ignoreResult; } /** * Sets whether or not to ignore the result when routing through this hop. @@ -122,7 +87,10 @@ public: * @param ignoreResult Whether or not to ignore the result. * @return This, to allow chaining. */ - HopSpec &setIgnoreResult(bool ignoreResult); + HopSpec &setIgnoreResult(bool ignoreResult) { + _ignoreResult = ignoreResult; + return *this; + } /** * Appends the content of this to the given config string. @@ -137,7 +105,7 @@ public: * * @return The string. */ - string toString() const; + [[nodiscard]] string toString() const; /** * Implements the equality operator. diff --git a/messagebus/src/vespa/messagebus/routing/route.cpp b/messagebus/src/vespa/messagebus/routing/route.cpp index b393c2ccd6c..6be6788045c 100644 --- a/messagebus/src/vespa/messagebus/routing/route.cpp +++ b/messagebus/src/vespa/messagebus/routing/route.cpp @@ -4,15 +4,13 @@ namespace mbus { -Route::Route() : - _hops() -{ } +Route::Route() = default; Route::Route(std::vector<Hop> lst) : _hops(std::move(lst)) { } -Route::~Route() { } +Route::~Route() = default; Route & Route::addHop(Hop hop) @@ -36,13 +34,6 @@ Route::removeHop(uint32_t i) return ret; } -Route & -Route::clearHops() -{ - _hops.clear(); - return *this; -} - string Route::toString() const { string ret = ""; diff --git a/messagebus/src/vespa/messagebus/routing/route.h b/messagebus/src/vespa/messagebus/routing/route.h index d5839c128d1..43b76de4aeb 100644 --- a/messagebus/src/vespa/messagebus/routing/route.h +++ b/messagebus/src/vespa/messagebus/routing/route.h @@ -49,21 +49,21 @@ public: * * @param hops The hops to instantiate with. */ - Route(std::vector<Hop> hops); + explicit Route(std::vector<Hop> hops); /** * Returns whether or not there are any hops in this route. * * @return True if there is at least one hop. */ - bool hasHops() const { return !_hops.empty(); } + [[nodiscard]] bool hasHops() const { return !_hops.empty(); } /** * Returns the number of hops that make up this route. * * @return The number of hops. */ - uint32_t getNumHops() const { return _hops.size(); } + [[nodiscard]] uint32_t getNumHops() const { return _hops.size(); } /** * Returns the hop at the given index. @@ -79,7 +79,7 @@ public: * @param i The index of the hop to return. * @return The hop. */ - const Hop &getHop(uint32_t i) const { return _hops[i]; } + [[nodiscard]] const Hop &getHop(uint32_t i) const { return _hops[i]; } /** * Adds a hop to the list of hops that make up this route. @@ -107,25 +107,18 @@ public: Hop removeHop(uint32_t i); /** - * Clears the list of hops that make up this route. - * - * @return This, to allow chaining. - */ - Route &clearHops(); - - /** * Returns a string representation of this route. * * @return A string representation. */ - string toString() const; + [[nodiscard]] string toString() const; /** * Returns a string representation of this that can be debugged but not parsed. * * @return The debug string. */ - string toDebugString() const; + [[nodiscard]] string toDebugString() const; }; } // namespace mbus diff --git a/messagebus/src/vespa/messagebus/routing/routedirective.h b/messagebus/src/vespa/messagebus/routing/routedirective.h index 26cbebb6d6f..9fdbe0bbf6b 100644 --- a/messagebus/src/vespa/messagebus/routing/routedirective.h +++ b/messagebus/src/vespa/messagebus/routing/routedirective.h @@ -23,7 +23,7 @@ public: * * @param name The name of the route to insert. */ - RouteDirective(vespalib::stringref name); + explicit RouteDirective(vespalib::stringref name); ~RouteDirective() override; /** @@ -31,12 +31,12 @@ public: * * @return The name name. */ - const string &getName() const { return _name; } + [[nodiscard]] const string &getName() const { return _name; } - Type getType() const override { return TYPE_ROUTE; } - bool matches(const IHopDirective &dir) const override; - string toString() const override; - string toDebugString() const override; + [[nodiscard]] Type getType() const override { return TYPE_ROUTE; } + [[nodiscard]] bool matches(const IHopDirective &dir) const override; + [[nodiscard]] string toString() const override; + [[nodiscard]] string toDebugString() const override; }; } // mbus diff --git a/messagebus/src/vespa/messagebus/routing/routespec.cpp b/messagebus/src/vespa/messagebus/routing/routespec.cpp index cc1f1a9301d..00b8d7ed322 100644 --- a/messagebus/src/vespa/messagebus/routing/routespec.cpp +++ b/messagebus/src/vespa/messagebus/routing/routespec.cpp @@ -7,29 +7,29 @@ using vespalib::make_string; namespace mbus { -RouteSpec::RouteSpec(const string &name) : +RouteSpec::RouteSpec(const string &name) noexcept : _name(name), _hops() { } RouteSpec::RouteSpec(const RouteSpec &) = default; RouteSpec & RouteSpec::operator = (const RouteSpec &) = default; +RouteSpec::RouteSpec(RouteSpec &&) noexcept = default; +RouteSpec & RouteSpec::operator = (RouteSpec &&) noexcept = default; -RouteSpec::~RouteSpec() {} +RouteSpec::~RouteSpec() = default; RouteSpec & -RouteSpec::addHops(const std::vector<string> &hops) -{ - _hops.insert(_hops.end(), hops.begin(), hops.end()); +RouteSpec::addHop(const string &hop) { + _hops.push_back(hop); return *this; } -string -RouteSpec::removeHop(uint32_t i) -{ - string ret = _hops[i]; - _hops.erase(_hops.begin() + i); - return ret; + +RouteSpec & +RouteSpec::setHop(uint32_t i, const string &hop) { + _hops[i] = hop; + return *this; } void diff --git a/messagebus/src/vespa/messagebus/routing/routespec.h b/messagebus/src/vespa/messagebus/routing/routespec.h index 61853571b9e..273836c94d5 100644 --- a/messagebus/src/vespa/messagebus/routing/routespec.h +++ b/messagebus/src/vespa/messagebus/routing/routespec.h @@ -27,12 +27,11 @@ public: * * @param name A protocol-unique name for this route. */ - RouteSpec(const string &name); + explicit RouteSpec(const string &name) noexcept; RouteSpec(const RouteSpec &); RouteSpec & operator = (const RouteSpec &); - RouteSpec(RouteSpec &&) = default; - RouteSpec & operator = (RouteSpec &&) = default; - + RouteSpec(RouteSpec &&) noexcept; + RouteSpec & operator = (RouteSpec &&) noexcept; ~RouteSpec(); /** @@ -40,7 +39,7 @@ public: * * @return The name. */ - const string &getName() const { return _name; } + [[nodiscard]] const string &getName() const { return _name; } /** * Returns the hop name at the given index. @@ -48,21 +47,14 @@ public: * @param i The index of the hop to return. * @return The hop at the given index. */ - const string &getHop(uint32_t i) const { return _hops[i]; } - - /** - * Returns whether or not there are any hops in this route. - * - * @return True if there is at least one hop. - */ - bool hasHops() const { return !_hops.empty(); } + [[nodiscard]] const string &getHop(uint32_t i) const { return _hops[i]; } /** * Returns the number of hops that make up this route. * * @return The number of hops. */ - uint32_t getNumHops() const { return _hops.size(); } + [[nodiscard]] uint32_t getNumHops() const { return _hops.size(); } /** * Adds the given hop name to this. @@ -70,15 +62,7 @@ public: * @param hop The hop to add. * @return This, to allow chaining. */ - RouteSpec &addHop(const string &hop) { _hops.push_back(hop); return *this; } - - /** - * Adds the given hop names to this. - * - * @param hops The hops to add. - * @return This, to allow chaining. - */ - RouteSpec &addHops(const std::vector<string> &hops); + RouteSpec & addHop(const string &hop); /** * Sets the hop name for a given index. @@ -87,22 +71,7 @@ public: * @param hop The hop to set. * @return This, to allow chaining. */ - RouteSpec &setHop(uint32_t i, const string &hop) { _hops[i] = hop; return *this; } - - /** - * Removes the hop name at the given index. - * - * @param i The index of the hop to remove. - * @return The removed hop. - */ - string removeHop(uint32_t i); - - /** - * Clears the list of hops that make up this route. - * - * @return This, to allow chaining. - */ - RouteSpec &clearHops() { _hops.clear(); return *this; } + RouteSpec &setHop(uint32_t i, const string &hop); /** * Appends the content of this to the given config string. @@ -117,7 +86,7 @@ public: * * @return The string. */ - string toString() const; + [[nodiscard]] string toString() const; /** * Implements the equality operator. diff --git a/messagebus/src/vespa/messagebus/routing/routingcontext.cpp b/messagebus/src/vespa/messagebus/routing/routingcontext.cpp index 085820d5c3c..03451c3de56 100644 --- a/messagebus/src/vespa/messagebus/routing/routingcontext.cpp +++ b/messagebus/src/vespa/messagebus/routing/routingcontext.cpp @@ -8,12 +8,14 @@ namespace mbus { RoutingContext::RoutingContext(RoutingNode &node, uint32_t directive) : _node(node), + _context(), _directive(directive), - _consumableErrors(), _selectOnRetry(true), - _context() + _consumableErrors() { } +RoutingContext::~RoutingContext() = default; + bool RoutingContext::hasRecipients() const { @@ -60,19 +62,6 @@ RoutingContext::getMatchedRecipients(std::vector<Route> &ret) const } } -bool -RoutingContext::getSelectOnRetry() const -{ - return _selectOnRetry; -} - -RoutingContext & -RoutingContext::setSelectOnRetry(bool selectOnRetry) -{ - _selectOnRetry = selectOnRetry; - return *this; -} - const Route & RoutingContext::getRoute() const { @@ -85,12 +74,6 @@ RoutingContext::getHop() const return _node.getRoute().getHop(0); } -uint32_t -RoutingContext::getDirectiveIndex() const -{ - return _directive; -} - const PolicyDirective & RoutingContext::getDirective() const { @@ -109,25 +92,6 @@ RoutingContext::getHopSuffix() const return getHop().getSuffix(_directive); } -Context & -RoutingContext::getContext() -{ - return _context; -} - -const Context & -RoutingContext::getContext() const -{ - return _context; -} - -RoutingContext & -RoutingContext::setContext(const Context &ctx) -{ - _context = ctx; - return *this; -} - const Message & RoutingContext::getMessage() const { @@ -140,12 +104,6 @@ RoutingContext::trace(uint32_t level, const string ¬e) _node.getTrace().trace(level, note); } -bool -RoutingContext::hasReply() const -{ - return _node.hasReply(); -} - const Reply & RoutingContext::getReply() const { diff --git a/messagebus/src/vespa/messagebus/routing/routingcontext.h b/messagebus/src/vespa/messagebus/routing/routingcontext.h index d1323c35f22..3f85b8161ed 100644 --- a/messagebus/src/vespa/messagebus/routing/routingcontext.h +++ b/messagebus/src/vespa/messagebus/routing/routingcontext.h @@ -22,38 +22,38 @@ class Error; class RoutingContext { private: RoutingNode &_node; + Context _context; uint32_t _directive; - std::set<uint32_t> _consumableErrors; bool _selectOnRetry; - Context _context; + std::set<uint32_t> _consumableErrors; public: /** - * Convenience typedefs. - */ - using UP = std::unique_ptr<RoutingContext>; - - /** * Constructs a new routing context for a given routing node and hop. * * @param node The owning routing node. * @param directive The index to the policy directive of the hop. */ RoutingContext(RoutingNode &node, uint32_t directive); + RoutingContext(const RoutingContext &) = delete; + RoutingContext(RoutingContext &&) = delete; + RoutingContext & operator=(const RoutingContext &) = delete; + RoutingContext & operator=(RoutingContext &&) = delete; + ~RoutingContext(); /** * Returns whether or not this hop has any configured recipients. * * @return True if there is at least one recipient. */ - bool hasRecipients() const; + [[nodiscard]] bool hasRecipients() const; /** * Returns the number of configured recipients for this hop. * * @return The recipient count. */ - uint32_t getNumRecipients() const; + [[nodiscard]] uint32_t getNumRecipients() const; /** * Returns the configured recipient at the given index. @@ -61,14 +61,14 @@ public: * @param idx The index of the recipient to return. * @return The reipient at the given index. */ - const Route &getRecipient(uint32_t idx) const; + [[nodiscard]] const Route &getRecipient(uint32_t idx) const; /** * Returns all configured recipients for this hop. * * @return An unmodifiable list of recipients. */ - const std::vector<Route> &getAllRecipients() const; + [[nodiscard]] const std::vector<Route> &getAllRecipients() const; /** * Returns a list of all configured recipients whose first hop matches this. @@ -82,7 +82,7 @@ public: * * @return True to invoke {@link RoutingPolicy#select(RoutingContext)} on resend. */ - bool getSelectOnRetry() const; + [[nodiscard]] bool getSelectOnRetry() const { return _selectOnRetry; } /** * Sets whether or not the policy is required to reselect if resending occurs. @@ -90,35 +90,38 @@ public: * @param selectOnRetry The value to set. * @return This, to allow chaining. */ - RoutingContext &setSelectOnRetry(bool selectOnRetry); + RoutingContext &setSelectOnRetry(bool selectOnRetry) { + _selectOnRetry = selectOnRetry; + return *this; + } /** * Returns the route that contains the routing policy that spawned this. * * @return The route. */ - const Route &getRoute() const; + [[nodiscard]] const Route &getRoute() const; /** * Returns the hop that contains the routing policy that spawned this. * * @return The hop. */ - const Hop &getHop() const; + [[nodiscard]] const Hop &getHop() const; /** * Returns the index of the hop directive that spawned this. * * @return The directive index. */ - uint32_t getDirectiveIndex() const; + [[nodiscard]] uint32_t getDirectiveIndex() const { return _directive; } /** * Returns the policy directive that spawned this. * * @return The directive object. */ - const PolicyDirective &getDirective() const; + [[nodiscard]] const PolicyDirective &getDirective() const; /** * Returns the part of the route string that precedes the active policy directive. This is the same as calling @@ -126,7 +129,7 @@ public: * * @return The hop prefix. */ - string getHopPrefix() const; + [[nodiscard]] string getHopPrefix() const; /** * Returns the remainder of the route string immediately following the active policy directive. This is the same as @@ -134,21 +137,21 @@ public: * * @return The hop suffix. */ - string getHopSuffix() const; + [[nodiscard]] string getHopSuffix() const; /** * Returns the routing specific context object. * * @return The context. */ - Context &getContext(); + Context &getContext() { return _context; } /** * Returns a const reference to the routing specific context object. * * @return The context. */ - const Context &getContext() const; + [[nodiscard]] const Context &getContext() const { return _context; } /** * Sets a routing specific context object that will be available at merge(). @@ -156,14 +159,17 @@ public: * @param context An arbitrary object. * @return This, to allow chaining. */ - RoutingContext &setContext(const Context &ctx); + RoutingContext &setContext(const Context &ctx) { + _context = ctx; + return *this; + } /** * Returns the message being routed. * * @return The message. */ - const Message &getMessage() const; + [[nodiscard]] const Message &getMessage() const; /** * Adds a string to the trace of the message being routed. @@ -174,18 +180,11 @@ public: void trace(uint32_t level, const string ¬e); /** - * Returns whether or not a reply is available. - * - * @return True if a reply is set. - */ - bool hasReply() const; - - /** * Returns the reply generated by the associated routing policy. * * @return The reply. */ - const Reply &getReply() const; + [[nodiscard]] const Reply &getReply() const; /** * Sets the reply generated by the associated routing policy. @@ -226,14 +225,14 @@ public: * * @return True if there is at least one child. */ - bool hasChildren() const; + [[nodiscard]] bool hasChildren() const; /** * Returns the number of children the owning routing node has. * * @return The child count. */ - uint32_t getNumChildren() const; + [[nodiscard]] uint32_t getNumChildren() const; /** * Returns an iterator for the child routing nodes. @@ -263,7 +262,7 @@ public: * * @return The mirror api. */ - const slobrok::api::IMirrorAPI &getMirror() const; + [[nodiscard]] const slobrok::api::IMirrorAPI &getMirror() const; /** * Adds the given error code to the list of codes that the associated routing policy <u>may</u> consume. This is diff --git a/messagebus/src/vespa/messagebus/routing/routingnode.h b/messagebus/src/vespa/messagebus/routing/routingnode.h index d6d4da64ae4..02f2a9f7012 100644 --- a/messagebus/src/vespa/messagebus/routing/routingnode.h +++ b/messagebus/src/vespa/messagebus/routing/routingnode.h @@ -43,10 +43,10 @@ private: Reply::UP _reply; Route _route; IRoutingPolicy::SP _policy; - RoutingContext::UP _routingContext; - IServiceAddress::UP _serviceAddress; - bool _isActive; - bool _shouldRetry; + std::unique_ptr<RoutingContext> _routingContext; + IServiceAddress::UP _serviceAddress; + bool _isActive; + bool _shouldRetry; /** * Constructs a new instance of this class. This is the child node @@ -233,7 +233,7 @@ public: * Destructor. Frees up any allocated resources, namely all child nodes of * this. */ - ~RoutingNode(); + ~RoutingNode() override; /** * Discards this routing node. Invoking this will notify the parent {@link diff --git a/messagebus/src/vespa/messagebus/routing/routingnodeiterator.cpp b/messagebus/src/vespa/messagebus/routing/routingnodeiterator.cpp index 6d78f851f22..83ca8d37ae4 100644 --- a/messagebus/src/vespa/messagebus/routing/routingnodeiterator.cpp +++ b/messagebus/src/vespa/messagebus/routing/routingnodeiterator.cpp @@ -8,12 +8,6 @@ RoutingNodeIterator::RoutingNodeIterator(std::vector<RoutingNode*> &children) : _end(children.end()) { } -bool -RoutingNodeIterator::isValid() -{ - return _pos != _end; -} - RoutingNodeIterator & RoutingNodeIterator::next() { @@ -36,12 +30,6 @@ RoutingNodeIterator::getRoute() const return (*_pos)->getRoute(); } -bool -RoutingNodeIterator::hasReply() const -{ - return (*_pos)->hasReply(); -} - Reply::UP RoutingNodeIterator::removeReply() { diff --git a/messagebus/src/vespa/messagebus/routing/routingnodeiterator.h b/messagebus/src/vespa/messagebus/routing/routingnodeiterator.h index 25c1de777d1..a1612fdf0bf 100644 --- a/messagebus/src/vespa/messagebus/routing/routingnodeiterator.h +++ b/messagebus/src/vespa/messagebus/routing/routingnodeiterator.h @@ -22,14 +22,14 @@ public: * * @param children The list to iterate through. */ - RoutingNodeIterator(std::vector<RoutingNode*> &children); + explicit RoutingNodeIterator(std::vector<RoutingNode*> &children); /** * Returns whether or not this iterator is valid. * * @return True if we are still pointing to a valid entry. */ - bool isValid(); + [[nodiscard]] bool isValid() const { return _pos != _end; } /** * Steps to the next child in the map. @@ -51,14 +51,7 @@ public: * * @return The route. */ - const Route &getRoute() const; - - /** - * Returns whether or not a reply is set in the current child. - * - * @return True if a reply is available. - */ - bool hasReply() const; + [[nodiscard]] const Route &getRoute() const; /** * Removes and returns the reply of the current child. This is the correct way of reusing a reply of a @@ -73,7 +66,7 @@ public: * * @return The reply. */ - const Reply &getReplyRef() const; + [[nodiscard]] const Reply &getReplyRef() const; }; } // mbus diff --git a/messagebus/src/vespa/messagebus/routing/routingtablespec.cpp b/messagebus/src/vespa/messagebus/routing/routingtablespec.cpp index 328565a3ef7..6b20bd4af9f 100644 --- a/messagebus/src/vespa/messagebus/routing/routingtablespec.cpp +++ b/messagebus/src/vespa/messagebus/routing/routingtablespec.cpp @@ -14,24 +14,20 @@ RoutingTableSpec::RoutingTableSpec(const string &protocol) : RoutingTableSpec::RoutingTableSpec(const RoutingTableSpec&) = default; -RoutingTableSpec::~RoutingTableSpec() {} +RoutingTableSpec::~RoutingTableSpec() = default; RoutingTableSpec& RoutingTableSpec::operator=(const RoutingTableSpec&) = default; -HopSpec -RoutingTableSpec::removeHop(uint32_t i) -{ - HopSpec ret = _hops[i]; - _hops.erase(_hops.begin() + i); - return ret; +RoutingTableSpec & +RoutingTableSpec::addHop(HopSpec && hop) { + _hops.emplace_back(std::move(hop)); + return *this; } -RouteSpec -RoutingTableSpec::removeRoute(uint32_t i) -{ - RouteSpec ret = _routes[i]; - _routes.erase(_routes.begin() + i); - return ret; +RoutingTableSpec & +RoutingTableSpec::setHop(uint32_t i, HopSpec &&hop) { + _hops[i] = std::move(hop); + return *this; } void diff --git a/messagebus/src/vespa/messagebus/routing/routingtablespec.h b/messagebus/src/vespa/messagebus/routing/routingtablespec.h index eb0078d1767..63f1a68ade2 100644 --- a/messagebus/src/vespa/messagebus/routing/routingtablespec.h +++ b/messagebus/src/vespa/messagebus/routing/routingtablespec.h @@ -20,7 +20,7 @@ namespace mbus { */ class RoutingTableSpec { private: - string _protocol; + string _protocol; std::vector<HopSpec> _hops; std::vector<RouteSpec> _routes; @@ -30,7 +30,7 @@ public: * * @param protocol The name of the protocol that this belongs to. */ - RoutingTableSpec(const string &protocol); + explicit RoutingTableSpec(const string &protocol); RoutingTableSpec(const RoutingTableSpec&); RoutingTableSpec(RoutingTableSpec&&) noexcept = default; ~RoutingTableSpec(); @@ -42,21 +42,14 @@ public: * * @return The protocol name. */ - const string &getProtocol() const { return _protocol; } - - /** - * Returns whether or not there are any hop specs contained in this. - * - * @return True if there is at least one hop. - */ - bool hasHops() const { return !_hops.empty(); } + [[nodiscard]] const string &getProtocol() const { return _protocol; } /** * Returns the number of hops that are contained in this table. * * @return The number of hops. */ - uint32_t getNumHops() const { return _hops.size(); } + [[nodiscard]] uint32_t getNumHops() const { return _hops.size(); } /** * Returns the hop spec at the given index. @@ -72,7 +65,7 @@ public: * @param i The index of the hop to return. * @return The hop at the given position. */ - const HopSpec &getHop(uint32_t i) const { return _hops[i]; } + [[nodiscard]] const HopSpec &getHop(uint32_t i) const { return _hops[i]; } /** * Adds the given hop spec to this. @@ -80,7 +73,7 @@ public: * @param hop The hop to add. * @return This, to allow chaining. */ - RoutingTableSpec &addHop(const HopSpec &hop) { _hops.push_back(hop); return *this; } + RoutingTableSpec &addHop(HopSpec && hop); /** * Sets the hop spec at the given index. @@ -89,36 +82,14 @@ public: * @param hop The hop to set. * @return This, to allow chaining. */ - RoutingTableSpec &setHop(uint32_t i, const HopSpec &hop) { _hops[i] = hop; return *this; } - - /** - * Removes the hop spec at the given index. - * - * @param i The index of the hop to remove. - * @return The removed hop. - */ - HopSpec removeHop(uint32_t i); - - /** - * Clears the list of hop specs contained in this. - * - * @return This, to allow chaining. - */ - RoutingTableSpec &clearHops() { _hops.clear(); return *this; } - - /** - * Returns whether or not there are any route specs contained in this. - * - * @return True if there is at least one route. - */ - bool hasRoutes() const { return !_routes.empty(); } + RoutingTableSpec &setHop(uint32_t i, HopSpec &&hop); /** * Returns the number of route specs contained in this. * * @return The number of routes. */ - uint32_t getNumRoutes() const { return _routes.size(); } + [[nodiscard]] uint32_t getNumRoutes() const { return _routes.size(); } /** * Returns the route spec at the given index. @@ -134,7 +105,7 @@ public: * @param i The index of the route to return. * @return The route at the given index. */ - const RouteSpec &getRoute(uint32_t i) const { return _routes[i]; } + [[nodiscard]] const RouteSpec &getRoute(uint32_t i) const { return _routes[i]; } /** * Adds a route spec to this. @@ -142,7 +113,7 @@ public: * @param route The route to add. * @return This, to allow chaining. */ - RoutingTableSpec &addRoute(const RouteSpec &route) { _routes.push_back(route); return *this; } + RoutingTableSpec &addRoute(RouteSpec &&route) { _routes.emplace_back(std::move(route)); return *this; } /** * Sets the route spec at the given index. @@ -151,15 +122,7 @@ public: * @param route The route to set. * @return This, to allow chaining. */ - RoutingTableSpec &setRoute(uint32_t i, const RouteSpec &route) { _routes[i] = route; return *this; } - - /** - * Removes a route spec at a given index. - * - * @param i The index of the route to remove. - * @return The removed route. - */ - RouteSpec removeRoute(uint32_t i); + RoutingTableSpec &setRoute(uint32_t i, RouteSpec && route) { _routes[i] = std::move(route); return *this; } /** * Appends the content of this to the given config string. @@ -174,7 +137,7 @@ public: * * @return The string. */ - string toString() const; + [[nodiscard]] string toString() const; /** * Implements the equality operator. diff --git a/messagebus/src/vespa/messagebus/testlib/slobrok.cpp b/messagebus/src/vespa/messagebus/testlib/slobrok.cpp index bcf6f829605..889daf538a3 100644 --- a/messagebus/src/vespa/messagebus/testlib/slobrok.cpp +++ b/messagebus/src/vespa/messagebus/testlib/slobrok.cpp @@ -18,6 +18,7 @@ private: std::condition_variable _cond; public: explicit WaitTask(FNET_Scheduler *s) : FNET_Task(s), _done(false), _mon() {} + ~WaitTask() override; void wait() { std::unique_lock guard(_mon); while (!_done) { @@ -31,6 +32,8 @@ public: _cond.notify_one(); } }; + +WaitTask::~WaitTask() = default; } // namespace <unnamed> namespace mbus { diff --git a/messagebus/src/vespa/messagebus/testlib/testserver.cpp b/messagebus/src/vespa/messagebus/testlib/testserver.cpp index d289c372fda..4393dfccccc 100644 --- a/messagebus/src/vespa/messagebus/testlib/testserver.cpp +++ b/messagebus/src/vespa/messagebus/testlib/testserver.cpp @@ -13,6 +13,8 @@ VersionedRPCNetwork::VersionedRPCNetwork(const RPCNetworkParams ¶ms) : _version(vespalib::Vtag::currentVersion) {} +VersionedRPCNetwork::~VersionedRPCNetwork() = default; + void VersionedRPCNetwork::setVersion(const vespalib::Version &version) { diff --git a/messagebus/src/vespa/messagebus/testlib/testserver.h b/messagebus/src/vespa/messagebus/testlib/testserver.h index 1ea96c3e13f..f3f349e52b9 100644 --- a/messagebus/src/vespa/messagebus/testlib/testserver.h +++ b/messagebus/src/vespa/messagebus/testlib/testserver.h @@ -23,6 +23,7 @@ protected: public: VersionedRPCNetwork(const RPCNetworkParams ¶ms); + ~VersionedRPCNetwork() override; void setVersion(const vespalib::Version &version); }; diff --git a/metrics/src/tests/metricmanagertest.cpp b/metrics/src/tests/metricmanagertest.cpp index 615baeccf43..604e9c46b80 100644 --- a/metrics/src/tests/metricmanagertest.cpp +++ b/metrics/src/tests/metricmanagertest.cpp @@ -898,13 +898,17 @@ struct NestedDimensionTestMetricSet : MetricSet { DimensionTestMetricSet nestedSet; - NestedDimensionTestMetricSet() - : MetricSet("outer", {{"fancy", "stuff"}}, ""), - nestedSet(this) - { - } + NestedDimensionTestMetricSet(); + ~NestedDimensionTestMetricSet(); }; +NestedDimensionTestMetricSet::NestedDimensionTestMetricSet() + : MetricSet("outer", {{"fancy", "stuff"}}, ""), + nestedSet(this) +{ +} +NestedDimensionTestMetricSet::~NestedDimensionTestMetricSet() = default; + } TEST_F(MetricManagerTest, json_output_can_nest_dimensions_from_multiple_metric_sets) @@ -933,13 +937,16 @@ struct DimensionOverridableTestMetricSet : MetricSet { DoubleValueMetric val; - DimensionOverridableTestMetricSet(const std::string& dimValue, - MetricSet* owner = nullptr) - : MetricSet("temp", {{"foo", dimValue}}, "", owner), - val("val", {}, "val desc", this) - { } + DimensionOverridableTestMetricSet(const std::string& dimValue, MetricSet* owner = nullptr); + ~DimensionOverridableTestMetricSet() override; }; +DimensionOverridableTestMetricSet::DimensionOverridableTestMetricSet(const std::string& dimValue, MetricSet* owner) + : MetricSet("temp", {{"foo", dimValue}}, "", owner), + val("val", {}, "val desc", this) +{ } +DimensionOverridableTestMetricSet::~DimensionOverridableTestMetricSet() = default; + struct SameNamesTestMetricSet : MetricSet { DimensionOverridableTestMetricSet set1; @@ -954,7 +961,7 @@ SameNamesTestMetricSet::SameNamesTestMetricSet() set1("bar", this), set2("baz", this) { } -SameNamesTestMetricSet::~SameNamesTestMetricSet() { } +SameNamesTestMetricSet::~SameNamesTestMetricSet() = default; } diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/coredump/CoredumpHandler.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/coredump/CoredumpHandler.java index bfc4c09cf9e..8c293fb40c9 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/coredump/CoredumpHandler.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/coredump/CoredumpHandler.java @@ -180,23 +180,21 @@ public class CoredumpHandler { return coreEncryptionPublicKeyIdFlag.with(FetchVector.Dimension.NODE_TYPE, context.nodeType().name()).value(); } - static OutputStream maybeWrapWithEncryption(OutputStream wrappedStream, Optional<SecretSharedKey> sharedCoreKey) { - return sharedCoreKey - .map(key -> key.makeEncryptionCipher().wrapOutputStream(wrappedStream)) - .orElse(wrappedStream); + static OutputStream wrapWithEncryption(OutputStream wrappedStream, SecretSharedKey sharedCoreKey) { + return sharedCoreKey.makeEncryptionCipher().wrapOutputStream(wrappedStream); } /** * Compresses and, if a key is provided, encrypts core file (and deletes the uncompressed core), then moves * the entire core dump processing directory to {@link #doneCoredumpsPath} for archive */ - private void finishProcessing(NodeAgentContext context, ContainerPath coredumpDirectory, Optional<SecretSharedKey> sharedCoreKey) { + private void finishProcessing(NodeAgentContext context, ContainerPath coredumpDirectory, SecretSharedKey sharedCoreKey) { ContainerPath coreFile = findCoredumpFileInProcessingDirectory(coredumpDirectory); - String extension = COMPRESSED_EXTENSION + (sharedCoreKey.isPresent() ? ENCRYPTED_EXTENSION : ""); + String extension = COMPRESSED_EXTENSION + ENCRYPTED_EXTENSION; ContainerPath compressedCoreFile = coreFile.resolveSibling(coreFile.getFileName() + extension); try (ZstdCompressingInputStream zcis = new ZstdCompressingInputStream(Files.newInputStream(coreFile)); - OutputStream fos = maybeWrapWithEncryption(Files.newOutputStream(compressedCoreFile), sharedCoreKey)) { + OutputStream fos = wrapWithEncryption(Files.newOutputStream(compressedCoreFile), sharedCoreKey)) { zcis.transferTo(fos); } catch (IOException e) { throw new UncheckedIOException(e); @@ -287,11 +285,12 @@ public class CoredumpHandler { dockerImage.ifPresent(metadata::setDockerImage); dockerImage.flatMap(DockerImage::tag).ifPresent(metadata::setVespaVersion); dockerImage.ifPresent(metadata::setDockerImage); - Optional<SecretSharedKey> sharedCoreKey = Optional.of(corePublicKeyFlagValue(context)) + SecretSharedKey sharedCoreKey = Optional.of(corePublicKeyFlagValue(context)) .filter(k -> !k.isEmpty()) .map(KeyId::ofString) - .flatMap(secretSharedKeySupplier::create); - sharedCoreKey.map(key -> key.sealedSharedKey().toTokenString()).ifPresent(metadata::setDecryptionToken); + .flatMap(secretSharedKeySupplier::create) + .orElseThrow(() -> ConvergenceException.ofError("No core dump encryption key provided")); + metadata.setDecryptionToken(sharedCoreKey.sealedSharedKey().toTokenString()); String coreDumpId = coreDumpDirectory.getFileName().toString(); cores.report(context.hostname(), coreDumpId, metadata); diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/coredump/CoredumpHandlerTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/coredump/CoredumpHandlerTest.java index 33d785eb04e..b0bd1b7b68f 100644 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/coredump/CoredumpHandlerTest.java +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/coredump/CoredumpHandlerTest.java @@ -12,6 +12,7 @@ import com.yahoo.vespa.hosted.node.admin.configserver.cores.CoreDumpMetadata; import com.yahoo.vespa.hosted.node.admin.configserver.cores.Cores; import com.yahoo.vespa.hosted.node.admin.container.metrics.DimensionMetrics; import com.yahoo.vespa.hosted.node.admin.container.metrics.Metrics; +import com.yahoo.vespa.hosted.node.admin.nodeadmin.ConvergenceException; import com.yahoo.vespa.hosted.node.admin.nodeagent.NodeAgentContext; import com.yahoo.vespa.hosted.node.admin.nodeagent.NodeAgentContextImpl; import com.yahoo.vespa.hosted.node.admin.task.util.file.UnixPath; @@ -196,15 +197,14 @@ public class CoredumpHandlerTest { }); } - void do_process_single_coredump_test(String expectedCoreFileName, boolean encrypt) throws IOException { + void do_process_single_coredump_test(String expectedCoreFileName) throws IOException { ContainerPath coredumpDirectory = context.paths().of("/path/to/coredump/proccessing/id-123"); Files.createDirectories(coredumpDirectory); Files.write(coredumpDirectory.resolve("metadata2.json"), "{\"test-metadata\":{}}".getBytes()); Files.createFile(coredumpDirectory.resolve("dump_bash.core.431")); assertFolderContents(coredumpDirectory, "metadata2.json", "dump_bash.core.431"); CoreDumpMetadata expectedMetadata = new CoreDumpMetadata(); - if (encrypt) - expectedMetadata.setDecryptionToken("131Q0MMF3hBuMVnXg1WnSFexZGrcwa9ZhfHlegLNwPIN6hQJnBxq5srLf3aZbYdlRVE"); + expectedMetadata.setDecryptionToken("131Q0MMF3hBuMVnXg1WnSFexZGrcwa9ZhfHlegLNwPIN6hQJnBxq5srLf3aZbYdlRVE"); coredumpHandler.processAndReportSingleCoreDump(context, coredumpDirectory, Optional.empty()); verify(coreCollector, never()).collect(any(), any()); @@ -215,31 +215,29 @@ public class CoredumpHandlerTest { } @Test - void process_single_coredump_test_without_encryption() throws IOException { - do_process_single_coredump_test("dump_bash.core.431.zst", false); + void processing_single_coredump_test_without_encryption_throws() throws IOException { + assertThrows(ConvergenceException.class, () -> do_process_single_coredump_test("dump_bash.core.431.zst")); } @Test void process_single_coredump_test_with_encryption() throws IOException { flagSource.withStringFlag(Flags.CORE_ENCRYPTION_PUBLIC_KEY_ID.id(), "bar-key"); when(secretSharedKeySupplier.create(KeyId.ofString("bar-key"))).thenReturn(Optional.of(makeFixedSecretSharedKey())); - do_process_single_coredump_test("dump_bash.core.431.zst.enc", true); + do_process_single_coredump_test("dump_bash.core.431.zst.enc"); } - // TODO fail closed instead of open @Test - void encryption_disabled_when_no_public_key_set_in_feature_flag() throws IOException { + void processing_throws_when_no_public_key_set_in_feature_flag() throws IOException { flagSource.withStringFlag(Flags.CORE_ENCRYPTION_PUBLIC_KEY_ID.id(), ""); // empty -> not set verify(secretSharedKeySupplier, never()).create(any()); - do_process_single_coredump_test("dump_bash.core.431.zst", false); // No .enc suffix; not encrypted + assertThrows(ConvergenceException.class, () -> do_process_single_coredump_test("dump_bash.core.431.zst")); } - // TODO fail closed instead of open @Test - void encryption_disabled_when_no_key_returned_for_key_id_specified_by_feature_flag() throws IOException { + void processing_throws_when_no_key_returned_for_key_id_specified_by_feature_flag() throws IOException { flagSource.withStringFlag(Flags.CORE_ENCRYPTION_PUBLIC_KEY_ID.id(), "baz-key"); when(secretSharedKeySupplier.create(KeyId.ofString("baz-key"))).thenReturn(Optional.empty()); - do_process_single_coredump_test("dump_bash.core.431.zst", false); // No .enc suffix; not encrypted + assertThrows(ConvergenceException.class, () -> do_process_single_coredump_test("dump_bash.core.431.zst")); } @Test diff --git a/searchcore/src/apps/verify_ranksetup/verify_ranksetup.cpp b/searchcore/src/apps/verify_ranksetup/verify_ranksetup.cpp index 43d0b709a1b..deba3925eb1 100644 --- a/searchcore/src/apps/verify_ranksetup/verify_ranksetup.cpp +++ b/searchcore/src/apps/verify_ranksetup/verify_ranksetup.cpp @@ -114,7 +114,7 @@ private: public: VerifyRankSetup(); ~VerifyRankSetup(); - const std::vector<search::fef::Message> & getMessages() const { return _messages; } + [[nodiscard]] const std::vector<search::fef::Message> & getMessages() const { return _messages; } bool verify(const std::string & configId); }; @@ -122,22 +122,25 @@ struct DummyRankingAssetsRepo : IRankingAssetsRepo { const RankingConstantsConfig &cfg; RankingExpressions _expressions; OnnxModels _onnxModels; - DummyRankingAssetsRepo(const RankingConstantsConfig &cfg_in, RankingExpressions expressions, OnnxModels onnxModels) - : cfg(cfg_in), - _expressions(std::move(expressions)), - _onnxModels(std::move(onnxModels)) - {} - vespalib::eval::ConstantValue::UP getConstant(const vespalib::string &name) const override; + DummyRankingAssetsRepo(const RankingConstantsConfig &cfg_in, RankingExpressions expressions, OnnxModels onnxModels); + ~DummyRankingAssetsRepo() override; + [[nodiscard]] vespalib::eval::ConstantValue::UP getConstant(const vespalib::string &name) const override; - vespalib::string getExpression(const vespalib::string & name) const override { + [[nodiscard]] vespalib::string getExpression(const vespalib::string & name) const override { return _expressions.loadExpression(name); } - const search::fef::OnnxModel *getOnnxModel(const vespalib::string & name) const override { + [[nodiscard]] const search::fef::OnnxModel *getOnnxModel(const vespalib::string & name) const override { return _onnxModels.getModel(name); } }; +DummyRankingAssetsRepo::DummyRankingAssetsRepo(const RankingConstantsConfig &cfg_in, RankingExpressions expressions, OnnxModels onnxModels) + : cfg(cfg_in), + _expressions(std::move(expressions)), + _onnxModels(std::move(onnxModels)) +{} +DummyRankingAssetsRepo::~DummyRankingAssetsRepo() = default; vespalib::eval::ConstantValue::UP DummyRankingAssetsRepo::getConstant(const vespalib::string &name) const { for (const auto &entry: cfg.constant) { @@ -178,14 +181,14 @@ VerifyRankSetup::verify(const search::index::Schema &schema, if (!rankSetup.getSecondPhaseRank().empty()) { ok = verifyFeature(factory, indexEnv, rankSetup.getSecondPhaseRank(), "second phase ranking", _messages) && ok; } - for (size_t i = 0; i < rankSetup.getSummaryFeatures().size(); ++i) { - ok = verifyFeature(factory, indexEnv, rankSetup.getSummaryFeatures()[i], "summary features", _messages) && ok; + for (const auto & i : rankSetup.getSummaryFeatures()) { + ok = verifyFeature(factory, indexEnv, i, "summary features", _messages) && ok; } for (const auto & feature : rankSetup.get_match_features()) { ok = verifyFeature(factory, indexEnv, feature, "match features", _messages) && ok; } - for (size_t i = 0; i < rankSetup.getDumpFeatures().size(); ++i) { - ok = verifyFeature(factory, indexEnv, rankSetup.getDumpFeatures()[i], "dump features", _messages) && ok; + for (const auto & i : rankSetup.getDumpFeatures()) { + ok = verifyFeature(factory, indexEnv, i, "dump features", _messages) && ok; } return ok; } @@ -205,12 +208,10 @@ VerifyRankSetup::verifyConfig(const VerifyRanksetupConfig &myCfg, search::index::SchemaBuilder::build(attributeCfg, schema); DummyRankingAssetsRepo repo(constantsCfg, make_expressions(expressionsCfg, myCfg, _messages), make_models(modelsCfg, myCfg, _messages)); - for(size_t i = 0; i < rankCfg.rankprofile.size(); i++) { + for(const auto & profile : rankCfg.rankprofile) { search::fef::Properties properties; - const RankProfilesConfig::Rankprofile &profile = rankCfg.rankprofile[i]; - for(size_t j = 0; j < profile.fef.property.size(); j++) { - properties.add(profile.fef.property[j].name, - profile.fef.property[j].value); + for(const auto & j : profile.fef.property) { + properties.add(j.name, j.value); } if (verify(schema, properties, repo)) { _messages.emplace_back(search::fef::Level::INFO, diff --git a/searchcore/src/tests/proton/attribute/attribute_manager/attribute_manager_test.cpp b/searchcore/src/tests/proton/attribute/attribute_manager/attribute_manager_test.cpp index 07167a91498..616bf9edf6a 100644 --- a/searchcore/src/tests/proton/attribute/attribute_manager/attribute_manager_test.cpp +++ b/searchcore/src/tests/proton/attribute/attribute_manager/attribute_manager_test.cpp @@ -181,7 +181,7 @@ struct AttributeManagerFixture proton::AttributeManager::SP _msp; proton::AttributeManager &_m; ImportedAttributesRepoBuilder _builder; - AttributeManagerFixture(BaseFixture &bf); + explicit AttributeManagerFixture(BaseFixture &bf); ~AttributeManagerFixture(); AttributeVector::SP addAttribute(const vespalib::string &name) { return _m.addAttribute({name, INT32_SINGLE}, createSerialNum); @@ -214,18 +214,21 @@ struct SequentialAttributeManager { SequentialAttributesInitializer initializer; proton::AttributeManager mgr; - SequentialAttributeManager(const AttributeManager &currMgr, AttrMgrSpec && newSpec) - : initializer(newSpec.getDocIdLimit()), - mgr(currMgr, std::move(newSpec), initializer) - { - mgr.addInitializedAttributes(initializer.getInitializedAttributes()); - } - ~SequentialAttributeManager() = default; + SequentialAttributeManager(const AttributeManager &currMgr, AttrMgrSpec && newSpec); + ~SequentialAttributeManager(); }; +SequentialAttributeManager::SequentialAttributeManager(const AttributeManager &currMgr, AttrMgrSpec && newSpec) + : initializer(newSpec.getDocIdLimit()), + mgr(currMgr, std::move(newSpec), initializer) +{ + mgr.addInitializedAttributes(initializer.getInitializedAttributes()); +} +SequentialAttributeManager::~SequentialAttributeManager() = default; + struct DummyInitializerTask : public InitializerTask { - virtual void run() override {} + void run() override {} }; struct ParallelAttributeManager @@ -240,12 +243,12 @@ struct ParallelAttributeManager ExecutorThreadService master; AttributeManagerInitializer::SP initializer; - ParallelAttributeManager(search::SerialNum configSerialNum, AttributeManager::SP baseAttrMgr, + ParallelAttributeManager(search::SerialNum configSerialNum, AttributeManager & baseAttrMgr, const AttributesConfig &attrCfg, uint32_t docIdLimit); ~ParallelAttributeManager(); }; -ParallelAttributeManager::ParallelAttributeManager(search::SerialNum configSerialNum, AttributeManager::SP baseAttrMgr, +ParallelAttributeManager::ParallelAttributeManager(search::SerialNum configSerialNum, AttributeManager & baseAttrMgr, const AttributesConfig &attrCfg, uint32_t docIdLimit) : documentMetaStoreInitTask(std::make_shared<DummyInitializerTask>()), bucketDbOwner(std::make_shared<bucketdb::BucketDBOwner>()), @@ -256,7 +259,7 @@ ParallelAttributeManager::ParallelAttributeManager(search::SerialNum configSeria masterExecutor(1), master(masterExecutor), initializer(std::make_shared<AttributeManagerInitializer>(configSerialNum, documentMetaStoreInitTask, - documentMetaStore, *baseAttrMgr, attrCfg, + documentMetaStore, baseAttrMgr, attrCfg, alloc_strategy, fastAccessAttributesOnly, master, mgr)) { @@ -389,7 +392,7 @@ TEST_F("require that predicate attributes are flushed and loaded", BaseFixture) AttributeVector::SP a1 = am.addAttribute({"a1", AttributeUtils::getPredicateConfig()}, createSerialNum); EXPECT_EQUAL(1u, a1->getNumDocs()); - auto &pa = static_cast<PredicateAttribute &>(*a1); + auto &pa = dynamic_cast<PredicateAttribute &>(*a1); PredicateIndex &index = pa.getIndex(); uint32_t doc_id; a1->addDoc(doc_id); @@ -409,7 +412,7 @@ TEST_F("require that predicate attributes are flushed and loaded", BaseFixture) AttributeVector::SP a1 = am.addAttribute({"a1", AttributeUtils::getPredicateConfig()}, createSerialNum); // loaded EXPECT_EQUAL(2u, a1->getNumDocs()); - auto &pa = static_cast<PredicateAttribute &>(*a1); + auto &pa = dynamic_cast<PredicateAttribute &>(*a1); PredicateIndex &index = pa.getIndex(); uint32_t doc_id; a1->addDoc(doc_id); @@ -441,9 +444,9 @@ TEST_F("require that reconfig can add attributes", Fixture) f._m.addExtraAttribute(ex); AttrSpecList newSpec; - newSpec.push_back(AttributeSpec("a1", INT32_SINGLE)); - newSpec.push_back(AttributeSpec("a2", INT32_SINGLE)); - newSpec.push_back(AttributeSpec("a3", INT32_SINGLE)); + newSpec.emplace_back("a1", INT32_SINGLE); + newSpec.emplace_back("a2", INT32_SINGLE); + newSpec.emplace_back("a3", INT32_SINGLE); SequentialAttributeManager sam(f._m, AttrMgrSpec(std::move(newSpec), f._m.getNumDocs(), 10)); std::vector<AttributeGuard> list; @@ -466,7 +469,7 @@ TEST_F("require that reconfig can remove attributes", Fixture) AttributeVector::SP a3 = f.addAttribute("a3"); AttrSpecList newSpec; - newSpec.push_back(AttributeSpec("a2", INT32_SINGLE)); + newSpec.emplace_back("a2", INT32_SINGLE); SequentialAttributeManager sam(f._m, AttrMgrSpec(std::move(newSpec), 1, 10)); std::vector<AttributeGuard> list; @@ -487,9 +490,9 @@ TEST_F("require that new attributes after reconfig are initialized", Fixture) EXPECT_EQUAL(3u, a1->getNumDocs()); AttrSpecList newSpec; - newSpec.push_back(AttributeSpec("a1", INT32_SINGLE)); - newSpec.push_back(AttributeSpec("a2", INT32_SINGLE)); - newSpec.push_back(AttributeSpec("a3", INT32_ARRAY)); + newSpec.emplace_back("a1", INT32_SINGLE); + newSpec.emplace_back("a2", INT32_SINGLE); + newSpec.emplace_back("a3", INT32_ARRAY); SequentialAttributeManager sam(f._m, AttrMgrSpec(std::move(newSpec), 3, 4)); AttributeGuard::UP a2ap = sam.mgr.getAttribute("a2"); @@ -521,7 +524,7 @@ TEST_F("require that removed attributes cannot resurrect", BaseFixture) am1.reset(); AttrSpecList ns2; - ns2.push_back(AttributeSpec("a1", INT32_SINGLE)); + ns2.emplace_back("a1", INT32_SINGLE); // 2 new documents added since a1 was removed SequentialAttributeManager am3(am2.mgr, AttrMgrSpec(std::move(ns2), 5, 20)); @@ -555,7 +558,7 @@ TEST_F("require that removed fields can be pruned", Fixture) f._m.flushAll(10); AttrSpecList newSpec; - newSpec.push_back(AttributeSpec("a2", INT32_SINGLE)); + newSpec.emplace_back("a2", INT32_SINGLE); SequentialAttributeManager sam(f._m, AttrMgrSpec(std::move(newSpec), 1, 11)); sam.mgr.pruneRemovedFields(11); @@ -684,9 +687,9 @@ TEST_F("require that attributes can be initialized and loaded in sequence", Base AttributeManagerFixture amf(f); AttrSpecList newSpec; - newSpec.push_back(AttributeSpec("a1", INT32_SINGLE)); - newSpec.push_back(AttributeSpec("a2", INT32_SINGLE)); - newSpec.push_back(AttributeSpec("a3", INT32_SINGLE)); + newSpec.emplace_back("a1", INT32_SINGLE); + newSpec.emplace_back("a2", INT32_SINGLE); + newSpec.emplace_back("a3", INT32_SINGLE); SequentialAttributeManager newMgr(amf._m, AttrMgrSpec(std::move(newSpec), 10, createSerialNum + 5)); @@ -723,7 +726,7 @@ TEST_F("require that attributes can be initialized and loaded in parallel", Base attrCfg.attribute.push_back(createAttributeConfig("a2")); attrCfg.attribute.push_back(createAttributeConfig("a3")); - ParallelAttributeManager newMgr(createSerialNum + 5, amf._msp, attrCfg, 10); + ParallelAttributeManager newMgr(createSerialNum + 5, *amf._msp, attrCfg, 10); AttributeGuard::UP a1 = newMgr.mgr->get()->getAttribute("a1"); TEST_DO(validateAttribute(*a1->get())); @@ -811,12 +814,12 @@ TEST_F("require that attribute vector of wrong type is dropped", BaseFixture) am1->addAttribute({"a5", predicate}, 5); am1->addAttribute({"a6", predicate}, 6); AttrSpecList newSpec; - newSpec.push_back(AttributeSpec("a1", INT32_SINGLE)); - newSpec.push_back(AttributeSpec("a2", INT32_ARRAY)); - newSpec.push_back(AttributeSpec("a3", generic_tensor)); - newSpec.push_back(AttributeSpec("a4", dense_tensor)); - newSpec.push_back(AttributeSpec("a5", predicate)); - newSpec.push_back(AttributeSpec("a6", predicate2)); + newSpec.emplace_back("a1", INT32_SINGLE); + newSpec.emplace_back("a2", INT32_ARRAY); + newSpec.emplace_back("a3", generic_tensor); + newSpec.emplace_back("a4", dense_tensor); + newSpec.emplace_back("a5", predicate); + newSpec.emplace_back("a6", predicate2); SequentialAttributeManager am2(*am1, AttrMgrSpec(std::move(newSpec), 5, 20)); TEST_DO(assertCreateSerialNum(*am1, "a1", 1)); TEST_DO(assertCreateSerialNum(*am1, "a2", 2)); @@ -858,7 +861,7 @@ TEST_F("require that shrink flushtarget is handed over to new attribute manager" auto am1 = f.make_manager(); am1->addAttribute({"a1", INT32_SINGLE}, 4); AttrSpecList newSpec; - newSpec.push_back(AttributeSpec("a1", INT32_SINGLE)); + newSpec.emplace_back("a1", INT32_SINGLE); auto am2 = am1->create(AttrMgrSpec(std::move(newSpec), 5, 20)); auto am3 = std::dynamic_pointer_cast<AttributeManager>(am2); TEST_DO(assertShrinkTargetSerial(*am3, "a1", 3)); diff --git a/searchcore/src/tests/proton/documentdb/buckethandler/buckethandler_test.cpp b/searchcore/src/tests/proton/documentdb/buckethandler/buckethandler_test.cpp index dbc0b890cba..c9262605274 100644 --- a/searchcore/src/tests/proton/documentdb/buckethandler/buckethandler_test.cpp +++ b/searchcore/src/tests/proton/documentdb/buckethandler/buckethandler_test.cpp @@ -31,33 +31,31 @@ struct MySubDb { DocumentMetaStore _metaStore; test::UserDocuments _docs; - MySubDb(std::shared_ptr<bucketdb::BucketDBOwner> bucketDB, SubDbType subDbType) - : _metaStore(bucketDB, - DocumentMetaStore::getFixedName(), - search::GrowStrategy(), - subDbType), - _docs() - { - } + MySubDb(std::shared_ptr<bucketdb::BucketDBOwner> bucketDB, SubDbType subDbType); + ~MySubDb(); void insertDocs(const test::UserDocuments &docs_) { _docs = docs_; - for (test::UserDocuments::Iterator itr = _docs.begin(); itr != _docs.end(); ++itr) { - const test::BucketDocuments &bucketDocs = itr->second; - for (size_t i = 0; i < bucketDocs.getDocs().size(); ++i) { - const test::Document &testDoc = bucketDocs.getDocs()[i]; + for (const auto & _doc : _docs) { + const test::BucketDocuments &bucketDocs = _doc.second; + for (const auto & testDoc : bucketDocs.getDocs()) { _metaStore.put(testDoc.getGid(), testDoc.getBucket(), testDoc.getTimestamp(), testDoc.getDocSize(), testDoc.getLid(), 0u); } } } - BucketId bucket(uint32_t userId) { + BucketId bucket(uint32_t userId) const { return _docs.getBucket(userId); } - test::DocumentVector docs(uint32_t userId) { + test::DocumentVector docs(uint32_t userId) const { return _docs.getGidOrderDocs(userId); } }; +MySubDb::MySubDb(std::shared_ptr<bucketdb::BucketDBOwner> bucketDB, SubDbType subDbType) + : _metaStore(std::move(bucketDB), DocumentMetaStore::getFixedName(), search::GrowStrategy(), subDbType), + _docs() +{ } +MySubDb::~MySubDb() = default; struct MyChangedHandler : public IBucketStateChangedHandler { @@ -65,8 +63,8 @@ struct MyChangedHandler : public IBucketStateChangedHandler BucketInfo::ActiveState _state; MyChangedHandler() : _bucket(), _state(BucketInfo::NOT_ACTIVE) {} - virtual void notifyBucketStateChanged(const document::BucketId &bucketId, - storage::spi::BucketInfo::ActiveState newState) override { + void notifyBucketStateChanged(const document::BucketId &bucketId, + storage::spi::BucketInfo::ActiveState newState) override { _bucket = bucketId; _state = newState; } diff --git a/searchcore/src/tests/proton/flushengine/prepare_restart_flush_strategy/prepare_restart_flush_strategy_test.cpp b/searchcore/src/tests/proton/flushengine/prepare_restart_flush_strategy/prepare_restart_flush_strategy_test.cpp index 6aa17151942..fb29f968d61 100644 --- a/searchcore/src/tests/proton/flushengine/prepare_restart_flush_strategy/prepare_restart_flush_strategy_test.cpp +++ b/searchcore/src/tests/proton/flushengine/prepare_restart_flush_strategy/prepare_restart_flush_strategy_test.cpp @@ -14,7 +14,6 @@ using search::SerialNum; using searchcorespi::IFlushTarget; using SimpleFlushHandler = test::DummyFlushHandler; -using FlushCandidatesList = std::vector<FlushTargetCandidates>; using Config = PrepareRestartFlushStrategy::Config; const Config DEFAULT_CFG(2.0, 0.0, 4.0); @@ -34,13 +33,13 @@ struct SimpleFlushTarget : public test::DummyFlushTarget approxDiskBytes(approxDiskBytes_), replay_operation_cost(replay_operation_cost_) {} - SerialNum getFlushedSerialNum() const override { + [[nodiscard]] SerialNum getFlushedSerialNum() const override { return flushedSerial; } - uint64_t getApproxBytesToWriteToDisk() const override { + [[nodiscard]] uint64_t getApproxBytesToWriteToDisk() const override { return approxDiskBytes; } - double get_replay_operation_cost() const override { + [[nodiscard]] double get_replay_operation_cost() const override { return replay_operation_cost; } }; @@ -62,7 +61,8 @@ private: } public: - ContextsBuilder() : _result(), _handlers() {} + ContextsBuilder() noexcept; + ~ContextsBuilder(); ContextsBuilder &add(const vespalib::string &handlerName, const vespalib::string &targetName, IFlushTarget::Type targetType, @@ -97,9 +97,12 @@ public: double replay_operation_cost = 0.0) { return add("handler1", targetName, IFlushTarget::Type::GC, flushedSerial, approxDiskBytes, replay_operation_cost); } - FlushContext::List build() const { return _result; } + [[nodiscard]] FlushContext::List build() const { return _result; } }; +ContextsBuilder::ContextsBuilder() noexcept = default; +ContextsBuilder::~ContextsBuilder() = default; + class CandidatesBuilder { private: @@ -110,7 +113,7 @@ private: Config _cfg; public: - CandidatesBuilder(const FlushContext::List &sortedFlushContexts) + explicit CandidatesBuilder(const FlushContext::List &sortedFlushContexts) : _sortedFlushContexts(&sortedFlushContexts), _numCandidates(sortedFlushContexts.size()), _candidates(), @@ -142,10 +145,7 @@ public: } FlushTargetCandidates build() const { setup_candidates(); - return FlushTargetCandidates(_candidates, - _numCandidates, - _tlsStats, - _cfg); + return {_candidates, _numCandidates, _tlsStats, _cfg}; } }; @@ -212,14 +212,14 @@ struct FlushStrategyFixture { flushengine::TlsStatsMap _tlsStatsMap; PrepareRestartFlushStrategy strategy; - FlushStrategyFixture(const Config &config) + explicit FlushStrategyFixture(const Config &config) : _tlsStatsMap(defaultTransactionLogStats()), strategy(config) {} FlushStrategyFixture() : FlushStrategyFixture(DEFAULT_CFG) {} - FlushContext::List getFlushTargets(const FlushContext::List &targetList, + [[nodiscard]] FlushContext::List getFlushTargets(const FlushContext::List &targetList, const flushengine::TlsStatsMap &tlsStatsMap) const { flushengine::ActiveFlushStats active_flushes; return strategy.getFlushTargets(targetList, tlsStatsMap, active_flushes); diff --git a/searchcore/src/tests/proton/reference/document_db_reference_resolver/CMakeLists.txt b/searchcore/src/tests/proton/reference/document_db_reference_resolver/CMakeLists.txt index c5081f27db6..aab02c2b48c 100644 --- a/searchcore/src/tests/proton/reference/document_db_reference_resolver/CMakeLists.txt +++ b/searchcore/src/tests/proton/reference/document_db_reference_resolver/CMakeLists.txt @@ -5,6 +5,7 @@ vespa_add_executable(searchcore_document_db_reference_resolver_test_app TEST DEPENDS searchcore_reference searchcore_attribute + searchcore_test searchlib_test ) vespa_add_test(NAME searchcore_document_db_reference_resolver_test_app COMMAND searchcore_document_db_reference_resolver_test_app) diff --git a/searchcore/src/tests/proton/reference/gid_to_lid_change_handler/CMakeLists.txt b/searchcore/src/tests/proton/reference/gid_to_lid_change_handler/CMakeLists.txt index 6ac416d2ce2..d9d6501af5e 100644 --- a/searchcore/src/tests/proton/reference/gid_to_lid_change_handler/CMakeLists.txt +++ b/searchcore/src/tests/proton/reference/gid_to_lid_change_handler/CMakeLists.txt @@ -5,5 +5,6 @@ vespa_add_executable(searchcore_gid_to_lid_change_handler_test_app TEST DEPENDS searchcore_reference searchcore_server + searchcore_test ) vespa_add_test(NAME searchcore_gid_to_lid_change_handler_test_app COMMAND searchcore_gid_to_lid_change_handler_test_app) diff --git a/searchcore/src/tests/proton/reference/gid_to_lid_change_registrator/CMakeLists.txt b/searchcore/src/tests/proton/reference/gid_to_lid_change_registrator/CMakeLists.txt index 30929059cab..6808258cbc8 100644 --- a/searchcore/src/tests/proton/reference/gid_to_lid_change_registrator/CMakeLists.txt +++ b/searchcore/src/tests/proton/reference/gid_to_lid_change_registrator/CMakeLists.txt @@ -5,5 +5,6 @@ vespa_add_executable(searchcore_gid_to_lid_change_registrator_test_app TEST DEPENDS searchcore_reference searchcore_server + searchcore_test ) vespa_add_test(NAME searchcore_gid_to_lid_change_registrator_test_app COMMAND searchcore_gid_to_lid_change_registrator_test_app) diff --git a/searchcore/src/vespa/searchcore/proton/matching/onnx_models.h b/searchcore/src/vespa/searchcore/proton/matching/onnx_models.h index c0582600973..4a8bdbca3d9 100644 --- a/searchcore/src/vespa/searchcore/proton/matching/onnx_models.h +++ b/searchcore/src/vespa/searchcore/proton/matching/onnx_models.h @@ -34,8 +34,8 @@ public: OnnxModels & operator =(const OnnxModels &) = delete; ~OnnxModels(); bool operator==(const OnnxModels &rhs) const; - const Model *getModel(const vespalib::string &name) const; - size_t size() const { return _models.size(); } + [[nodiscard]] const Model *getModel(const vespalib::string &name) const; + [[nodiscard]] size_t size() const { return _models.size(); } static void configure(const ModelConfig &config, Model &model); }; diff --git a/searchcore/src/vespa/searchcore/proton/matching/querynodes.h b/searchcore/src/vespa/searchcore/proton/matching/querynodes.h index fc3f12af4f7..4e9f03f2fac 100644 --- a/searchcore/src/vespa/searchcore/proton/matching/querynodes.h +++ b/searchcore/src/vespa/searchcore/proton/matching/querynodes.h @@ -36,12 +36,11 @@ public: attribute_field(false), filter_field(false) {} - FieldSpec fieldSpec() const { - return FieldSpec(field_name, getFieldId(), - getHandle(), filter_field); + [[nodiscard]] FieldSpec fieldSpec() const { + return {field_name, getFieldId(), getHandle(), filter_field}; } using SimpleTermFieldData::getHandle; - search::fef::TermFieldHandle getHandle(search::fef::MatchDataDetails requested_details) const override; + [[nodiscard]] search::fef::TermFieldHandle getHandle(search::fef::MatchDataDetails requested_details) const override; }; private: @@ -67,10 +66,10 @@ public: void setDocumentFrequency(uint32_t estHits, uint32_t numDocs); // ITermData interface - std::optional<vespalib::string> query_tensor_name() const override { return std::nullopt; } - size_t numFields() const override final { return _fields.size(); } - const FieldEntry &field(size_t i) const override final { return _fields[i]; } - const FieldEntry *lookupField(uint32_t fieldId) const override final; + [[nodiscard]] std::optional<vespalib::string> query_tensor_name() const override { return std::nullopt; } + [[nodiscard]] size_t numFields() const final { return _fields.size(); } + [[nodiscard]] const FieldEntry &field(size_t i) const final { return _fields[i]; } + [[nodiscard]] const FieldEntry *lookupField(uint32_t fieldId) const final; }; template <typename NodeType> inline uint32_t numTerms(const NodeType &) { return 1; } @@ -85,6 +84,7 @@ struct ProtonTermBase : public Base, public ProtonTermData { using Base::Base; + ~ProtonTermBase() override; void resolve(const ViewResolver &resolver, const search::fef::IIndexEnvironment &idxEnv) { @@ -93,12 +93,15 @@ struct ProtonTermBase : public Base, } // ITermData interface - uint32_t getPhraseLength() const final { return numTerms<Base>(*this); } - search::query::Weight getWeight() const final { return Base::getWeight(); } - uint32_t getUniqueId() const final { return Base::getId(); } + [[nodiscard]] uint32_t getPhraseLength() const final { return numTerms<Base>(*this); } + [[nodiscard]] search::query::Weight getWeight() const final { return Base::getWeight(); } + [[nodiscard]] uint32_t getUniqueId() const final { return Base::getId(); } }; template <typename Base> +ProtonTermBase<Base>::~ProtonTermBase() = default; + +template <typename Base> struct ProtonTerm final : public ProtonTermBase<Base> { using ProtonTermBase<Base>::ProtonTermBase; ~ProtonTerm(); @@ -128,7 +131,7 @@ struct ProtonSameElement final : public ProtonTermBase<search::query::SameElemen struct ProtonNearestNeighborTerm : public ProtonTermBase<search::query::NearestNeighborTerm> { using ProtonTermBase::ProtonTermBase; - std::optional<vespalib::string> query_tensor_name() const override { + [[nodiscard]] std::optional<vespalib::string> query_tensor_name() const override { return ProtonTermBase::NearestNeighborTerm::get_query_tensor_name(); } }; diff --git a/searchcore/src/vespa/searchcore/proton/server/CMakeLists.txt b/searchcore/src/vespa/searchcore/proton/server/CMakeLists.txt index c1f7b24b05c..34e2ee283f1 100644 --- a/searchcore/src/vespa/searchcore/proton/server/CMakeLists.txt +++ b/searchcore/src/vespa/searchcore/proton/server/CMakeLists.txt @@ -19,6 +19,7 @@ vespa_add_library(searchcore_server STATIC document_db_explorer.cpp document_db_flush_config.cpp document_db_maintenance_config.cpp + document_db_reconfig.cpp document_meta_store_read_guards.cpp document_scan_iterator.cpp document_subdb_collection_explorer.cpp @@ -26,6 +27,7 @@ vespa_add_library(searchcore_server STATIC document_subdb_explorer.cpp document_subdb_initializer.cpp document_subdb_initializer_result.cpp + document_subdb_reconfig.cpp documentbucketmover.cpp documentdb.cpp documentdb_metrics_updater.cpp diff --git a/searchcore/src/vespa/searchcore/proton/server/document_db_reconfig.cpp b/searchcore/src/vespa/searchcore/proton/server/document_db_reconfig.cpp new file mode 100644 index 00000000000..bd7b14483ad --- /dev/null +++ b/searchcore/src/vespa/searchcore/proton/server/document_db_reconfig.cpp @@ -0,0 +1,16 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include "document_db_reconfig.h" +#include "document_subdb_reconfig.h" + +namespace proton { + +DocumentDBReconfig::DocumentDBReconfig(std::unique_ptr<const DocumentSubDBReconfig> ready_reconfig_in, + std::unique_ptr<const DocumentSubDBReconfig> not_ready_reconfig_in) + : _ready_reconfig(std::move(ready_reconfig_in)), + _not_ready_reconfig(std::move(not_ready_reconfig_in)) +{ +} + +} + diff --git a/searchcore/src/vespa/searchcore/proton/server/document_db_reconfig.h b/searchcore/src/vespa/searchcore/proton/server/document_db_reconfig.h new file mode 100644 index 00000000000..c1f768511c8 --- /dev/null +++ b/searchcore/src/vespa/searchcore/proton/server/document_db_reconfig.h @@ -0,0 +1,35 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#pragma once + +#include <memory> + +namespace proton { + +class DocumentSubDBReconfig; + +/** + * Class representing the result of the prepare step of a DocumentDB reconfig. + * + * The reconfig is performed in two steps: + * Prepare: + * Based on the config that is changed, new components are instantiated in each subdb. + * This can be costly and is handled by helper threads from the shared executor pool. + * + * Apply: + * The new components are swapped with the old ones in the DocumentDB master write thread. + */ +class DocumentDBReconfig { +private: + std::unique_ptr<const DocumentSubDBReconfig> _ready_reconfig; + std::unique_ptr<const DocumentSubDBReconfig> _not_ready_reconfig; + +public: + DocumentDBReconfig(std::unique_ptr<const DocumentSubDBReconfig> ready_reconfig_in, + std::unique_ptr<const DocumentSubDBReconfig> not_ready_reconfig_in); + + const DocumentSubDBReconfig& ready_reconfig() const { return *_ready_reconfig; } + const DocumentSubDBReconfig& not_ready_reconfig() const { return *_not_ready_reconfig; } +}; + +} + diff --git a/searchcore/src/vespa/searchcore/proton/server/document_subdb_reconfig.cpp b/searchcore/src/vespa/searchcore/proton/server/document_subdb_reconfig.cpp new file mode 100644 index 00000000000..3692fa07c06 --- /dev/null +++ b/searchcore/src/vespa/searchcore/proton/server/document_subdb_reconfig.cpp @@ -0,0 +1,14 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include "document_subdb_reconfig.h" + +namespace proton { + +DocumentSubDBReconfig::DocumentSubDBReconfig(std::shared_ptr<Matchers> matchers_in) + : _old_matchers(matchers_in), + _new_matchers(matchers_in) +{ +} + +} + diff --git a/searchcore/src/vespa/searchcore/proton/server/document_subdb_reconfig.h b/searchcore/src/vespa/searchcore/proton/server/document_subdb_reconfig.h new file mode 100644 index 00000000000..ed9faeee6fc --- /dev/null +++ b/searchcore/src/vespa/searchcore/proton/server/document_subdb_reconfig.h @@ -0,0 +1,31 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#pragma once + +#include <memory> + +namespace proton { + +class Matchers; + +/** + * Class representing the result of the prepare step of a IDocumentSubDB reconfig. + */ +class DocumentSubDBReconfig { +private: + std::shared_ptr<Matchers> _old_matchers; + std::shared_ptr<Matchers> _new_matchers; + +public: + DocumentSubDBReconfig(std::shared_ptr<Matchers> matchers_in); + + void set_matchers(std::shared_ptr<Matchers> value) { + _new_matchers = std::move(value); + } + bool has_matchers_changed() const noexcept { + return _old_matchers != _new_matchers; + } + std::shared_ptr<Matchers> matchers() const { return _new_matchers; } +}; + +} + diff --git a/searchcore/src/vespa/searchcore/proton/server/maintenance_controller_explorer.cpp b/searchcore/src/vespa/searchcore/proton/server/maintenance_controller_explorer.cpp index f7123c1f862..170632739da 100644 --- a/searchcore/src/vespa/searchcore/proton/server/maintenance_controller_explorer.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/maintenance_controller_explorer.cpp @@ -41,6 +41,7 @@ MaintenanceControllerExplorer(std::vector<MaintenanceJobRunner::SP> jobs) : _jobs(std::move(jobs)) { } +MaintenanceControllerExplorer::~MaintenanceControllerExplorer() = default; void MaintenanceControllerExplorer::get_state(const Inserter &inserter, bool full) const diff --git a/searchcore/src/vespa/searchcore/proton/server/maintenance_controller_explorer.h b/searchcore/src/vespa/searchcore/proton/server/maintenance_controller_explorer.h index 76aef09e58c..aabb9836dcb 100644 --- a/searchcore/src/vespa/searchcore/proton/server/maintenance_controller_explorer.h +++ b/searchcore/src/vespa/searchcore/proton/server/maintenance_controller_explorer.h @@ -16,7 +16,8 @@ private: std::vector<MaintenanceJobRunner::SP> _jobs; public: - MaintenanceControllerExplorer(std::vector<MaintenanceJobRunner::SP> jobs); + explicit MaintenanceControllerExplorer(std::vector<MaintenanceJobRunner::SP> jobs); + ~MaintenanceControllerExplorer() override; void get_state(const vespalib::slime::Inserter &inserter, bool full) const override; }; diff --git a/searchcore/src/vespa/searchcore/proton/test/CMakeLists.txt b/searchcore/src/vespa/searchcore/proton/test/CMakeLists.txt index 79b0582e1d0..e3b1aad3c60 100644 --- a/searchcore/src/vespa/searchcore/proton/test/CMakeLists.txt +++ b/searchcore/src/vespa/searchcore/proton/test/CMakeLists.txt @@ -11,6 +11,7 @@ vespa_add_library(searchcore_test STATIC dummy_feed_view.cpp dummy_flush_target.cpp dummydbowner.cpp + mock_gid_to_lid_change_handler.cpp mock_index_manager.cpp mock_shared_threading_service.cpp userdocumentsbuilder.cpp diff --git a/searchcore/src/vespa/searchcore/proton/test/mock_gid_to_lid_change_handler.cpp b/searchcore/src/vespa/searchcore/proton/test/mock_gid_to_lid_change_handler.cpp new file mode 100644 index 00000000000..ea011dd2d37 --- /dev/null +++ b/searchcore/src/vespa/searchcore/proton/test/mock_gid_to_lid_change_handler.cpp @@ -0,0 +1,17 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include "mock_gid_to_lid_change_handler.h" + +namespace proton::test { + +MockGidToLidChangeHandler::MockGidToLidChangeHandler() noexcept + : IGidToLidChangeHandler(), + _adds(), + _removes(), + _listeners() +{ +} + +MockGidToLidChangeHandler::~MockGidToLidChangeHandler() = default; + +} diff --git a/searchcore/src/vespa/searchcore/proton/test/mock_gid_to_lid_change_handler.h b/searchcore/src/vespa/searchcore/proton/test/mock_gid_to_lid_change_handler.h index 5931defbe7e..d7c72e3109a 100644 --- a/searchcore/src/vespa/searchcore/proton/test/mock_gid_to_lid_change_handler.h +++ b/searchcore/src/vespa/searchcore/proton/test/mock_gid_to_lid_change_handler.h @@ -6,6 +6,7 @@ #include <vespa/searchcore/proton/reference/i_pending_gid_to_lid_changes.h> #include <vespa/vespalib/testkit/testapp.h> #include <vespa/vespalib/test/insertion_operators.h> +#include <vespa/document/base/globalid.h> namespace proton::test { @@ -24,15 +25,8 @@ private: std::vector<std::unique_ptr<IGidToLidChangeListener>> _listeners; public: - MockGidToLidChangeHandler() noexcept - : IGidToLidChangeHandler(), - _adds(), - _removes(), - _listeners() - { - } - - ~MockGidToLidChangeHandler() override = default; + MockGidToLidChangeHandler() noexcept; + ~MockGidToLidChangeHandler() override; void addListener(std::unique_ptr<IGidToLidChangeListener> listener) override { _adds.emplace_back(listener->getDocTypeName(), listener->getName()); diff --git a/searchlib/src/tests/diskindex/bitvector/bitvector_test.cpp b/searchlib/src/tests/diskindex/bitvector/bitvector_test.cpp index 9bc0382f3ab..839ead48908 100644 --- a/searchlib/src/tests/diskindex/bitvector/bitvector_test.cpp +++ b/searchlib/src/tests/diskindex/bitvector/bitvector_test.cpp @@ -69,13 +69,11 @@ private: Schema _schema; uint32_t _indexId; public: - void - requireThatDictionaryHandlesNoEntries(bool directio, bool readmmap); - - void - requireThatDictionaryHandlesMultipleEntries(bool directio, bool readmmap); + void requireThatDictionaryHandlesNoEntries(bool directio, bool readmmap); + void requireThatDictionaryHandlesMultipleEntries(bool directio, bool readmmap); Test(); + ~Test() override; int Main() override; }; @@ -104,8 +102,8 @@ Test::requireThatDictionaryHandlesNoEntries(bool directio, bool readmmap) EXPECT_TRUE(dict.open("dump/1/", tuneFileRead, bvScope)); EXPECT_EQUAL(5u, dict.getDocIdLimit()); EXPECT_EQUAL(0u, dict.getEntries().size()); - EXPECT_TRUE(dict.lookup(1).get() == NULL); - EXPECT_TRUE(dict.lookup(2).get() == NULL); + EXPECT_FALSE(dict.lookup(1)); + EXPECT_FALSE(dict.lookup(2)); } void @@ -162,17 +160,17 @@ Test::requireThatDictionaryHandlesMultipleEntries(bool directio, bool readmmap) EXPECT_EQUAL(5u, e._wordNum); EXPECT_EQUAL(23u, e._numDocs); - EXPECT_TRUE(dict.lookup(2).get() == NULL); - EXPECT_TRUE(dict.lookup(3).get() == NULL); - EXPECT_TRUE(dict.lookup(4).get() == NULL); - EXPECT_TRUE(dict.lookup(6).get() == NULL); + EXPECT_FALSE(dict.lookup(2)); + EXPECT_FALSE(dict.lookup(3)); + EXPECT_FALSE(dict.lookup(4)); + EXPECT_FALSE(dict.lookup(6)); BitVector::UP bv1act = dict.lookup(1); - EXPECT_TRUE(bv1act.get() != NULL); + EXPECT_TRUE(bv1act); EXPECT_TRUE(*bv1exp == *bv1act); BitVector::UP bv5act = dict.lookup(5); - EXPECT_TRUE(bv5act.get() != NULL); + EXPECT_TRUE(bv5act); EXPECT_TRUE(*bv5exp == *bv5act); } @@ -183,6 +181,8 @@ Test::Test() _schema.addIndexField(Schema::IndexField("f1", DataType::STRING)); } +Test::~Test() = default; + int Test::Main() { diff --git a/searchlib/src/tests/docstore/store_by_bucket/store_by_bucket_test.cpp b/searchlib/src/tests/docstore/store_by_bucket/store_by_bucket_test.cpp index b849143427c..13aa1880e8c 100644 --- a/searchlib/src/tests/docstore/store_by_bucket/store_by_bucket_test.cpp +++ b/searchlib/src/tests/docstore/store_by_bucket/store_by_bucket_test.cpp @@ -57,6 +57,7 @@ public: _lastBucketId = bucketId; EXPECT_EQUAL(0, memcmp(buffer, createPayload(bucketId).c_str(), sz)); } + ~VerifyBucketOrder() override; private: uint32_t _lastLid; BucketId _lastBucketId; @@ -64,6 +65,8 @@ private: vespalib::hash_set<uint64_t> _uniqueBucket; }; +VerifyBucketOrder::~VerifyBucketOrder() = default; + TEST("require that StoreByBucket gives bucket by bucket and ordered within") { std::mutex backing_lock; diff --git a/searchlib/src/tests/features/onnx_feature/onnx_feature_test.cpp b/searchlib/src/tests/features/onnx_feature/onnx_feature_test.cpp index bf7cbeeb5a2..f6f6c62321e 100644 --- a/searchlib/src/tests/features/onnx_feature/onnx_feature_test.cpp +++ b/searchlib/src/tests/features/onnx_feature/onnx_feature_test.cpp @@ -179,9 +179,12 @@ struct MyIssues : Issue::Handler { std::vector<vespalib::string> list; Issue::Binding capture; MyIssues() : list(), capture(Issue::listen(*this)) {} + ~MyIssues() override; void handle(const Issue &issue) override { list.push_back(issue.message()); } }; +MyIssues::~MyIssues() = default; + TEST_F(OnnxFeatureTest, broken_model_evaluates_to_all_zeros) { add_expr("in1", "tensor<float>(x[2]):[docid,5]"); add_expr("in2", "tensor<float>(x[3]):[docid,10,31515]"); diff --git a/searchlib/src/tests/fef/featureoverride/featureoverride.cpp b/searchlib/src/tests/fef/featureoverride/featureoverride.cpp index 90ee5519cc4..f5082871a3f 100644 --- a/searchlib/src/tests/fef/featureoverride/featureoverride.cpp +++ b/searchlib/src/tests/fef/featureoverride/featureoverride.cpp @@ -245,9 +245,12 @@ struct MyIssues : Issue::Handler { std::vector<vespalib::string> list; Issue::Binding capture; MyIssues() : list(), capture(Issue::listen(*this)) {} + ~MyIssues() override; void handle(const Issue &issue) override { list.push_back(issue.message()); } }; +MyIssues::~MyIssues() = default; + //----------------------------------------------------------------------------- TEST_F("require expression without override works", SimpleRankFixture) { diff --git a/searchlib/src/tests/fef/phrasesplitter/benchmark.cpp b/searchlib/src/tests/fef/phrasesplitter/benchmark.cpp index b2a5cf0f4c0..89fdff7819c 100644 --- a/searchlib/src/tests/fef/phrasesplitter/benchmark.cpp +++ b/searchlib/src/tests/fef/phrasesplitter/benchmark.cpp @@ -24,9 +24,12 @@ private: public: Benchmark() : _timer(), _sample(0) {} + ~Benchmark() override; int Main() override; }; +Benchmark::~Benchmark() = default; + void Benchmark::run(size_t numRuns, size_t numPositions) { diff --git a/searchlib/src/tests/memoryindex/datastore/word_store_test.cpp b/searchlib/src/tests/memoryindex/datastore/word_store_test.cpp index 1de7b4a3efd..5c2bf0d634f 100644 --- a/searchlib/src/tests/memoryindex/datastore/word_store_test.cpp +++ b/searchlib/src/tests/memoryindex/datastore/word_store_test.cpp @@ -1,6 +1,8 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include <vespa/searchlib/memoryindex/word_store.h> #include <vespa/vespalib/gtest/gtest.h> +#include <vespa/vespalib/util/exceptions.h> +#include <vespa/vespalib/util/size_literals.h> #include <vespa/log/log.h> LOG_SETUP("word_store_test"); @@ -64,6 +66,13 @@ TEST(WordStoreTest, add_word_triggers_change_of_buffer) EXPECT_EQ(4u, lastId); } +TEST(WordStoreTest, long_word_triggers_exception) +{ + WordStore ws; + vespalib::string word(16_Mi + 1_Ki, 'z'); + EXPECT_THROW(ws.addWord(word), vespalib::OverflowException); +} + } GTEST_MAIN_RUN_ALL_TESTS() diff --git a/searchlib/src/tests/memoryindex/field_index/field_index_test.cpp b/searchlib/src/tests/memoryindex/field_index/field_index_test.cpp index 76056029f78..b57a2f42ea7 100644 --- a/searchlib/src/tests/memoryindex/field_index/field_index_test.cpp +++ b/searchlib/src/tests/memoryindex/field_index/field_index_test.cpp @@ -76,15 +76,8 @@ private: bool _firstDoc; public: - explicit MyBuilder(const Schema &schema) - : IndexBuilder(schema), - _ss(), - _insideWord(false), - _insideField(false), - _firstWord(true), - _firstField(true), - _firstDoc(true) - {} + explicit MyBuilder(const Schema &schema); + ~MyBuilder() override; void startWord(vespalib::stringref word) override { assert(_insideField); @@ -153,6 +146,17 @@ public: } }; +MyBuilder::MyBuilder(const Schema &schema) + : IndexBuilder(schema), + _ss(), + _insideWord(false), + _insideField(false), + _firstWord(true), + _firstField(true), + _firstDoc(true) +{} +MyBuilder::~MyBuilder() = default; + struct SimpleMatchData { TermFieldMatchData term; TermFieldMatchDataArray array; @@ -395,7 +399,7 @@ public: if (_inserter != nullptr) { _inserter->flush(); } - for (auto wfp : _mock) { + for (const auto& wfp : _mock) { auto &wf = wfp.first; auto &word = wf.first; auto fieldId = wf.second; @@ -522,7 +526,7 @@ myCompactFeatures(FieldIndexCollection &fieldIndexes, ISequencedTaskExecutor &pu Schema make_all_index_schema(DocBuilder::AddFieldsType add_fields) { - DocBuilder db(add_fields); + DocBuilder db(std::move(add_fields)); return SchemaBuilder(db).add_all_indexes().build(); } @@ -732,12 +736,8 @@ make_multi_field_add_fields() struct FieldIndexCollectionTest : public ::testing::Test { Schema schema; FieldIndexCollection fic; - FieldIndexCollectionTest() - : schema(make_all_index_schema(make_multi_field_add_fields())), - fic(schema, MockFieldLengthInspector()) - { - } - ~FieldIndexCollectionTest(); + FieldIndexCollectionTest(); + ~FieldIndexCollectionTest() override; [[nodiscard]]NormalFieldIndex::PostingList::Iterator find(const vespalib::stringref word, uint32_t field_id) const { @@ -745,6 +745,11 @@ struct FieldIndexCollectionTest : public ::testing::Test { } }; +FieldIndexCollectionTest::FieldIndexCollectionTest() + : schema(make_all_index_schema(make_multi_field_add_fields())), + fic(schema, MockFieldLengthInspector()) +{ +} FieldIndexCollectionTest::~FieldIndexCollectionTest() = default; TEST_F(FieldIndexCollectionTest, require_that_multiple_posting_lists_across_multiple_fields_can_exist) @@ -938,8 +943,8 @@ public: DocumentInverterContext _inv_context; DocumentInverter _inv; - InverterTest(DocBuilder::AddFieldsType add_fields) - : _b(add_fields), + explicit InverterTest(DocBuilder::AddFieldsType add_fields) + : _b(std::move(add_fields)), _schema(SchemaBuilder(_b).add_all_indexes().build()), _fic(_schema, MockFieldLengthInspector()), _invertThreads(SequencedTaskExecutor::create(invert_executor, 2)), @@ -948,14 +953,14 @@ public: _inv(_inv_context) { } - NormalFieldIndex::PostingList::Iterator find(const vespalib::stringref word, uint32_t field_id) const { + [[nodiscard]] NormalFieldIndex::PostingList::Iterator find(const vespalib::stringref word, uint32_t field_id) const { return find_in_field_index<false>(word, field_id, _fic); } - NormalFieldIndex::PostingList::ConstIterator findFrozen(const vespalib::stringref word, uint32_t field_id) const { + [[nodiscard]] NormalFieldIndex::PostingList::ConstIterator findFrozen(const vespalib::stringref word, uint32_t field_id) const { return find_frozen_in_field_index<false>(word, field_id, _fic); } - SearchIterator::UP search(const vespalib::stringref word, uint32_t field_id, - const SimpleMatchData& match_data) { + [[nodiscard]] SearchIterator::UP + search(const vespalib::stringref word, uint32_t field_id,const SimpleMatchData& match_data) const { return make_search_iterator<false>(findFrozen(word, field_id), featureStoreRef(_fic, field_id), field_id, match_data.array); } diff --git a/searchlib/src/tests/memoryindex/field_index_remover/field_index_remover_test.cpp b/searchlib/src/tests/memoryindex/field_index_remover/field_index_remover_test.cpp index 0a9964d5c31..dee8b7f9ace 100644 --- a/searchlib/src/tests/memoryindex/field_index_remover/field_index_remover_test.cpp +++ b/searchlib/src/tests/memoryindex/field_index_remover/field_index_remover_test.cpp @@ -63,20 +63,8 @@ struct FieldIndexRemoverTest : public ::testing::Test { std::vector<std::map<vespalib::string, vespalib::datastore::EntryRef>> _wordToRefMaps; std::vector<std::unique_ptr<FieldIndexRemover>> _removers; - FieldIndexRemoverTest() - : _listener(), - _wordStores(), - _wordToRefMaps(), - _removers() - { - uint32_t numFields = 4; - for (uint32_t fieldId = 0; fieldId < numFields; ++fieldId) { - _wordStores.push_back(std::make_unique<WordStore>()); - _removers.push_back(std::make_unique<FieldIndexRemover> - (*_wordStores.back())); - } - _wordToRefMaps.resize(numFields); - } + FieldIndexRemoverTest(); + ~FieldIndexRemoverTest() override; vespalib::datastore::EntryRef getWordRef(const vespalib::string &word, uint32_t fieldId) { auto &wordToRefMap = _wordToRefMaps[fieldId]; WordStore &wordStore = *_wordStores[fieldId]; @@ -110,6 +98,21 @@ struct FieldIndexRemoverTest : public ::testing::Test { } }; +FieldIndexRemoverTest::FieldIndexRemoverTest() + : _listener(), + _wordStores(), + _wordToRefMaps(), + _removers() +{ + uint32_t numFields = 4; + for (uint32_t fieldId = 0; fieldId < numFields; ++fieldId) { + _wordStores.push_back(std::make_unique<WordStore>()); + _removers.push_back(std::make_unique<FieldIndexRemover>(*_wordStores.back())); + } + _wordToRefMaps.resize(numFields); +} +FieldIndexRemoverTest::~FieldIndexRemoverTest() = default; + TEST_F(FieldIndexRemoverTest, word_field_id_pairs_for_multiple_doc_ids_can_be_inserted) { insert("a", 1, 10).insert("a", 1, 20).insert("a", 1, 30); diff --git a/searchlib/src/tests/memoryindex/field_inverter/field_inverter_test.cpp b/searchlib/src/tests/memoryindex/field_inverter/field_inverter_test.cpp index 6470d6593cd..2e3e56a123b 100644 --- a/searchlib/src/tests/memoryindex/field_inverter/field_inverter_test.cpp +++ b/searchlib/src/tests/memoryindex/field_inverter/field_inverter_test.cpp @@ -161,27 +161,8 @@ struct FieldInverterTest : public ::testing::Test { }; } - FieldInverterTest() - : _b(make_add_fields()), - _schema(SchemaBuilder(_b).add_all_indexes().build()), - _word_store(), - _remover(_word_store), - _inserter_backend(), - _calculators(), - _inserters(), - _inverters() - { - for (uint32_t fieldId = 0; fieldId < _schema.getNumIndexFields(); - ++fieldId) { - _calculators.emplace_back(std::make_unique<FieldLengthCalculator>()); - _inserters.emplace_back(std::make_unique<test::OrderedFieldIndexInserter>(_inserter_backend, fieldId)); - _inverters.push_back(std::make_unique<FieldInverter>(_schema, - fieldId, - _remover, - *_inserters.back(), - *_calculators.back())); - } - } + FieldInverterTest(); + ~FieldInverterTest() override; void invertDocument(uint32_t docId, const Document &doc) { uint32_t fieldId = 0; @@ -213,6 +194,24 @@ struct FieldInverterTest : public ::testing::Test { }; +FieldInverterTest::FieldInverterTest() + : _b(make_add_fields()), + _schema(SchemaBuilder(_b).add_all_indexes().build()), + _word_store(), + _remover(_word_store), + _inserter_backend(), + _calculators(), + _inserters(), + _inverters() +{ + for (uint32_t fieldId = 0; fieldId < _schema.getNumIndexFields(); ++fieldId) { + _calculators.emplace_back(std::make_unique<FieldLengthCalculator>()); + _inserters.emplace_back(std::make_unique<test::OrderedFieldIndexInserter>(_inserter_backend, fieldId)); + _inverters.push_back(std::make_unique<FieldInverter>(_schema, fieldId, _remover, *_inserters.back(), *_calculators.back())); + } +} +FieldInverterTest::~FieldInverterTest() = default; + TEST_F(FieldInverterTest, require_that_fresh_insert_works) { invertDocument(10, *makeDoc10(_b)); diff --git a/searchlib/src/tests/memoryindex/memory_index/memory_index_test.cpp b/searchlib/src/tests/memoryindex/memory_index/memory_index_test.cpp index 69063f38aeb..4f7a7934d4c 100644 --- a/searchlib/src/tests/memoryindex/memory_index/memory_index_test.cpp +++ b/searchlib/src/tests/memoryindex/memory_index/memory_index_test.cpp @@ -55,6 +55,8 @@ using namespace search::queryeval; struct MySetup : public IFieldLengthInspector { std::vector<vespalib::string> fields; std::map<vespalib::string, FieldLengthInfo> field_lengths; + MySetup(); + ~MySetup() override; MySetup &field(const std::string &name) { fields.emplace_back(name); return *this; @@ -84,6 +86,9 @@ struct MySetup : public IFieldLengthInspector { }; +MySetup::MySetup() = default; +MySetup::~MySetup() = default; + //----------------------------------------------------------------------------- struct Index { diff --git a/searchlib/src/tests/queryeval/nearest_neighbor/nearest_neighbor_test.cpp b/searchlib/src/tests/queryeval/nearest_neighbor/nearest_neighbor_test.cpp index 03cf22af2de..2801bf90080 100644 --- a/searchlib/src/tests/queryeval/nearest_neighbor/nearest_neighbor_test.cpp +++ b/searchlib/src/tests/queryeval/nearest_neighbor/nearest_neighbor_test.cpp @@ -188,18 +188,18 @@ struct TestParam { vespalib::string attribute_tensor_type_spec; vespalib::string query_tensor_type_spec; TestParam(const vespalib::string& attribute_tensor_type_spec_in, - const vespalib::string& query_tensor_type_spec_in) + const vespalib::string& query_tensor_type_spec_in) noexcept : attribute_tensor_type_spec(attribute_tensor_type_spec_in), query_tensor_type_spec(query_tensor_type_spec_in) {} - TestParam(const TestParam &); + TestParam(const TestParam &) noexcept; TestParam & operator=(TestParam &) noexcept = delete; TestParam(TestParam &&) noexcept = default; TestParam & operator=(TestParam &&) noexcept = default; ~TestParam(); }; -TestParam::TestParam(const TestParam &) = default; +TestParam::TestParam(const TestParam &) noexcept = default; TestParam::~TestParam() = default; std::ostream& operator<<(std::ostream& os, const TestParam& param) diff --git a/searchlib/src/tests/queryeval/weak_and/wand_bench_setup.hpp b/searchlib/src/tests/queryeval/weak_and/wand_bench_setup.hpp index fd73243b4fa..0cd31cd491a 100644 --- a/searchlib/src/tests/queryeval/weak_and/wand_bench_setup.hpp +++ b/searchlib/src/tests/queryeval/weak_and/wand_bench_setup.hpp @@ -54,13 +54,13 @@ struct ModSearch : SearchIterator { uint32_t limit; MinMaxPostingInfo info; TermFieldMatchData *tfmd; - ModSearch(Stats &stats_in, uint32_t step_in, uint32_t limit_in, int32_t maxWeight, TermFieldMatchData *tfmd_in) - : stats(stats_in), step(step_in), limit(limit_in), info(0, maxWeight), tfmd(tfmd_in) { } + ModSearch(Stats &stats_in, uint32_t step_in, uint32_t limit_in, int32_t maxWeight, TermFieldMatchData *tfmd_in); + ~ModSearch() override; void initRange(uint32_t begin, uint32_t end) override { SearchIterator::initRange(begin, end); setDocId(step); } - virtual void doSeek(uint32_t docid) override { + void doSeek(uint32_t docid) override { assert(docid > getDocId()); uint32_t skippedDocs = (docid - getDocId() - 1); uint32_t skippedHits = (skippedDocs / step); @@ -76,7 +76,7 @@ struct ModSearch : SearchIterator { setAtEnd(); } } - virtual void doUnpack(uint32_t docid) override { + void doUnpack(uint32_t docid) override { if (tfmd != NULL) { tfmd->reset(docid); search::fef::TermFieldMatchDataPosition pos; @@ -85,9 +85,14 @@ struct ModSearch : SearchIterator { } stats.unpack(); } - virtual const PostingInfo *getPostingInfo() const override { return &info; } + const PostingInfo *getPostingInfo() const override { return &info; } }; +ModSearch::ModSearch(Stats &stats_in, uint32_t step_in, uint32_t limit_in, int32_t maxWeight, TermFieldMatchData *tfmd_in) + : stats(stats_in), step(step_in), limit(limit_in), info(0, maxWeight), tfmd(tfmd_in) +{ } +ModSearch::~ModSearch() = default; + struct WandFactory { virtual std::string name() const = 0; virtual SearchIterator::UP create(const wand::Terms &terms) = 0; diff --git a/searchlib/src/tests/queryeval/weighted_set_term/weighted_set_term_test.cpp b/searchlib/src/tests/queryeval/weighted_set_term/weighted_set_term_test.cpp index 8f4381c48f9..51594c478e5 100644 --- a/searchlib/src/tests/queryeval/weighted_set_term/weighted_set_term_test.cpp +++ b/searchlib/src/tests/queryeval/weighted_set_term/weighted_set_term_test.cpp @@ -44,28 +44,20 @@ struct WS { bool field_is_filter; bool term_is_not_needed; - WS() - : layout(), - handle(layout.allocTermField(fieldId)), - tokens(), - field_is_filter(false), - term_is_not_needed(false) - { - MatchData::UP tmp = layout.createMatchData(); - ASSERT_TRUE(tmp->resolveTermField(handle)->getFieldId() == fieldId); - } + WS(); + ~WS(); WS &add(const std::string &token, uint32_t weight) { - tokens.push_back(std::make_pair(token, weight)); + tokens.emplace_back(token, weight); return *this; } WS& set_field_is_filter(bool value) { field_is_filter = value; return *this; } WS& set_term_is_not_needed(bool value) { term_is_not_needed = value; return *this; } - Node::UP createNode() const { - SimpleWeightedSetTerm *node = new SimpleWeightedSetTerm(tokens.size(), "view", 0, Weight(0)); - for (size_t i = 0; i < tokens.size(); ++i) { - node->addTerm(tokens[i].first,Weight(tokens[i].second)); + [[nodiscard]] Node::UP createNode() const { + auto *node = new SimpleWeightedSetTerm(tokens.size(), "view", 0, Weight(0)); + for (const auto & token : tokens) { + node->addTerm(token.first,Weight(token.second)); } return Node::UP(node); } @@ -78,7 +70,7 @@ struct WS { queryeval::Blueprint::UP bp = searchable.createBlueprint(requestContext, fields, *node); bp->fetchPostings(ExecuteInfo::create(strict)); SearchIterator::UP sb = bp->createSearch(*md, strict); - return (dynamic_cast<WeightedSetTermSearch*>(sb.get()) != 0); + return (dynamic_cast<WeightedSetTermSearch*>(sb.get()) != nullptr); } FakeResult search(Searchable &searchable, const std::string &field, bool strict) const { @@ -111,26 +103,38 @@ struct WS { } }; +WS::WS() + : layout(), + handle(layout.allocTermField(fieldId)), + tokens(), + field_is_filter(false), + term_is_not_needed(false) +{ + MatchData::UP tmp = layout.createMatchData(); + ASSERT_TRUE(tmp->resolveTermField(handle)->getFieldId() == fieldId); +} +WS::~WS() = default; + struct MockSearch : public SearchIterator { int seekCnt; int _initial; - MockSearch(uint32_t initial) : SearchIterator(), seekCnt(0), _initial(initial) { } + explicit MockSearch(uint32_t initial) : SearchIterator(), seekCnt(0), _initial(initial) { } void initRange(uint32_t begin, uint32_t end) override { SearchIterator::initRange(begin, end); setDocId(_initial); } - virtual void doSeek(uint32_t) override { + void doSeek(uint32_t) override { ++seekCnt; setAtEnd(); } - virtual void doUnpack(uint32_t) override {} + void doUnpack(uint32_t) override {} }; struct MockFixture { MockSearch *mock; TermFieldMatchData tfmd; std::unique_ptr<SearchIterator> search; - MockFixture(uint32_t initial) : mock(0), tfmd(), search() { + explicit MockFixture(uint32_t initial) : mock(nullptr), tfmd(), search() { std::vector<SearchIterator*> children; std::vector<int32_t> weights; mock = new MockSearch(initial); @@ -156,10 +160,11 @@ void run_simple(bool field_is_filter, bool term_is_not_needed) .doc(5).elem(0).weight(50).pos(0) .doc(7).elem(0).weight(70).pos(0); } - WS ws = WS().add("7", 70).add("5", 50).add("3", 30).add("100", 1000) - .set_field_is_filter(field_is_filter) - .set_term_is_not_needed(term_is_not_needed); -; + WS ws; + ws.add("7", 70).add("5", 50).add("3", 30).add("100", 1000) + .set_field_is_filter(field_is_filter) + .set_term_is_not_needed(term_is_not_needed); + EXPECT_TRUE(ws.isGenericSearch(index, "field", true)); EXPECT_TRUE(ws.isGenericSearch(index, "field", false)); EXPECT_TRUE(ws.isGenericSearch(index, "multi-field", true)); @@ -197,11 +202,12 @@ void run_multi(bool field_is_filter, bool term_is_not_needed) .doc(5).elem(0).weight(150).pos(0).elem(0).weight(50).pos(0) .doc(7).elem(0).weight(70).pos(0); } - WS ws = WS().add("7", 70).add("5", 50).add("3", 30) - .add("15", 150).add("13", 130) - .add("23", 230).add("100", 1000) - .set_field_is_filter(field_is_filter) - .set_term_is_not_needed(term_is_not_needed); + WS ws; + ws.add("7", 70).add("5", 50).add("3", 30) + .add("15", 150).add("13", 130) + .add("23", 230).add("100", 1000) + .set_field_is_filter(field_is_filter) + .set_term_is_not_needed(term_is_not_needed); EXPECT_TRUE(ws.isGenericSearch(index, "multi-field", true)); EXPECT_TRUE(ws.isGenericSearch(index, "multi-field", false)); @@ -274,9 +280,9 @@ TEST("verify search iterator conformance with document weight iterator children" struct VerifyMatchData { struct MyBlueprint : search::queryeval::SimpleLeafBlueprint { VerifyMatchData &vmd; - MyBlueprint(VerifyMatchData &vmd_in, FieldSpec spec_in) + MyBlueprint(VerifyMatchData &vmd_in, const FieldSpec & spec_in) : SimpleLeafBlueprint(spec_in), vmd(vmd_in) {} - SearchIterator::UP createLeafSearch(const fef::TermFieldMatchDataArray &tfmda, bool) const override { + [[nodiscard]] SearchIterator::UP createLeafSearch(const fef::TermFieldMatchDataArray &tfmda, bool) const override { EXPECT_EQUAL(tfmda.size(), 1u); EXPECT_TRUE(tfmda[0] != nullptr); if (vmd.child_tfmd == nullptr) { @@ -287,7 +293,7 @@ struct VerifyMatchData { ++vmd.child_cnt; return std::make_unique<EmptySearch>(); } - SearchIteratorUP createFilterSearch(bool strict, FilterConstraint constraint) const override { + [[nodiscard]] SearchIteratorUP createFilterSearch(bool strict, FilterConstraint constraint) const override { return create_default_filter(strict, constraint); } }; diff --git a/searchlib/src/tests/sort/sortbenchmark.cpp b/searchlib/src/tests/sort/sortbenchmark.cpp index 04832593b28..94f9424575c 100644 --- a/searchlib/src/tests/sort/sortbenchmark.cpp +++ b/searchlib/src/tests/sort/sortbenchmark.cpp @@ -18,9 +18,15 @@ public: V merge(); void twoWayMerge(); V cat() const; + Test(); + ~Test() override; }; -void Test::generateVectors(size_t numVectors, size_t values) +Test::Test() = default; +Test::~Test() = default; + +void +Test::generateVectors(size_t numVectors, size_t values) { _data.resize(numVectors); for (size_t j(0); j < numVectors; j++) { @@ -32,13 +38,15 @@ void Test::generateVectors(size_t numVectors, size_t values) } } -Test::V Test::merge() +Test::V +Test::merge() { twoWayMerge(); return _data[0]; } -void Test::twoWayMerge() +void +Test::twoWayMerge() { std::vector<V> n((_data.size()+1)/2); @@ -57,7 +65,8 @@ void Test::twoWayMerge() } } -Test::V Test::cat() const +Test::V +Test::cat() const { size_t sum(0); for (size_t i(0), m(_data.size()); i < m; i++) { diff --git a/searchlib/src/tests/tensor/hnsw_index/hnsw_index_test.cpp b/searchlib/src/tests/tensor/hnsw_index/hnsw_index_test.cpp index dd248b07fff..a3f07959094 100644 --- a/searchlib/src/tests/tensor/hnsw_index/hnsw_index_test.cpp +++ b/searchlib/src/tests/tensor/hnsw_index/hnsw_index_test.cpp @@ -768,7 +768,7 @@ class MyGlobalFilter : public GlobalFilter { std::shared_ptr<GlobalFilter> _filter; mutable uint32_t _max_docid; public: - MyGlobalFilter(std::shared_ptr<GlobalFilter> filter) + MyGlobalFilter(std::shared_ptr<GlobalFilter> filter) noexcept : _filter(std::move(filter)), _max_docid(0) { diff --git a/searchlib/src/vespa/searchlib/aggregation/hitsaggregationresult.cpp b/searchlib/src/vespa/searchlib/aggregation/hitsaggregationresult.cpp index b3dff78abfb..01559952f28 100644 --- a/searchlib/src/vespa/searchlib/aggregation/hitsaggregationresult.cpp +++ b/searchlib/src/vespa/searchlib/aggregation/hitsaggregationresult.cpp @@ -2,7 +2,6 @@ #include "hitsaggregationresult.h" #include <vespa/document/fieldvalue/document.h> -#include <cassert> #include <vespa/log/log.h> LOG_SETUP(".searchlib.aggregation.hitsaggregationresult"); @@ -21,9 +20,13 @@ HitsAggregationResult::HitsAggregationResult() : _hits(), _isOrdered(false), _bestHitRank(), - _summaryGenerator(0) + _summaryGenerator(nullptr) {} -HitsAggregationResult::~HitsAggregationResult() {} +HitsAggregationResult::HitsAggregationResult(HitsAggregationResult &&) noexcept = default; +HitsAggregationResult & HitsAggregationResult::operator=(HitsAggregationResult &&) noexcept = default; +HitsAggregationResult::HitsAggregationResult(const HitsAggregationResult &) = default; +HitsAggregationResult & HitsAggregationResult::operator=(const HitsAggregationResult &) = default; +HitsAggregationResult::~HitsAggregationResult() = default; void HitsAggregationResult::onPrepare(const ResultNode & result, bool useForInit) { @@ -34,7 +37,7 @@ void HitsAggregationResult::onPrepare(const ResultNode & result, bool useForInit void HitsAggregationResult::onMerge(const AggregationResult &b) { - const HitsAggregationResult &rhs = (const HitsAggregationResult &)b; + const auto &rhs = (const HitsAggregationResult &)b; _hits.onMerge(rhs._hits); } diff --git a/searchlib/src/vespa/searchlib/aggregation/hitsaggregationresult.h b/searchlib/src/vespa/searchlib/aggregation/hitsaggregationresult.h index 97595a4119c..30059078920 100644 --- a/searchlib/src/vespa/searchlib/aggregation/hitsaggregationresult.h +++ b/searchlib/src/vespa/searchlib/aggregation/hitsaggregationresult.h @@ -43,7 +43,11 @@ public: DECLARE_AGGREGATIONRESULT(HitsAggregationResult); HitsAggregationResult(); - ~HitsAggregationResult(); + HitsAggregationResult(HitsAggregationResult &&) noexcept; + HitsAggregationResult & operator=(HitsAggregationResult &&) noexcept; + HitsAggregationResult(const HitsAggregationResult &); + HitsAggregationResult & operator=(const HitsAggregationResult &); + ~HitsAggregationResult() override; void postMerge() override { _hits.postMerge(_maxHits); } void setSummaryGenerator(SummaryGenerator & summaryGenerator) { _summaryGenerator = &summaryGenerator; } const SummaryClassType & getSummaryClass() const { return _summaryClass; } diff --git a/searchlib/src/vespa/searchlib/attribute/multienumattribute.cpp b/searchlib/src/vespa/searchlib/attribute/multienumattribute.cpp index 817470deb69..a8006d4e3b5 100644 --- a/searchlib/src/vespa/searchlib/attribute/multienumattribute.cpp +++ b/searchlib/src/vespa/searchlib/attribute/multienumattribute.cpp @@ -2,6 +2,7 @@ #include "multienumattribute.hpp" #include "enummodifier.h" +#include <vespa/vespalib/datastore/unique_store_remapper.hpp> #include <stdexcept> namespace search::multienumattribute { diff --git a/searchlib/src/vespa/searchlib/attribute/search_context.cpp b/searchlib/src/vespa/searchlib/attribute/search_context.cpp index 3deacca5764..a0345ddce70 100644 --- a/searchlib/src/vespa/searchlib/attribute/search_context.cpp +++ b/searchlib/src/vespa/searchlib/attribute/search_context.cpp @@ -10,7 +10,7 @@ using search::queryeval::SearchIterator; namespace search::attribute { -SearchContext::SearchContext(const AttributeVector &attr) +SearchContext::SearchContext(const AttributeVector &attr) noexcept : _attr(attr), _plsc(nullptr) { diff --git a/searchlib/src/vespa/searchlib/attribute/search_context.h b/searchlib/src/vespa/searchlib/attribute/search_context.h index 8e1297ad2df..025b0fdf113 100644 --- a/searchlib/src/vespa/searchlib/attribute/search_context.h +++ b/searchlib/src/vespa/searchlib/attribute/search_context.h @@ -47,7 +47,7 @@ public: const AttributeVector& attribute() const { return _attr; } protected: - SearchContext(const AttributeVector& attr); + SearchContext(const AttributeVector& attr) noexcept; const AttributeVector& _attr; attribute::IPostingListSearchContext* _plsc; diff --git a/searchlib/src/vespa/searchlib/attribute/string_search_context.cpp b/searchlib/src/vespa/searchlib/attribute/string_search_context.cpp index e548ab8078c..fadf7a3151d 100644 --- a/searchlib/src/vespa/searchlib/attribute/string_search_context.cpp +++ b/searchlib/src/vespa/searchlib/attribute/string_search_context.cpp @@ -21,6 +21,9 @@ StringSearchContext::StringSearchContext(const AttributeVector& to_be_searched, { } +StringSearchContext::StringSearchContext(StringSearchContext &&) noexcept = default; +StringSearchContext::~StringSearchContext() = default; + const QueryTermUCS4* StringSearchContext::queryTerm() const { diff --git a/searchlib/src/vespa/searchlib/attribute/string_search_context.h b/searchlib/src/vespa/searchlib/attribute/string_search_context.h index fc9f3688a7a..a0014379436 100644 --- a/searchlib/src/vespa/searchlib/attribute/string_search_context.h +++ b/searchlib/src/vespa/searchlib/attribute/string_search_context.h @@ -26,6 +26,8 @@ protected: public: StringSearchContext(const AttributeVector& to_be_searched, std::unique_ptr<QueryTermSimple> query_term, bool cased); StringSearchContext(const AttributeVector& to_be_searched, StringMatcher&& matcher); + StringSearchContext(StringSearchContext &&) noexcept; + ~StringSearchContext() override; const QueryTermUCS4* queryTerm() const override; bool valid() const override; diff --git a/searchlib/src/vespa/searchlib/common/converters.h b/searchlib/src/vespa/searchlib/common/converters.h index 718f575b4e8..2c076fb5eed 100644 --- a/searchlib/src/vespa/searchlib/common/converters.h +++ b/searchlib/src/vespa/searchlib/common/converters.h @@ -16,7 +16,7 @@ private: class LowercaseConverter : public BlobConverter { public: - LowercaseConverter(); + LowercaseConverter() noexcept; private: ConstBufferRef onConvert(const ConstBufferRef & src) const override; mutable vespalib::string _buffer; diff --git a/searchlib/src/vespa/searchlib/common/sortspec.cpp b/searchlib/src/vespa/searchlib/common/sortspec.cpp index 16f0c884535..00de3111ac9 100644 --- a/searchlib/src/vespa/searchlib/common/sortspec.cpp +++ b/searchlib/src/vespa/searchlib/common/sortspec.cpp @@ -16,8 +16,8 @@ PassThroughConverter::onConvert(const ConstBufferRef & src) const return src; } -LowercaseConverter::LowercaseConverter() : - _buffer() +LowercaseConverter::LowercaseConverter() noexcept + : _buffer() { } diff --git a/searchlib/src/vespa/searchlib/diskindex/indexbuilder.cpp b/searchlib/src/vespa/searchlib/diskindex/indexbuilder.cpp index f552c35e941..542b6f5f6eb 100644 --- a/searchlib/src/vespa/searchlib/diskindex/indexbuilder.cpp +++ b/searchlib/src/vespa/searchlib/diskindex/indexbuilder.cpp @@ -60,7 +60,7 @@ private: const uint32_t _fieldId; const bool _valid; public: - FieldHandle(const Schema &schema, uint32_t fieldId, IndexBuilder & builder, bool valid); + FieldHandle(const Schema &schema, uint32_t fieldId, IndexBuilder & builder, bool valid) noexcept; ~FieldHandle(); void new_word(vespalib::stringref word); @@ -126,7 +126,7 @@ FileHandle::close() (void) ret; } -IndexBuilder::FieldHandle::FieldHandle(const Schema &schema, uint32_t fieldId, IndexBuilder &builder, bool valid) +IndexBuilder::FieldHandle::FieldHandle(const Schema &schema, uint32_t fieldId, IndexBuilder &builder, bool valid) noexcept : _schema(schema), _builder(builder), _file(), diff --git a/searchlib/src/vespa/searchlib/expression/functionnodes.cpp b/searchlib/src/vespa/searchlib/expression/functionnodes.cpp index 9046a5427ce..cc6a8c137a7 100644 --- a/searchlib/src/vespa/searchlib/expression/functionnodes.cpp +++ b/searchlib/src/vespa/searchlib/expression/functionnodes.cpp @@ -3,9 +3,7 @@ #include "floatresultnode.h" #include "stringresultnode.h" #include "rawresultnode.h" -#include "enumresultnode.h" #include "constantnode.h" -#include "relevancenode.h" #include "addfunctionnode.h" #include "dividefunctionnode.h" #include "multiplyfunctionnode.h" @@ -29,8 +27,6 @@ #include "xorbitfunctionnode.h" #include "md5bitfunctionnode.h" #include "binaryfunctionnode.h" -#include "nullresultnode.h" -#include "positiveinfinityresultnode.h" #include "resultvector.h" #include "catserializer.h" #include "strcatserializer.h" @@ -320,6 +316,8 @@ ResultNode::CP DivideFunctionNode::getInitialValue() const throw std::runtime_error("DivideFunctionNode::getInitialValue() const not implemented since it shall never be used."); } +UnaryBitFunctionNode::~UnaryBitFunctionNode() = default; + void UnaryBitFunctionNode::onPrepareResult() { setResultType(std::unique_ptr<ResultNode>(new RawResultNode())); diff --git a/searchlib/src/vespa/searchlib/expression/unarybitfunctionnode.h b/searchlib/src/vespa/searchlib/expression/unarybitfunctionnode.h index 63114188b1f..536d881cb90 100644 --- a/searchlib/src/vespa/searchlib/expression/unarybitfunctionnode.h +++ b/searchlib/src/vespa/searchlib/expression/unarybitfunctionnode.h @@ -4,8 +4,7 @@ #include "unaryfunctionnode.h" #include <vespa/vespalib/objects/nbostream.h> -namespace search { -namespace expression { +namespace search::expression { class UnaryBitFunctionNode : public UnaryFunctionNode { @@ -15,6 +14,7 @@ public: DECLARE_ABSTRACT_EXPRESSIONNODE(UnaryBitFunctionNode); UnaryBitFunctionNode() : _numBits(0) { } UnaryBitFunctionNode(ExpressionNode::UP arg, unsigned numBits) : UnaryFunctionNode(std::move(arg)), _numBits(numBits) { } + ~UnaryBitFunctionNode() override; protected: size_t getNumBits() const { return _numBits; } size_t getNumBytes() const { return (_numBits+7)/8; } @@ -28,5 +28,3 @@ private: }; } -} - diff --git a/searchlib/src/vespa/searchlib/features/element_similarity_feature.cpp b/searchlib/src/vespa/searchlib/features/element_similarity_feature.cpp index a58f97a2755..d039b80930b 100644 --- a/searchlib/src/vespa/searchlib/features/element_similarity_feature.cpp +++ b/searchlib/src/vespa/searchlib/features/element_similarity_feature.cpp @@ -236,15 +236,8 @@ private: const fef::MatchData *_md; public: - ElementSimilarityExecutor(VectorizedQueryTerms &&terms, std::vector<OutputSpec> &&outputs_in) - : _terms(std::move(terms)), - _pos(_terms.handles.size(), nullptr), - _end(_terms.handles.size(), nullptr), - _position_queue(CmpPosition(_pos.data())), - _element_queue(CmpElement(_pos.data())), - _outputs(std::move(outputs_in)), - _md(nullptr) - { } + ElementSimilarityExecutor(VectorizedQueryTerms &&terms, std::vector<OutputSpec> &&outputs_in); + ~ElementSimilarityExecutor() override; bool isPure() override { return _terms.handles.empty(); } @@ -317,6 +310,17 @@ public: } }; +ElementSimilarityExecutor::ElementSimilarityExecutor(VectorizedQueryTerms &&terms, std::vector<OutputSpec> &&outputs_in) + : _terms(std::move(terms)), + _pos(_terms.handles.size(), nullptr), + _end(_terms.handles.size(), nullptr), + _position_queue(CmpPosition(_pos.data())), + _element_queue(CmpElement(_pos.data())), + _outputs(std::move(outputs_in)), + _md(nullptr) +{ } +ElementSimilarityExecutor::~ElementSimilarityExecutor() = default; + //----------------------------------------------------------------------------- std::vector<std::pair<vespalib::string, vespalib::string> > diff --git a/searchlib/src/vespa/searchlib/fef/blueprintresolver.cpp b/searchlib/src/vespa/searchlib/fef/blueprintresolver.cpp index dd94e26b056..cc2cae1c8cb 100644 --- a/searchlib/src/vespa/searchlib/fef/blueprintresolver.cpp +++ b/searchlib/src/vespa/searchlib/fef/blueprintresolver.cpp @@ -51,7 +51,7 @@ struct Compiler : public Blueprint::DependencyHandler { struct Frame { ExecutorSpec spec; const FeatureNameParser &parser; - Frame(Blueprint::SP blueprint, const FeatureNameParser &parser_in) + Frame(Blueprint::SP blueprint, const FeatureNameParser &parser_in) noexcept : spec(std::move(blueprint)), parser(parser_in) {} }; using Stack = std::vector<Frame>; @@ -91,7 +91,7 @@ struct Compiler : public Blueprint::DependencyHandler { failed_set(), min_stack(nullptr), max_stack(nullptr) {} - ~Compiler(); + ~Compiler() override; void probe_stack() { const char c = 'X'; @@ -99,12 +99,12 @@ struct Compiler : public Blueprint::DependencyHandler { max_stack = (max_stack == nullptr) ? &c : std::max(max_stack, &c); } - int stack_usage() const { + [[nodiscard]] int stack_usage() const { return (max_stack - min_stack); } Frame &self() { return resolve_stack.back(); } - bool failed() const { return !failed_set.empty(); } + [[nodiscard]] bool failed() const { return !failed_set.empty(); } vespalib::string make_trace(bool skip_self) { vespalib::string trace; @@ -142,7 +142,7 @@ struct Compiler : public Blueprint::DependencyHandler { errors.emplace_back(msg); } probe_stack(); - return FeatureRef(); + return {}; } void fail_self(const vespalib::string &reason) { @@ -245,12 +245,14 @@ Compiler::~Compiler() = default; } // namespace search::fef::<unnamed> -BlueprintResolver::ExecutorSpec::ExecutorSpec(Blueprint::SP blueprint_in) +BlueprintResolver::ExecutorSpec::ExecutorSpec(Blueprint::SP blueprint_in) noexcept : blueprint(std::move(blueprint_in)), inputs(), output_types() { } - +BlueprintResolver::ExecutorSpec::ExecutorSpec(ExecutorSpec &&) noexcept = default; +BlueprintResolver::ExecutorSpec & BlueprintResolver::ExecutorSpec::operator =(ExecutorSpec &&) noexcept = default; +BlueprintResolver::ExecutorSpec::ExecutorSpec(const ExecutorSpec &) = default; BlueprintResolver::ExecutorSpec::~ExecutorSpec() = default; BlueprintResolver::~BlueprintResolver() = default; diff --git a/searchlib/src/vespa/searchlib/fef/blueprintresolver.h b/searchlib/src/vespa/searchlib/fef/blueprintresolver.h index cbc8614dbde..3cf52a90684 100644 --- a/searchlib/src/vespa/searchlib/fef/blueprintresolver.h +++ b/searchlib/src/vespa/searchlib/fef/blueprintresolver.h @@ -43,7 +43,7 @@ public: FeatureRef() : executor(undef), output(0) {} FeatureRef(uint32_t executor_in, uint32_t output_in) : executor(executor_in), output(output_in) {} - bool valid() { return (executor != undef); } + [[nodiscard]] bool valid() const { return (executor != undef); } }; using FeatureMap = std::map<vespalib::string, FeatureRef>; @@ -57,7 +57,10 @@ public: std::vector<FeatureRef> inputs; std::vector<FeatureType> output_types; - ExecutorSpec(BlueprintSP blueprint_in); + explicit ExecutorSpec(BlueprintSP blueprint_in) noexcept; + ExecutorSpec(ExecutorSpec &&) noexcept; + ExecutorSpec & operator =(ExecutorSpec &&) noexcept; + ExecutorSpec(const ExecutorSpec &); ~ExecutorSpec(); }; using ExecutorSpecList = std::vector<ExecutorSpec>; @@ -136,7 +139,7 @@ public: * * @return feature executor assembly directions **/ - const ExecutorSpecList &getExecutorSpecs() const { return _executorSpecs; } + [[nodiscard]] const ExecutorSpecList &getExecutorSpecs() const { return _executorSpecs; } /** * Obtain the location of all named features known to this @@ -147,7 +150,7 @@ public: * * @return feature locations **/ - const FeatureMap &getFeatureMap() const { return _featureMap; } + [[nodiscard]] const FeatureMap &getFeatureMap() const { return _featureMap; } /** * Obtain the location of all seeds used by this resolver. This @@ -158,13 +161,13 @@ public: * * @return seed locations **/ - const FeatureMap &getSeedMap() const { return _seedMap; } + [[nodiscard]] const FeatureMap &getSeedMap() const { return _seedMap; } /** * Will return any accumulated warnings during compile * @return list of warnings **/ - const Warnings & getWarnings() const { return _warnings; } + [[nodiscard]] const Warnings & getWarnings() const { return _warnings; } }; } diff --git a/searchlib/src/vespa/searchlib/fef/feature_type.h b/searchlib/src/vespa/searchlib/fef/feature_type.h index 643d0b9ae0f..dea1920e840 100644 --- a/searchlib/src/vespa/searchlib/fef/feature_type.h +++ b/searchlib/src/vespa/searchlib/fef/feature_type.h @@ -23,12 +23,12 @@ private: using TYPE_UP = std::unique_ptr<TYPE>; TYPE_UP _type; static const FeatureType _number; - FeatureType(TYPE_UP type_in) : _type(std::move(type_in)) {} + explicit FeatureType(TYPE_UP type_in) : _type(std::move(type_in)) {} public: FeatureType(const FeatureType &rhs); - FeatureType(FeatureType &&rhs) = default; - bool is_object() const { return (_type.get() != nullptr); } - const TYPE &type() const; + FeatureType(FeatureType &&rhs) noexcept = default; + [[nodiscard]] bool is_object() const { return (_type.get() != nullptr); } + [[nodiscard]] const TYPE &type() const; static const FeatureType &number() { return _number; } static FeatureType object(const TYPE &type_in); }; diff --git a/searchlib/src/vespa/searchlib/fef/phrase_splitter_query_env.cpp b/searchlib/src/vespa/searchlib/fef/phrase_splitter_query_env.cpp index 02b1e3616a0..dfa1d9886f7 100644 --- a/searchlib/src/vespa/searchlib/fef/phrase_splitter_query_env.cpp +++ b/searchlib/src/vespa/searchlib/fef/phrase_splitter_query_env.cpp @@ -21,7 +21,7 @@ PhraseSplitterQueryEnv::considerTerm(uint32_t termIdx, const ITermData &term, ui prototype.addField(fieldId); _phrase_terms.push_back(PhraseTerm(term, _terms.size(), h)); for (uint32_t i = 0; i < term.getPhraseLength(); ++i) { - _terms.push_back(prototype); + _terms.emplace_back(prototype); _termIdxMap.push_back(TermIdx(_terms.size() - 1, true)); } return; diff --git a/searchlib/src/vespa/searchlib/fef/simpletermdata.cpp b/searchlib/src/vespa/searchlib/fef/simpletermdata.cpp index fdea499f9b4..92245ef21b3 100644 --- a/searchlib/src/vespa/searchlib/fef/simpletermdata.cpp +++ b/searchlib/src/vespa/searchlib/fef/simpletermdata.cpp @@ -4,10 +4,9 @@ namespace search::fef { -SimpleTermData::SimpleTermData() +SimpleTermData::SimpleTermData() noexcept : _weight(0), _numTerms(0), - _termIndex(0), _uniqueId(0), _query_tensor_name(), _fields() @@ -21,11 +20,16 @@ SimpleTermData::SimpleTermData(const ITermData &rhs) _query_tensor_name(rhs.query_tensor_name()), _fields() { + _fields.reserve(rhs.numFields()); for (size_t i(0), m(rhs.numFields()); i < m; ++i) { - _fields.push_back(SimpleTermFieldData(rhs.field(i))); + _fields.emplace_back(rhs.field(i)); } } +SimpleTermData::SimpleTermData(const SimpleTermData &) = default; +SimpleTermData::SimpleTermData(SimpleTermData &&) noexcept = default; +SimpleTermData & SimpleTermData::operator=(SimpleTermData &&) noexcept = default; + SimpleTermData::~SimpleTermData() = default; } diff --git a/searchlib/src/vespa/searchlib/fef/simpletermdata.h b/searchlib/src/vespa/searchlib/fef/simpletermdata.h index c59816dc687..d501d0848e8 100644 --- a/searchlib/src/vespa/searchlib/fef/simpletermdata.h +++ b/searchlib/src/vespa/searchlib/fef/simpletermdata.h @@ -9,8 +9,7 @@ #include <vector> #include <cassert> -namespace search { -namespace fef { +namespace search::fef { /** * Static match data for a single unit (term/phrase/etc). @@ -20,49 +19,42 @@ class SimpleTermData final : public ITermData private: query::Weight _weight; uint32_t _numTerms; - uint32_t _termIndex; uint32_t _uniqueId; - std::optional<vespalib::string> _query_tensor_name; - + std::optional<vespalib::string> _query_tensor_name; std::vector<SimpleTermFieldData> _fields; public: - /** - * Creates a new object. - **/ - SimpleTermData(); - - /** - * Side-cast copy constructor. - **/ + SimpleTermData() noexcept; SimpleTermData(const ITermData &rhs); - - ~SimpleTermData(); + SimpleTermData(const SimpleTermData &); + SimpleTermData(SimpleTermData &&) noexcept; + SimpleTermData & operator=(SimpleTermData &&) noexcept; + ~SimpleTermData() override; //----------- ITermData implementation ------------------------------------ - query::Weight getWeight() const override { return _weight; } + [[nodiscard]] query::Weight getWeight() const override { return _weight; } - uint32_t getPhraseLength() const override { return _numTerms; } + [[nodiscard]] uint32_t getPhraseLength() const override { return _numTerms; } - uint32_t getUniqueId() const override { return _uniqueId; } + [[nodiscard]] uint32_t getUniqueId() const override { return _uniqueId; } - std::optional<vespalib::string> query_tensor_name() const override { return _query_tensor_name; } + [[nodiscard]] std::optional<vespalib::string> query_tensor_name() const override { return _query_tensor_name; } - size_t numFields() const override { return _fields.size(); } + [[nodiscard]] size_t numFields() const override { return _fields.size(); } - const ITermFieldData &field(size_t i) const override { + [[nodiscard]] const ITermFieldData &field(size_t i) const override { return _fields[i]; } - const ITermFieldData *lookupField(uint32_t fieldId) const override { + [[nodiscard]] const ITermFieldData *lookupField(uint32_t fieldId) const override { for (size_t fieldIdx(0), m(numFields()); fieldIdx < m; ++fieldIdx) { const ITermFieldData &tfd = field(fieldIdx); if (tfd.getFieldId() == fieldId) { return &tfd; } } - return 0; + return nullptr; } //----------- Utility functions ------------------------------------------- @@ -84,17 +76,6 @@ public: } /** - * Set the location of this term in the original user query. - * - * @return this to allow chaining. - * @param idx term index - **/ - SimpleTermData &setTermIndex(uint32_t idx) { - _termIndex = idx; - return *this; - } - - /** * Set the unique id of this term. 0 means not set. * * @param id unique id or 0 @@ -117,7 +98,7 @@ public: * @param fieldId field id of the added field **/ SimpleTermFieldData &addField(uint32_t fieldId) { - _fields.push_back(SimpleTermFieldData(fieldId)); + _fields.emplace_back(fieldId); return _fields.back(); } @@ -143,11 +124,10 @@ public: return &tfd; } } - return 0; + return nullptr; } }; - /** * convenience adapter for easy iteration **/ @@ -161,14 +141,11 @@ public: : _ref(ref), _idx(0), _lim(ref.numFields()) {} - bool valid() const { return (_idx < _lim); } + [[nodiscard]] bool valid() const { return (_idx < _lim); } - SimpleTermFieldData& get() const { return _ref.field(_idx); } + [[nodiscard]] SimpleTermFieldData& get() const { return _ref.field(_idx); } void next() { assert(valid()); ++_idx; } }; - -} // namespace fef -} // namespace search - +} diff --git a/searchlib/src/vespa/searchlib/grouping/sketch.h b/searchlib/src/vespa/searchlib/grouping/sketch.h index b973a99da92..6a2082b793d 100644 --- a/searchlib/src/vespa/searchlib/grouping/sketch.h +++ b/searchlib/src/vespa/searchlib/grouping/sketch.h @@ -74,6 +74,7 @@ struct SparseSketch : Sketch<BucketBits, HashT> { }; std::unordered_set<HashT, IdentityHash> hash_set; + ~SparseSketch() override; [[nodiscard]] size_t getSize() const { return hash_set.size(); } int aggregate(HashT hash) override { @@ -184,6 +185,8 @@ struct NormalSketch : Sketch<BucketBits, HashT> { } }; +template <int BucketBits, typename HashT> +SparseSketch<BucketBits, HashT>::~SparseSketch() = default; template <int BucketBits, typename HashT> void SparseSketch<BucketBits, HashT>:: diff --git a/searchlib/src/vespa/searchlib/memoryindex/field_inverter.cpp b/searchlib/src/vespa/searchlib/memoryindex/field_inverter.cpp index a05b39a74a9..25aff06b5ef 100644 --- a/searchlib/src/vespa/searchlib/memoryindex/field_inverter.cpp +++ b/searchlib/src/vespa/searchlib/memoryindex/field_inverter.cpp @@ -18,7 +18,9 @@ #include <vespa/searchlib/util/url.h> #include <vespa/vespalib/datastore/aligner.h> #include <vespa/vespalib/text/utf8.h> +#include <vespa/vespalib/util/exceptions.h> #include <vespa/vespalib/util/stringfmt.h> +#include <vespa/vespalib/stllike/asciistream.h> #include <vespa/vespalib/stllike/hash_map.hpp> #include <stdexcept> @@ -519,7 +521,7 @@ FieldInverter::applyRemoves() } void -FieldInverter::pushDocuments() +FieldInverter::push_documents_internal() { trimAbortedDocs(); @@ -605,5 +607,18 @@ FieldInverter::pushDocuments() reset(); } +void +FieldInverter::pushDocuments() +{ + try { + push_documents_internal(); + } catch (vespalib::OverflowException &e) { + const Schema::IndexField &field = _schema.getIndexField(_fieldId); + vespalib::asciistream s; + s << "FieldInverter::pushDocuments(), caught exception for field " << field.getName(); + throw vespalib::OverflowException(s.c_str(), e); + } +} + } diff --git a/searchlib/src/vespa/searchlib/memoryindex/field_inverter.h b/searchlib/src/vespa/searchlib/memoryindex/field_inverter.h index 7995dc56de8..1a582bf8099 100644 --- a/searchlib/src/vespa/searchlib/memoryindex/field_inverter.h +++ b/searchlib/src/vespa/searchlib/memoryindex/field_inverter.h @@ -248,6 +248,8 @@ public: VESPA_DLL_LOCAL void processAnnotations(const document::StringFieldValue &value); + void push_documents_internal(); + private: void processNormalDocTextField(const document::StringFieldValue &field); void processNormalDocArrayTextField(const document::ArrayFieldValue &field); diff --git a/searchlib/src/vespa/searchlib/query/tree/termnodes.cpp b/searchlib/src/vespa/searchlib/query/tree/termnodes.cpp index 6e889e76f21..00f17f7963c 100644 --- a/searchlib/src/vespa/searchlib/query/tree/termnodes.cpp +++ b/searchlib/src/vespa/searchlib/query/tree/termnodes.cpp @@ -30,7 +30,8 @@ namespace { class StringTermVector final : public MultiTerm::TermVector { public: - StringTermVector(uint32_t sz) : _terms() { _terms.reserve(sz); } + explicit StringTermVector(uint32_t sz) : _terms() { _terms.reserve(sz); } + ~StringTermVector() override; void addTerm(stringref term, Weight weight) override { _terms.emplace_back(term, weight); } @@ -39,27 +40,27 @@ public: auto res = std::to_chars(buf, buf + sizeof(buf), value, 10); addTerm(stringref(buf, res.ptr - buf), weight); } - StringAndWeight getAsString(uint32_t index) const override { + [[nodiscard]] StringAndWeight getAsString(uint32_t index) const override { const auto & v = _terms[index]; - return StringAndWeight(v.first, v.second); + return {v.first, v.second}; } - IntegerAndWeight getAsInteger(uint32_t index) const override { + [[nodiscard]] IntegerAndWeight getAsInteger(uint32_t index) const override { const auto & v = _terms[index]; int64_t value(0); std::from_chars(v.first.c_str(), v.first.c_str() + v.first.size(), value); - return IntegerAndWeight(value, v.second); + return {value, v.second}; } - Weight getWeight(uint32_t index) const override { + [[nodiscard]] Weight getWeight(uint32_t index) const override { return _terms[index].second; } - uint32_t size() const override { return _terms.size(); } + [[nodiscard]] uint32_t size() const override { return _terms.size(); } private: std::vector<std::pair<vespalib::string, Weight>> _terms; }; class IntegerTermVector final : public MultiTerm::TermVector { public: - IntegerTermVector(uint32_t sz) : _terms() { _terms.reserve(sz); } + explicit IntegerTermVector(uint32_t sz) : _terms() { _terms.reserve(sz); } void addTerm(stringref, Weight) override { // Will/should never happen assert(false); @@ -71,7 +72,7 @@ public: const auto & v = _terms[index]; auto res = std::to_chars(_scratchPad, _scratchPad + sizeof(_scratchPad)-1, v.first, 10); res.ptr[0] = '\0'; - return StringAndWeight(stringref(_scratchPad, res.ptr - _scratchPad), v.second); + return {stringref(_scratchPad, res.ptr - _scratchPad), v.second}; } IntegerAndWeight getAsInteger(uint32_t index) const override { return _terms[index]; @@ -85,6 +86,8 @@ private: mutable char _scratchPad[24]; }; +StringTermVector::~StringTermVector() = default; + } MultiTerm::MultiTerm(uint32_t num_terms) diff --git a/searchlib/src/vespa/searchlib/queryeval/global_filter.cpp b/searchlib/src/vespa/searchlib/queryeval/global_filter.cpp index 3ef28b461b8..2d88883780a 100644 --- a/searchlib/src/vespa/searchlib/queryeval/global_filter.cpp +++ b/searchlib/src/vespa/searchlib/queryeval/global_filter.cpp @@ -24,16 +24,19 @@ struct Inactive : GlobalFilter { struct EmptyFilter : GlobalFilter { uint32_t docid_limit; - EmptyFilter(uint32_t docid_limit_in) : docid_limit(docid_limit_in) {} + EmptyFilter(uint32_t docid_limit_in) noexcept : docid_limit(docid_limit_in) {} + ~EmptyFilter() override; bool is_active() const override { return true; } uint32_t size() const override { return docid_limit; } uint32_t count() const override { return 0; } bool check(uint32_t) const override { return false; } }; +EmptyFilter::~EmptyFilter() = default; + struct BitVectorFilter : public GlobalFilter { std::unique_ptr<BitVector> vector; - BitVectorFilter(std::unique_ptr<BitVector> vector_in) + BitVectorFilter(std::unique_ptr<BitVector> vector_in) noexcept : vector(std::move(vector_in)) {} bool is_active() const override { return true; } uint32_t size() const override { return vector->size(); } @@ -49,7 +52,7 @@ struct MultiBitVectorFilter : public GlobalFilter { MultiBitVectorFilter(std::vector<std::unique_ptr<BitVector>> vectors_in, std::vector<uint32_t> splits_in, uint32_t total_size_in, - uint32_t total_count_in) + uint32_t total_count_in) noexcept : vectors(std::move(vectors_in)), splits(std::move(splits_in)), total_size(total_size_in), @@ -105,7 +108,7 @@ struct MakePart : Runnable { } -GlobalFilter::GlobalFilter() = default; +GlobalFilter::GlobalFilter() noexcept = default; GlobalFilter::~GlobalFilter() = default; std::shared_ptr<GlobalFilter> diff --git a/searchlib/src/vespa/searchlib/queryeval/global_filter.h b/searchlib/src/vespa/searchlib/queryeval/global_filter.h index acae2bf6297..66f85299dd1 100644 --- a/searchlib/src/vespa/searchlib/queryeval/global_filter.h +++ b/searchlib/src/vespa/searchlib/queryeval/global_filter.h @@ -22,7 +22,7 @@ class Blueprint; class GlobalFilter : public std::enable_shared_from_this<GlobalFilter> { public: - GlobalFilter(); + GlobalFilter() noexcept; GlobalFilter(const GlobalFilter &) = delete; GlobalFilter(GlobalFilter &&) = delete; virtual bool is_active() const = 0; diff --git a/searchlib/src/vespa/searchlib/queryeval/intermediate_blueprints.cpp b/searchlib/src/vespa/searchlib/queryeval/intermediate_blueprints.cpp index 9a29c4c0caf..8e7bd185f85 100644 --- a/searchlib/src/vespa/searchlib/queryeval/intermediate_blueprints.cpp +++ b/searchlib/src/vespa/searchlib/queryeval/intermediate_blueprints.cpp @@ -601,6 +601,8 @@ SourceBlenderBlueprint::SourceBlenderBlueprint(const ISourceSelector &selector) { } +SourceBlenderBlueprint::~SourceBlenderBlueprint() = default; + Blueprint::HitEstimate SourceBlenderBlueprint::combine(const std::vector<HitEstimate> &data) const { diff --git a/searchlib/src/vespa/searchlib/queryeval/intermediate_blueprints.h b/searchlib/src/vespa/searchlib/queryeval/intermediate_blueprints.h index f4ae2e21b41..8cced6097b5 100644 --- a/searchlib/src/vespa/searchlib/queryeval/intermediate_blueprints.h +++ b/searchlib/src/vespa/searchlib/queryeval/intermediate_blueprints.h @@ -177,6 +177,7 @@ private: public: SourceBlenderBlueprint(const ISourceSelector &selector); + ~SourceBlenderBlueprint() override; HitEstimate combine(const std::vector<HitEstimate> &data) const override; FieldSpecBaseList exposeFields() const override; void sort(Children &children) const override; diff --git a/searchlib/src/vespa/searchlib/queryeval/profiled_iterator.h b/searchlib/src/vespa/searchlib/queryeval/profiled_iterator.h index f20ad1752d3..e1e16e164dc 100644 --- a/searchlib/src/vespa/searchlib/queryeval/profiled_iterator.h +++ b/searchlib/src/vespa/searchlib/queryeval/profiled_iterator.h @@ -48,6 +48,7 @@ public: void or_hits_into(BitVector &result, uint32_t begin_id) override; void and_hits_into(BitVector &result, uint32_t begin_id) override; void visitMembers(vespalib::ObjectVisitor &visitor) const override; + UP andWith(UP filter, uint32_t estimate) override { return _search->andWith(std::move(filter), estimate); } Trinary is_strict() const override { return _search->is_strict(); } Trinary matches_any() const override { return _search->matches_any(); } const PostingInfo *getPostingInfo() const override { return _search->getPostingInfo(); } diff --git a/searchlib/src/vespa/searchlib/tensor/hnsw_index.cpp b/searchlib/src/vespa/searchlib/tensor/hnsw_index.cpp index d22f24cc7da..af332189b61 100644 --- a/searchlib/src/vespa/searchlib/tensor/hnsw_index.cpp +++ b/searchlib/src/vespa/searchlib/tensor/hnsw_index.cpp @@ -67,12 +67,12 @@ template <> class GlobalFilterWrapper<HnswIndexType::SINGLE> { const GlobalFilter *_filter; public: - GlobalFilterWrapper(const GlobalFilter *filter) + explicit GlobalFilterWrapper(const GlobalFilter *filter) : _filter(filter) { } - bool check(uint32_t docid) const noexcept { return !_filter || _filter->check(docid); } + [[nodiscard]] bool check(uint32_t docid) const noexcept { return !_filter || _filter->check(docid); } void clamp_nodeid_limit(uint32_t& nodeid_limit) { if (_filter) { @@ -86,18 +86,38 @@ class GlobalFilterWrapper<HnswIndexType::MULTI> { const GlobalFilter *_filter; uint32_t _docid_limit; public: - GlobalFilterWrapper(const GlobalFilter *filter) + explicit GlobalFilterWrapper(const GlobalFilter *filter) : _filter(filter), _docid_limit(filter ? filter->size() : 0u) { } - bool check(uint32_t docid) const noexcept { return !_filter || (docid < _docid_limit && _filter->check(docid)); } + [[nodiscard]] bool check(uint32_t docid) const noexcept { return !_filter || (docid < _docid_limit && _filter->check(docid)); } static void clamp_nodeid_limit(uint32_t&) { } }; } +namespace internal { + +PreparedAddNode::PreparedAddNode() noexcept + : connections() +{ } +PreparedAddNode::PreparedAddNode(std::vector<Links>&& connections_in) noexcept + : connections(std::move(connections_in)) +{ } +PreparedAddNode::~PreparedAddNode() = default; +PreparedAddNode::PreparedAddNode(PreparedAddNode&& other) noexcept = default; + +PreparedAddDoc::PreparedAddDoc(uint32_t docid_in, ReadGuard read_guard_in) noexcept + : docid(docid_in), + read_guard(std::move(read_guard_in)), + nodes() +{} +PreparedAddDoc::~PreparedAddDoc() = default; +PreparedAddDoc::PreparedAddDoc(PreparedAddDoc&& other) noexcept = default; +} + template <HnswIndexType type> vespalib::datastore::ArrayStoreConfig HnswIndex<type>::make_default_level_array_store_config() @@ -415,6 +435,10 @@ HnswIndex<type>::HnswIndex(const DocVectorAccess& vectors, DistanceFunction::UP template <HnswIndexType type> HnswIndex<type>::~HnswIndex() = default; +using internal::PreparedAddNode; +using internal::PreparedAddDoc; +using internal::PreparedFirstAddDoc; + template <HnswIndexType type> void HnswIndex<type>::add_document(uint32_t docid) @@ -434,7 +458,7 @@ HnswIndex<type>::add_document(uint32_t docid) } template <HnswIndexType type> -typename HnswIndex<type>::PreparedAddDoc +PreparedAddDoc HnswIndex<type>::internal_prepare_add(uint32_t docid, VectorBundle input_vectors, vespalib::GenerationHandler::Guard read_guard) const { PreparedAddDoc op(docid, std::move(read_guard)); @@ -449,10 +473,10 @@ HnswIndex<type>::internal_prepare_add(uint32_t docid, VectorBundle input_vectors template <HnswIndexType type> void -HnswIndex<type>::internal_prepare_add_node(typename HnswIndex::PreparedAddDoc& op, TypedCells input_vector, const typename GraphType::EntryNode& entry) const +HnswIndex<type>::internal_prepare_add_node(PreparedAddDoc& op, TypedCells input_vector, const typename GraphType::EntryNode& entry) const { int node_max_level = std::min(_level_generator->max_level(), max_max_level); - std::vector<typename PreparedAddNode::Links> connections(node_max_level + 1); + std::vector<PreparedAddNode::Links> connections(node_max_level + 1); if (entry.nodeid == 0) { // graph has no entry point op.nodes.emplace_back(std::move(connections)); diff --git a/searchlib/src/vespa/searchlib/tensor/hnsw_index.h b/searchlib/src/vespa/searchlib/tensor/hnsw_index.h index 8d348d13f15..984acc6c9a1 100644 --- a/searchlib/src/vespa/searchlib/tensor/hnsw_index.h +++ b/searchlib/src/vespa/searchlib/tensor/hnsw_index.h @@ -39,6 +39,29 @@ namespace search::tensor { * TODO: Add details on how to handle removes. */ +namespace internal { +struct PreparedAddNode { + using Links = std::vector<std::pair<uint32_t, vespalib::datastore::EntryRef>>; + std::vector<Links> connections; + + PreparedAddNode() noexcept; + explicit PreparedAddNode(std::vector<Links>&& connections_in) noexcept; + ~PreparedAddNode(); + PreparedAddNode(PreparedAddNode&& other) noexcept; +}; + +struct PreparedFirstAddDoc : public PrepareResult {}; + +struct PreparedAddDoc : public PrepareResult { + using ReadGuard = vespalib::GenerationHandler::Guard; + uint32_t docid; + ReadGuard read_guard; + std::vector<PreparedAddNode> nodes; + PreparedAddDoc(uint32_t docid_in, ReadGuard read_guard_in) noexcept; + ~PreparedAddDoc(); + PreparedAddDoc(PreparedAddDoc&& other) noexcept; +}; +} template <HnswIndexType type> class HnswIndex : public NearestNeighborIndex { public: @@ -155,43 +178,12 @@ protected: const GlobalFilter *filter, uint32_t explore_k, double distance_threshold) const; - struct PreparedAddNode { - using Links = std::vector<std::pair<uint32_t, vespalib::datastore::EntryRef>>; - std::vector<Links> connections; - - PreparedAddNode() noexcept - : connections() - { - } - PreparedAddNode(std::vector<Links>&& connections_in) noexcept - : connections(std::move(connections_in)) - { - } - ~PreparedAddNode() = default; - PreparedAddNode(PreparedAddNode&& other) noexcept = default; - }; - - struct PreparedFirstAddDoc : public PrepareResult {}; - - struct PreparedAddDoc : public PrepareResult { - using ReadGuard = vespalib::GenerationHandler::Guard; - uint32_t docid; - ReadGuard read_guard; - std::vector<PreparedAddNode> nodes; - PreparedAddDoc(uint32_t docid_in, ReadGuard read_guard_in) - : docid(docid_in), - read_guard(std::move(read_guard_in)), - nodes() - {} - ~PreparedAddDoc() = default; - PreparedAddDoc(PreparedAddDoc&& other) = default; - }; - PreparedAddDoc internal_prepare_add(uint32_t docid, VectorBundle input_vectors, + internal::PreparedAddDoc internal_prepare_add(uint32_t docid, VectorBundle input_vectors, vespalib::GenerationHandler::Guard read_guard) const; - void internal_prepare_add_node(HnswIndex::PreparedAddDoc& op, TypedCells input_vector, const typename GraphType::EntryNode& entry) const; - LinkArray filter_valid_nodeids(uint32_t level, const typename PreparedAddNode::Links &neighbors, uint32_t self_nodeid); - void internal_complete_add(uint32_t docid, PreparedAddDoc &op); - void internal_complete_add_node(uint32_t nodeid, uint32_t docid, uint32_t subspace, PreparedAddNode &prepared_node); + void internal_prepare_add_node(internal::PreparedAddDoc& op, TypedCells input_vector, const typename GraphType::EntryNode& entry) const; + LinkArray filter_valid_nodeids(uint32_t level, const internal::PreparedAddNode::Links &neighbors, uint32_t self_nodeid); + void internal_complete_add(uint32_t docid, internal::PreparedAddDoc &op); + void internal_complete_add_node(uint32_t nodeid, uint32_t docid, uint32_t subspace, internal::PreparedAddNode &prepared_node); public: HnswIndex(const DocVectorAccess& vectors, DistanceFunction::UP distance_func, RandomLevelGenerator::UP level_generator, const HnswIndexConfig& cfg); diff --git a/searchlib/src/vespa/searchlib/test/weightedchildrenverifiers.h b/searchlib/src/vespa/searchlib/test/weightedchildrenverifiers.h index 1f3017237a5..208e5cb6de3 100644 --- a/searchlib/src/vespa/searchlib/test/weightedchildrenverifiers.h +++ b/searchlib/src/vespa/searchlib/test/weightedchildrenverifiers.h @@ -8,10 +8,10 @@ namespace search::test { class WeightedChildrenVerifier : public SearchIteratorVerifier { public: - WeightedChildrenVerifier() : - _weights(_num_children, 1) + WeightedChildrenVerifier() + : _weights(_num_children, 1) { } - ~WeightedChildrenVerifier() {} + ~WeightedChildrenVerifier() override {} protected: static constexpr size_t _num_children = 7; @@ -21,15 +21,16 @@ protected: class IteratorChildrenVerifier : public WeightedChildrenVerifier { public: - IteratorChildrenVerifier() : - WeightedChildrenVerifier(), - _split_lists(_num_children) + IteratorChildrenVerifier() + : WeightedChildrenVerifier(), + _split_lists(_num_children) { auto full_list = getExpectedDocIds(); for (size_t i = 0; i < full_list.size(); ++i) { _split_lists[i % _num_children].push_back(full_list[i]); } } + ~IteratorChildrenVerifier() override { } SearchIterator::UP create(bool strict) const override { (void) strict; std::vector<SearchIterator*> children; @@ -58,7 +59,7 @@ public: _helper.set_doc(full_list[i], i % _num_children, 1); } } - ~DwaIteratorChildrenVerifier() {} + ~DwaIteratorChildrenVerifier() override {} SearchIterator::UP create(bool strict) const override { (void) strict; std::vector<DocumentWeightIterator> children; @@ -69,9 +70,8 @@ public: return create(std::move(children)); } protected: - virtual SearchIterator::UP create(std::vector<DocumentWeightIterator> && children) const { - (void) children; - return SearchIterator::UP(); + virtual SearchIterator::UP create(std::vector<DocumentWeightIterator> &&) const { + return {}; } DocumentWeightAttributeHelper _helper; }; diff --git a/slobrok/src/tests/local_rpc_monitor_map/local_rpc_monitor_map_test.cpp b/slobrok/src/tests/local_rpc_monitor_map/local_rpc_monitor_map_test.cpp index 5b0829f1dff..b7235155f8c 100644 --- a/slobrok/src/tests/local_rpc_monitor_map/local_rpc_monitor_map_test.cpp +++ b/slobrok/src/tests/local_rpc_monitor_map/local_rpc_monitor_map_test.cpp @@ -60,6 +60,7 @@ public: }; struct MapLog : CallLog<MapCall>, MapListener { + ~MapLog() override; void add(const ServiceMapping &mapping) override { log(MapCall::add(mapping)); } @@ -73,6 +74,8 @@ struct MapLog : CallLog<MapCall>, MapListener { } }; +MapLog::~MapLog() = default; + struct MonitorLog : CallLog<MonitorCall>, MappingMonitor { void start(const ServiceMapping& mapping, bool hurry) override { log(MonitorCall::start(mapping, hurry)); @@ -84,7 +87,7 @@ struct MonitorLog : CallLog<MonitorCall>, MappingMonitor { struct MyMappingMonitor : MappingMonitor { MonitorLog &monitor; - MyMappingMonitor(MonitorLog &m) : monitor(m) {} + explicit MyMappingMonitor(MonitorLog &m) : monitor(m) {} void start(const ServiceMapping& mapping, bool hurry) override { monitor.start(mapping, hurry); } @@ -160,7 +163,7 @@ struct LocalRpcMonitorMapTest : public ::testing::Test { map_log.expect({}); } } - ~LocalRpcMonitorMapTest(); + ~LocalRpcMonitorMapTest() override; }; LocalRpcMonitorMapTest::~LocalRpcMonitorMapTest() = default; diff --git a/slobrok/src/tests/service_map_history/service_map_history_test.cpp b/slobrok/src/tests/service_map_history/service_map_history_test.cpp index 398ec91feb2..81058d9c725 100644 --- a/slobrok/src/tests/service_map_history/service_map_history_test.cpp +++ b/slobrok/src/tests/service_map_history/service_map_history_test.cpp @@ -14,7 +14,7 @@ using Map = std::map<vespalib::string, vespalib::string>; struct Dumper : ServiceMapHistory::DiffCompletionHandler { std::unique_ptr<MapDiff> got = {}; void handle(MapDiff diff) override { - got = std::make_unique<MapDiff>(diff); + got = std::make_unique<MapDiff>(std::move(diff)); } }; @@ -22,7 +22,7 @@ MapDiff diffGen(ServiceMapHistory &history, uint32_t gen) { Dumper dumper; history.asyncGenerationDiff(&dumper, GenCnt(gen)); EXPECT_TRUE(dumper.got); - return *dumper.got; + return std::move(*dumper.got); } Map dump(ServiceMapHistory &history) { @@ -60,7 +60,7 @@ TEST(ServiceMapHistoryTest, empty_inspection) { empty2.asyncGenerationDiff(&dumper, gen); } EXPECT_TRUE(dumper.got); - auto diff1 = *dumper.got; + auto diff1 = std::move(*dumper.got); EXPECT_FALSE(diff1.is_full_dump()); EXPECT_EQ(diff1.fromGen, gen); EXPECT_TRUE(diff1.removed.empty()); @@ -140,7 +140,7 @@ TEST(ServiceMapHistoryTest, full_inspection) { EXPECT_FALSE(dumper.got); } EXPECT_TRUE(dumper.got); - auto diff1 = *dumper.got; + auto diff1 = std::move(*dumper.got); EXPECT_EQ(diff1.fromGen, GenCnt(1987)); EXPECT_TRUE(diff1.removed.empty()); EXPECT_TRUE(diff1.updated.empty()); @@ -162,7 +162,7 @@ public: got_updates = diff.updated.size(); } - ~MockListener(); + ~MockListener() override; }; MockListener::~MockListener() = default; diff --git a/slobrok/src/vespa/slobrok/server/map_diff.cpp b/slobrok/src/vespa/slobrok/server/map_diff.cpp index ea71e1c1269..fb086fbe45b 100644 --- a/slobrok/src/vespa/slobrok/server/map_diff.cpp +++ b/slobrok/src/vespa/slobrok/server/map_diff.cpp @@ -4,6 +4,7 @@ namespace slobrok { +MapDiff::MapDiff(MapDiff &&) noexcept = default; MapDiff::~MapDiff() = default; } // namespace slobrok diff --git a/slobrok/src/vespa/slobrok/server/map_diff.h b/slobrok/src/vespa/slobrok/server/map_diff.h index a28336eb2e4..638d4cd5d4c 100644 --- a/slobrok/src/vespa/slobrok/server/map_diff.h +++ b/slobrok/src/vespa/slobrok/server/map_diff.h @@ -29,6 +29,7 @@ struct MapDiff { : MapDiff(0, {}, std::move(mappings), to) {} + MapDiff(MapDiff &&) noexcept; ~MapDiff(); // is this a diff from the empty map: diff --git a/standalone-container/src/main/java/com/yahoo/container/standalone/CloudConfigInstallVariables.java b/standalone-container/src/main/java/com/yahoo/container/standalone/CloudConfigInstallVariables.java index 5560a633f66..74aacceaf9e 100644 --- a/standalone-container/src/main/java/com/yahoo/container/standalone/CloudConfigInstallVariables.java +++ b/standalone-container/src/main/java/com/yahoo/container/standalone/CloudConfigInstallVariables.java @@ -126,6 +126,12 @@ public class CloudConfigInstallVariables implements CloudConfigOptions { return getInstallVariable("zts_url"); } + @Override + public String zooKeeperSnapshotMethod() { + String vespaZookeeperSnapshotMethod = System.getenv("VESPA_ZOOKEEPER_SNAPSHOT_METHOD"); + return vespaZookeeperSnapshotMethod == null ? "" : vespaZookeeperSnapshotMethod; + } + static ConfigServer[] toConfigServers(String configserversString) { return multiValueParameterStream(configserversString) .map(CloudConfigInstallVariables::toConfigServer) diff --git a/storage/src/tests/distributor/CMakeLists.txt b/storage/src/tests/distributor/CMakeLists.txt index c8dde75c73d..c59b56eb68f 100644 --- a/storage/src/tests/distributor/CMakeLists.txt +++ b/storage/src/tests/distributor/CMakeLists.txt @@ -22,6 +22,7 @@ vespa_add_executable(storage_distributor_gtest_runner_app TEST gtest_runner.cpp idealstatemanagertest.cpp joinbuckettest.cpp + maintenancemocks.cpp maintenanceschedulertest.cpp mergelimitertest.cpp mergeoperationtest.cpp diff --git a/storage/src/tests/distributor/distributor_host_info_reporter_test.cpp b/storage/src/tests/distributor/distributor_host_info_reporter_test.cpp index a02e98a93b8..6dfab5abc21 100644 --- a/storage/src/tests/distributor/distributor_host_info_reporter_test.cpp +++ b/storage/src/tests/distributor/distributor_host_info_reporter_test.cpp @@ -21,14 +21,9 @@ using BucketSpacesStats = BucketSpacesStatsProvider::BucketSpacesStats; using namespace ::testing; struct DistributorHostInfoReporterTest : Test { - void verifyBucketSpaceStats(const vespalib::Slime& root, - uint16_t nodeIndex, - const vespalib::string& bucketSpaceName, - size_t bucketsTotal, - size_t bucketsPending); - void verifyBucketSpaceStats(const vespalib::Slime& root, - uint16_t nodeIndex, - const vespalib::string& bucketSpaceName); + static void verifyBucketSpaceStats(const vespalib::Slime& root, uint16_t nodeIndex, const vespalib::string& bucketSpaceName, + size_t bucketsTotal, size_t bucketsPending); + static void verifyBucketSpaceStats(const vespalib::Slime& root, uint16_t nodeIndex, const vespalib::string& bucketSpaceName); }; using ms = std::chrono::milliseconds; @@ -40,19 +35,26 @@ struct MockedMinReplicaProvider : MinReplicaProvider { MinReplicaStats minReplica; + ~MockedMinReplicaProvider() override; std::unordered_map<uint16_t, uint32_t> getMinReplica() const override { return minReplica; } }; +MockedMinReplicaProvider::~MockedMinReplicaProvider() = default; + + struct MockedBucketSpacesStatsProvider : public BucketSpacesStatsProvider { PerNodeBucketSpacesStats stats; + ~MockedBucketSpacesStatsProvider() override; PerNodeBucketSpacesStats getBucketSpacesStats() const override { return stats; } }; +MockedBucketSpacesStatsProvider::~MockedBucketSpacesStatsProvider() = default; + const vespalib::slime::Inspector& getNode(const vespalib::Slime& root, uint16_t nodeIndex) { diff --git a/storage/src/tests/distributor/maintenancemocks.cpp b/storage/src/tests/distributor/maintenancemocks.cpp new file mode 100644 index 00000000000..e942a3889c1 --- /dev/null +++ b/storage/src/tests/distributor/maintenancemocks.cpp @@ -0,0 +1,13 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include "maintenancemocks.h" + +namespace storage::distributor { + +MockOperationStarter::MockOperationStarter() noexcept + : _shouldStart(true) +{} + +MockOperationStarter::~MockOperationStarter() = default; + +} diff --git a/storage/src/tests/distributor/maintenancemocks.h b/storage/src/tests/distributor/maintenancemocks.h index 5bcd8f962a6..0698c5e096b 100644 --- a/storage/src/tests/distributor/maintenancemocks.h +++ b/storage/src/tests/distributor/maintenancemocks.h @@ -23,9 +23,7 @@ class MockMaintenancePriorityGenerator { stats.incMovingOut(1, makeBucketSpace()); stats.incCopyingIn(2, makeBucketSpace()); - return MaintenancePriorityAndType( - MaintenancePriority(MaintenancePriority::VERY_HIGH), - MaintenanceOperation::MERGE_BUCKET); + return { MaintenancePriority(MaintenancePriority::VERY_HIGH), MaintenanceOperation::MERGE_BUCKET }; } }; @@ -38,41 +36,41 @@ class MockOperation : public MaintenanceOperation bool _was_blocked; bool _was_throttled; public: - MockOperation(const document::Bucket &bucket) + explicit MockOperation(const document::Bucket &bucket) : _bucket(bucket), _shouldBlock(false), _was_blocked(false), _was_throttled(false) {} - std::string toString() const override { + [[nodiscard]] std::string toString() const override { return _bucket.toString(); } void onClose(DistributorStripeMessageSender&) override {} - const char* getName() const noexcept override { return "MockOperation"; } - const std::string& getDetailedReason() const override { + [[nodiscard]] const char* getName() const noexcept override { return "MockOperation"; } + [[nodiscard]] const std::string& getDetailedReason() const override { return _reason; } void onStart(DistributorStripeMessageSender&) override {} void onReceive(DistributorStripeMessageSender&, const std::shared_ptr<api::StorageReply>&) override {} void on_blocked() override { _was_blocked = true; } void on_throttled() override { _was_throttled = true; } - bool isBlocked(const DistributorStripeOperationContext&, const OperationSequencer&) const override { + [[nodiscard]] bool isBlocked(const DistributorStripeOperationContext&, const OperationSequencer&) const override { return _shouldBlock; } void setShouldBlock(bool shouldBlock) { _shouldBlock = shouldBlock; } - bool get_was_blocked() const noexcept { return _was_blocked; } - bool get_was_throttled() const noexcept { return _was_throttled; } + [[nodiscard]] bool get_was_blocked() const noexcept { return _was_blocked; } + [[nodiscard]] bool get_was_throttled() const noexcept { return _was_throttled; } }; class MockMaintenanceOperationGenerator : public MaintenanceOperationGenerator { public: - MaintenanceOperation::SP generate(const document::Bucket&bucket) const override { + [[nodiscard]] MaintenanceOperation::SP generate(const document::Bucket&bucket) const override { return std::make_shared<MockOperation>(bucket); } @@ -95,9 +93,8 @@ class MockOperationStarter std::vector<Operation::SP> _operations; bool _shouldStart; public: - MockOperationStarter() - : _shouldStart(true) - {} + MockOperationStarter() noexcept; + ~MockOperationStarter() override; bool start(const std::shared_ptr<Operation>& operation, Priority priority) override { @@ -130,7 +127,7 @@ public: _allow = allow; } - bool may_allow_operation_with_priority(OperationStarter::Priority) const noexcept override { + [[nodiscard]] bool may_allow_operation_with_priority(OperationStarter::Priority) const noexcept override { return _allow; } }; diff --git a/storage/src/tests/storageserver/service_layer_error_listener_test.cpp b/storage/src/tests/storageserver/service_layer_error_listener_test.cpp index 571bebd9c86..11ecc11f810 100644 --- a/storage/src/tests/storageserver/service_layer_error_listener_test.cpp +++ b/storage/src/tests/storageserver/service_layer_error_listener_test.cpp @@ -41,9 +41,11 @@ struct Fixture { TestShutdownListener shutdown_listener; ServiceLayerErrorListener error_listener{component, merge_throttler}; + Fixture(); ~Fixture(); }; +Fixture::Fixture() = default; Fixture::~Fixture() = default; } diff --git a/storage/src/tests/storageserver/statemanagertest.cpp b/storage/src/tests/storageserver/statemanagertest.cpp index d7fc04ebe8c..5764460f5bb 100644 --- a/storage/src/tests/storageserver/statemanagertest.cpp +++ b/storage/src/tests/storageserver/statemanagertest.cpp @@ -5,7 +5,6 @@ #include <vespa/storageapi/message/state.h> #include <vespa/vdslib/state/cluster_state_bundle.h> #include <vespa/vdslib/state/clusterstate.h> -#include <vespa/storage/frameworkimpl/component/storagecomponentregisterimpl.h> #include <vespa/storage/storageserver/statemanager.h> #include <tests/common/teststorageapp.h> #include <tests/common/testhelper.h> @@ -159,9 +158,8 @@ struct MyStateListener : public StateListener { lib::NodeState current; std::ostringstream ost; - MyStateListener(const NodeStateUpdater& upd) - : updater(upd), current(*updater.getReportedNodeState()) {} - ~MyStateListener() override = default; + MyStateListener(const NodeStateUpdater& upd); + ~MyStateListener() override; void handleNewState() noexcept override { ost << current << " -> "; @@ -169,6 +167,11 @@ struct MyStateListener : public StateListener { ost << current << "\n"; } }; + +MyStateListener::MyStateListener(const NodeStateUpdater& upd) + : updater(upd), current(*updater.getReportedNodeState()) +{} +MyStateListener::~MyStateListener() = default; } TEST_F(StateManagerTest, reported_node_state) { diff --git a/storage/src/tests/storageserver/statereportertest.cpp b/storage/src/tests/storageserver/statereportertest.cpp index d7c5bdbf6ea..47d70cf436e 100644 --- a/storage/src/tests/storageserver/statereportertest.cpp +++ b/storage/src/tests/storageserver/statereportertest.cpp @@ -42,10 +42,10 @@ struct StateReporterTest : Test { std::shared_ptr<FileStorMetrics> _filestorMetrics; StateReporterTest(); + ~StateReporterTest() override; void SetUp() override; void TearDown() override; - void runLoad(uint32_t count = 1); }; namespace { @@ -68,6 +68,8 @@ StateReporterTest::StateReporterTest() { } +StateReporterTest::~StateReporterTest() = default; + void StateReporterTest::SetUp() { _config = std::make_unique<vdstestlib::DirConfig>(getStandardConfig(true, "statereportertest")); assert(system(("rm -rf " + getRootFolder(*_config)).c_str()) == 0); diff --git a/storage/src/tests/visiting/visitortest.cpp b/storage/src/tests/visiting/visitortest.cpp index 7dc34bb4b6f..f3a538b7832 100644 --- a/storage/src/tests/visiting/visitortest.cpp +++ b/storage/src/tests/visiting/visitortest.cpp @@ -65,7 +65,8 @@ struct VisitorTest : Test { DummyStorageLink* _bottom; VisitorManager* _manager; - VisitorTest() : _node() {} + VisitorTest(); + ~VisitorTest() override; // Not using setUp since can't throw exception out of it. void initializeTest(const TestParams& params = TestParams()); @@ -138,6 +139,9 @@ protected: uint32_t VisitorTest::docCount = 10; +VisitorTest::VisitorTest() = default; +VisitorTest::~VisitorTest() = default; + void VisitorTest::initializeTest(const TestParams& params) { @@ -157,18 +161,16 @@ VisitorTest::initializeTest(const TestParams& params) std::filesystem::create_directories(std::filesystem::path(vespalib::make_string("%s/disks/d0", rootFolder.c_str()))); std::filesystem::create_directories(std::filesystem::path(vespalib::make_string("%s/disks/d1", rootFolder.c_str()))); - _messageSessionFactory.reset( - new TestVisitorMessageSessionFactory(config.getConfigId())); + _messageSessionFactory = std::make_unique<TestVisitorMessageSessionFactory>(config.getConfigId()); if (params._autoReplyError.getCode() != mbus::ErrorCode::NONE) { _messageSessionFactory->_autoReplyError = params._autoReplyError; _messageSessionFactory->_createAutoReplyVisitorSessions = true; } - _node.reset(new TestServiceLayerApp(config.getConfigId())); - _top.reset(new DummyStorageLink()); + _node = std::make_unique<TestServiceLayerApp>(config.getConfigId()); + _top = std::make_unique<DummyStorageLink>(); _top->push_back(std::unique_ptr<StorageLink>(_manager - = new VisitorManager( - config::ConfigUri(config.getConfigId()), - _node->getComponentRegister(), *_messageSessionFactory))); + = new VisitorManager(config::ConfigUri(config.getConfigId()), + _node->getComponentRegister(), *_messageSessionFactory))); _bottom = new DummyStorageLink(); _top->push_back(std::unique_ptr<StorageLink>(_bottom)); _manager->setTimeBetweenTicks(10); @@ -296,13 +298,13 @@ VisitorTest::getMessagesAndReply( switch (msg->getType()) { case documentapi::DocumentProtocol::MESSAGE_PUTDOCUMENT: - docs.push_back(static_cast<documentapi::PutDocumentMessage&>(*msg).getDocumentSP()); + docs.push_back(dynamic_cast<documentapi::PutDocumentMessage&>(*msg).getDocumentSP()); break; case documentapi::DocumentProtocol::MESSAGE_REMOVEDOCUMENT: - docIds.push_back(static_cast<documentapi::RemoveDocumentMessage&>(*msg).getDocumentId()); + docIds.push_back(dynamic_cast<documentapi::RemoveDocumentMessage&>(*msg).getDocumentId()); break; case documentapi::DocumentProtocol::MESSAGE_VISITORINFO: - infoMessages.push_back(static_cast<documentapi::VisitorInfoMessage&>(*msg).getErrorMessage()); + infoMessages.push_back(dynamic_cast<documentapi::VisitorInfoMessage&>(*msg).getErrorMessage()); break; default: break; @@ -357,10 +359,10 @@ VisitorTest::verifyCreateVisitorReply( uint32_t VisitorTest::getMatchingDocuments(std::vector<Document::SP >& docs) { uint32_t equalCount = 0; - for (uint32_t i=0; i<docs.size(); ++i) { - for (uint32_t j=0; j<_documents.size(); ++j) { - if (*docs[i] == *_documents[j] && - docs[i]->getId() == _documents[j]->getId()) + for (auto & doc : docs) { + for (auto & _document : _documents) { + if (*doc == *_document && + doc->getId() == _document->getId()) { equalCount++; } diff --git a/storage/src/vespa/storage/distributor/ideal_service_layer_nodes_bundle.cpp b/storage/src/vespa/storage/distributor/ideal_service_layer_nodes_bundle.cpp index 315b7f3dd52..0d37219356e 100644 --- a/storage/src/vespa/storage/distributor/ideal_service_layer_nodes_bundle.cpp +++ b/storage/src/vespa/storage/distributor/ideal_service_layer_nodes_bundle.cpp @@ -12,6 +12,8 @@ IdealServiceLayerNodesBundle::IdealServiceLayerNodesBundle() noexcept { } +IdealServiceLayerNodesBundle::IdealServiceLayerNodesBundle(IdealServiceLayerNodesBundle &&) noexcept = default; + IdealServiceLayerNodesBundle::~IdealServiceLayerNodesBundle() = default; } diff --git a/storage/src/vespa/storage/distributor/ideal_service_layer_nodes_bundle.h b/storage/src/vespa/storage/distributor/ideal_service_layer_nodes_bundle.h index f98c4699eef..929ec7aadc1 100644 --- a/storage/src/vespa/storage/distributor/ideal_service_layer_nodes_bundle.h +++ b/storage/src/vespa/storage/distributor/ideal_service_layer_nodes_bundle.h @@ -15,6 +15,7 @@ class IdealServiceLayerNodesBundle { std::vector<uint16_t> _available_nonretired_or_maintenance_nodes; public: IdealServiceLayerNodesBundle() noexcept; + IdealServiceLayerNodesBundle(IdealServiceLayerNodesBundle &&) noexcept; ~IdealServiceLayerNodesBundle(); void set_available_nodes(std::vector<uint16_t> available_nodes) { diff --git a/storage/src/vespa/storage/frameworkimpl/status/statuswebserver.cpp b/storage/src/vespa/storage/frameworkimpl/status/statuswebserver.cpp index 40738ccb60f..4d7c9dceb12 100644 --- a/storage/src/vespa/storage/frameworkimpl/status/statuswebserver.cpp +++ b/storage/src/vespa/storage/frameworkimpl/status/statuswebserver.cpp @@ -128,18 +128,24 @@ StatusWebServer::WebServer::handle_get(vespalib::Portal::GetRequest request) } namespace { - class IndexPageReporter : public framework::HtmlStatusReporter { - std::ostringstream ost; - void reportHtmlStatus(std::ostream& out,const framework::HttpUrlPath&) const override { - out << ost.str(); - } - public: - IndexPageReporter() : framework::HtmlStatusReporter("", "Index page") {} +class IndexPageReporter : public framework::HtmlStatusReporter { + std::ostringstream ost; + void reportHtmlStatus(std::ostream& out,const framework::HttpUrlPath&) const override { + out << ost.str(); + } + +public: + IndexPageReporter(); + ~IndexPageReporter() override; + + template<typename T> + IndexPageReporter& operator<<(const T& t) { ost << t; return *this; } +}; + +IndexPageReporter::IndexPageReporter() : framework::HtmlStatusReporter("", "Index page") {} +IndexPageReporter::~IndexPageReporter() = default; - template<typename T> - IndexPageReporter& operator<<(const T& t) { ost << t; return *this; } - }; } diff --git a/storage/src/vespa/storage/persistence/provider_error_wrapper.cpp b/storage/src/vespa/storage/persistence/provider_error_wrapper.cpp index 9e55c0f9088..48345ff09bd 100644 --- a/storage/src/vespa/storage/persistence/provider_error_wrapper.cpp +++ b/storage/src/vespa/storage/persistence/provider_error_wrapper.cpp @@ -14,6 +14,12 @@ ProviderErrorWrapper::checkResult(ResultType&& result) const return std::forward<ResultType>(result); } +ProviderErrorWrapper::ProviderErrorWrapper(spi::PersistenceProvider& impl) noexcept + : _impl(impl), + _mutex() +{ } +ProviderErrorWrapper::~ProviderErrorWrapper() = default; + void ProviderErrorWrapper::handle(const spi::Result & result) const { if (result.getErrorCode() == spi::Result::ErrorType::FATAL_ERROR) { diff --git a/storage/src/vespa/storage/persistence/provider_error_wrapper.h b/storage/src/vespa/storage/persistence/provider_error_wrapper.h index 82447fe4549..da32cbb4270 100644 --- a/storage/src/vespa/storage/persistence/provider_error_wrapper.h +++ b/storage/src/vespa/storage/persistence/provider_error_wrapper.h @@ -32,11 +32,8 @@ public: class ProviderErrorWrapper : public spi::PersistenceProvider, public spi::ResultHandler { public: - explicit ProviderErrorWrapper(spi::PersistenceProvider& impl) - : _impl(impl), - _mutex() - { - } + explicit ProviderErrorWrapper(spi::PersistenceProvider& impl) noexcept; + ~ProviderErrorWrapper() override; spi::Result initialize() override; spi::BucketIdListResult listBuckets(BucketSpace bucketSpace) const override; diff --git a/storage/src/vespa/storage/storageserver/configurable_bucket_resolver.cpp b/storage/src/vespa/storage/storageserver/configurable_bucket_resolver.cpp index 2e10a7c4836..fd01210ae9e 100644 --- a/storage/src/vespa/storage/storageserver/configurable_bucket_resolver.cpp +++ b/storage/src/vespa/storage/storageserver/configurable_bucket_resolver.cpp @@ -9,6 +9,11 @@ using namespace document; namespace storage { +ConfigurableBucketResolver::ConfigurableBucketResolver(BucketSpaceMapping type_to_space) noexcept + : _type_to_space(std::move(type_to_space)) +{} +ConfigurableBucketResolver::~ConfigurableBucketResolver() = default; + document::Bucket ConfigurableBucketResolver::bucketFromId(const DocumentId& id) const { if (!id.hasDocType()) { // Legacy document ids without document type maps to default bucket space diff --git a/storage/src/vespa/storage/storageserver/configurable_bucket_resolver.h b/storage/src/vespa/storage/storageserver/configurable_bucket_resolver.h index 995e253f22c..5088b1f15b1 100644 --- a/storage/src/vespa/storage/storageserver/configurable_bucket_resolver.h +++ b/storage/src/vespa/storage/storageserver/configurable_bucket_resolver.h @@ -21,9 +21,8 @@ public: using BucketSpaceMapping = std::unordered_map<vespalib::string, document::BucketSpace, vespalib::hash<vespalib::string>>; const BucketSpaceMapping _type_to_space; public: - explicit ConfigurableBucketResolver(BucketSpaceMapping type_to_space) noexcept - : _type_to_space(std::move(type_to_space)) - {} + explicit ConfigurableBucketResolver(BucketSpaceMapping type_to_space) noexcept; + ~ConfigurableBucketResolver() override; document::Bucket bucketFromId(const document::DocumentId&) const override; document::BucketSpace bucketSpaceFromName(const vespalib::string& name) const override; diff --git a/storage/src/vespa/storage/storageserver/documentapiconverter.cpp b/storage/src/vespa/storage/storageserver/documentapiconverter.cpp index 9a5fb2bdd13..9ae6aaf0653 100644 --- a/storage/src/vespa/storage/storageserver/documentapiconverter.cpp +++ b/storage/src/vespa/storage/storageserver/documentapiconverter.cpp @@ -42,7 +42,7 @@ DocumentApiConverter::toStorageAPI(documentapi::DocumentMessage& fromMsg) switch (fromMsg.getType()) { case DocumentProtocol::MESSAGE_PUTDOCUMENT: { - documentapi::PutDocumentMessage& from(static_cast<documentapi::PutDocumentMessage&>(fromMsg)); + auto & from(static_cast<documentapi::PutDocumentMessage&>(fromMsg)); document::Bucket bucket = bucketResolver()->bucketFromId(from.getDocument().getId()); auto to = std::make_unique<api::PutCommand>(bucket, from.stealDocument(), from.getTimestamp()); to->setCondition(from.getCondition()); @@ -51,7 +51,7 @@ DocumentApiConverter::toStorageAPI(documentapi::DocumentMessage& fromMsg) } case DocumentProtocol::MESSAGE_UPDATEDOCUMENT: { - documentapi::UpdateDocumentMessage& from(static_cast<documentapi::UpdateDocumentMessage&>(fromMsg)); + auto & from(static_cast<documentapi::UpdateDocumentMessage&>(fromMsg)); document::Bucket bucket = bucketResolver()->bucketFromId(from.getDocumentUpdate().getId()); auto to = std::make_unique<api::UpdateCommand>(bucket, from.stealDocumentUpdate(), from.getNewTimestamp()); to->setOldTimestamp(from.getOldTimestamp()); @@ -69,13 +69,13 @@ DocumentApiConverter::toStorageAPI(documentapi::DocumentMessage& fromMsg) } case DocumentProtocol::MESSAGE_GETDOCUMENT: { - documentapi::GetDocumentMessage& from(static_cast<documentapi::GetDocumentMessage&>(fromMsg)); + auto & from(static_cast<documentapi::GetDocumentMessage&>(fromMsg)); toMsg = std::make_unique<api::GetCommand>(bucketResolver()->bucketFromId(from.getDocumentId()), from.getDocumentId(), from.getFieldSet()); break; } case DocumentProtocol::MESSAGE_CREATEVISITOR: { - documentapi::CreateVisitorMessage& from(static_cast<documentapi::CreateVisitorMessage&>(fromMsg)); + auto & from(static_cast<documentapi::CreateVisitorMessage&>(fromMsg)); auto to = std::make_unique<api::CreateVisitorCommand>(bucketResolver()->bucketSpaceFromName(from.getBucketSpace()), from.getLibraryName(), from.getInstanceId(), from.getDocumentSelection()); @@ -97,28 +97,28 @@ DocumentApiConverter::toStorageAPI(documentapi::DocumentMessage& fromMsg) } case DocumentProtocol::MESSAGE_DESTROYVISITOR: { - documentapi::DestroyVisitorMessage& from(static_cast<documentapi::DestroyVisitorMessage&>(fromMsg)); + auto & from(static_cast<documentapi::DestroyVisitorMessage&>(fromMsg)); toMsg = std::make_unique<api::DestroyVisitorCommand>(from.getInstanceId()); break; } case DocumentProtocol::MESSAGE_STATBUCKET: { - documentapi::StatBucketMessage& from(static_cast<documentapi::StatBucketMessage&>(fromMsg)); + auto & from(static_cast<documentapi::StatBucketMessage&>(fromMsg)); document::Bucket bucket(bucketResolver()->bucketSpaceFromName(from.getBucketSpace()), from.getBucketId()); toMsg = std::make_unique<api::StatBucketCommand>(bucket, from.getDocumentSelection()); break; } case DocumentProtocol::MESSAGE_GETBUCKETLIST: { - documentapi::GetBucketListMessage& from(static_cast<documentapi::GetBucketListMessage&>(fromMsg)); + auto & from(static_cast<documentapi::GetBucketListMessage&>(fromMsg)); document::Bucket bucket(bucketResolver()->bucketSpaceFromName(from.getBucketSpace()), from.getBucketId()); toMsg = std::make_unique<api::GetBucketListCommand>(bucket); break; } case DocumentProtocol::MESSAGE_VISITORINFO: { - documentapi::VisitorInfoMessage& from(static_cast<documentapi::VisitorInfoMessage&>(fromMsg)); - api::VisitorInfoCommand::UP to(new api::VisitorInfoCommand); + auto & from(static_cast<documentapi::VisitorInfoMessage&>(fromMsg)); + auto to = std::make_unique<api::VisitorInfoCommand>(); for (uint32_t i = 0; i < from.getFinishedBuckets().size(); ++i) { to->setBucketCompleted(from.getFinishedBuckets()[i], 0); } @@ -130,7 +130,7 @@ DocumentApiConverter::toStorageAPI(documentapi::DocumentMessage& fromMsg) } case DocumentProtocol::MESSAGE_REMOVELOCATION: { - documentapi::RemoveLocationMessage& from(static_cast<documentapi::RemoveLocationMessage&>(fromMsg)); + auto & from(static_cast<documentapi::RemoveLocationMessage&>(fromMsg)); document::Bucket bucket(bucketResolver()->bucketSpaceFromName(from.getBucketSpace()), document::BucketId(0)); toMsg = std::make_unique<api::RemoveLocationCommand>(from.getDocumentSelection(), bucket); break; @@ -166,8 +166,8 @@ DocumentApiConverter::toStorageAPI(documentapi::DocumentReply& fromReply, switch (fromReply.getType()) { case documentapi::DocumentProtocol::REPLY_CREATEVISITOR: { - documentapi::CreateVisitorReply& fromRep(static_cast<documentapi::CreateVisitorReply&>(fromReply)); - const api::CreateVisitorCommand& fromCmd(static_cast<const api::CreateVisitorCommand&>(fromCommand)); + auto & fromRep(static_cast<documentapi::CreateVisitorReply&>(fromReply)); + const auto & fromCmd(static_cast<const api::CreateVisitorCommand&>(fromCommand)); api::CreateVisitorReply::UP to(new api::CreateVisitorReply(fromCmd)); to->setVisitorStatistics(fromRep.getVisitorStatistics()); @@ -176,8 +176,8 @@ DocumentApiConverter::toStorageAPI(documentapi::DocumentReply& fromReply, } case documentapi::DocumentProtocol::REPLY_STATBUCKET: { - documentapi::StatBucketReply& fromRep(static_cast<documentapi::StatBucketReply&>(fromReply)); - const api::StatBucketCommand& fromCmd(static_cast<const api::StatBucketCommand&>(fromCommand)); + auto & fromRep(static_cast<documentapi::StatBucketReply&>(fromReply)); + const auto & fromCmd(static_cast<const api::StatBucketCommand&>(fromCommand)); toMsg = std::make_unique<api::StatBucketReply>(fromCmd, fromRep.getResults()); break; @@ -204,16 +204,16 @@ DocumentApiConverter::toDocumentAPI(api::StorageCommand& fromMsg) switch (fromMsg.getType().getId()) { case api::MessageType::PUT_ID: { - api::PutCommand& from(static_cast<api::PutCommand&>(fromMsg)); - documentapi::PutDocumentMessage::UP to(new documentapi::PutDocumentMessage(from.getDocument())); + auto & from(static_cast<api::PutCommand&>(fromMsg)); + auto to = std::make_unique<documentapi::PutDocumentMessage>(from.getDocument()); to->setTimestamp(from.getTimestamp()); toMsg = std::move(to); break; } case api::MessageType::UPDATE_ID: { - api::UpdateCommand& from(static_cast<api::UpdateCommand&>(fromMsg)); - documentapi::UpdateDocumentMessage::UP to(new documentapi::UpdateDocumentMessage(from.getUpdate())); + auto & from(static_cast<api::UpdateCommand&>(fromMsg)); + auto to = std::make_unique<documentapi::UpdateDocumentMessage>(from.getUpdate()); to->setOldTimestamp(from.getOldTimestamp()); to->setNewTimestamp(from.getTimestamp()); toMsg = std::move(to); @@ -221,14 +221,14 @@ DocumentApiConverter::toDocumentAPI(api::StorageCommand& fromMsg) } case api::MessageType::REMOVE_ID: { - api::RemoveCommand& from(static_cast<api::RemoveCommand&>(fromMsg)); + auto & from(static_cast<api::RemoveCommand&>(fromMsg)); toMsg = std::make_unique<documentapi::RemoveDocumentMessage>(from.getDocumentId()); break; } case api::MessageType::VISITOR_INFO_ID: { - api::VisitorInfoCommand& from(static_cast<api::VisitorInfoCommand&>(fromMsg)); - documentapi::VisitorInfoMessage::UP to(new documentapi::VisitorInfoMessage); + auto & from(static_cast<api::VisitorInfoCommand&>(fromMsg)); + auto to = std::make_unique<documentapi::VisitorInfoMessage>(); for (uint32_t i = 0; i < from.getCompletedBucketsList().size(); ++i) { to->getFinishedBuckets().push_back(from.getCompletedBucketsList()[i].bucketId); @@ -239,25 +239,25 @@ DocumentApiConverter::toDocumentAPI(api::StorageCommand& fromMsg) } case api::MessageType::SEARCHRESULT_ID: { - api::SearchResultCommand& from(static_cast<api::SearchResultCommand&>(fromMsg)); - toMsg = std::make_unique<documentapi::SearchResultMessage>(from); + auto & from(static_cast<api::SearchResultCommand&>(fromMsg)); + toMsg = std::make_unique<documentapi::SearchResultMessage>(std::move(from)); break; } case api::MessageType::QUERYRESULT_ID: { - api::QueryResultCommand& from(static_cast<api::QueryResultCommand&>(fromMsg)); - toMsg = std::make_unique<documentapi::QueryResultMessage>(from.getSearchResult(), from.getDocumentSummary()); + auto & from(static_cast<api::QueryResultCommand&>(fromMsg)); + toMsg = std::make_unique<documentapi::QueryResultMessage>(std::move(from.getSearchResult()), from.getDocumentSummary()); break; } case api::MessageType::DOCUMENTSUMMARY_ID: { - api::DocumentSummaryCommand& from(static_cast<api::DocumentSummaryCommand&>(fromMsg)); + auto & from(static_cast<api::DocumentSummaryCommand&>(fromMsg)); toMsg = std::make_unique<documentapi::DocumentSummaryMessage>(from); break; } case api::MessageType::MAPVISITOR_ID: { - api::MapVisitorCommand& from(static_cast<api::MapVisitorCommand&>(fromMsg)); + auto & from(static_cast<api::MapVisitorCommand&>(fromMsg)); documentapi::MapVisitorMessage::UP to(new documentapi::MapVisitorMessage); to->getData() = from.getData(); toMsg = std::move(to); @@ -265,16 +265,15 @@ DocumentApiConverter::toDocumentAPI(api::StorageCommand& fromMsg) } case api::MessageType::EMPTYBUCKETS_ID: { - api::EmptyBucketsCommand& from(static_cast<api::EmptyBucketsCommand&>(fromMsg)); + auto & from(static_cast<api::EmptyBucketsCommand&>(fromMsg)); toMsg = std::make_unique<documentapi::EmptyBucketsMessage>(from.getBuckets()); break; } case api::MessageType::VISITOR_CREATE_ID: { - api::CreateVisitorCommand& from(static_cast<api::CreateVisitorCommand&>(fromMsg)); - documentapi::CreateVisitorMessage::UP to( - new documentapi::CreateVisitorMessage(from.getLibraryName(), from.getInstanceId(), - from.getControlDestination(), from.getDataDestination())); + auto & from(static_cast<api::CreateVisitorCommand&>(fromMsg)); + auto to = std::make_unique<documentapi::CreateVisitorMessage>(from.getLibraryName(), from.getInstanceId(), + from.getControlDestination(), from.getDataDestination()); to->setBucketSpace(bucketResolver()->nameFromBucketSpace(from.getBucketSpace())); to->setDocumentSelection(from.getDocumentSelection()); to->setMaximumPendingReplyCount(from.getMaximumPendingReplyCount()); @@ -291,15 +290,15 @@ DocumentApiConverter::toDocumentAPI(api::StorageCommand& fromMsg) } case api::MessageType::VISITOR_DESTROY_ID: { - api::DestroyVisitorCommand& from(static_cast<api::DestroyVisitorCommand&>(fromMsg)); - documentapi::DestroyVisitorMessage::UP to(new documentapi::DestroyVisitorMessage); + auto & from(static_cast<api::DestroyVisitorCommand&>(fromMsg)); + auto to = std::make_unique<documentapi::DestroyVisitorMessage>(); to->setInstanceId(from.getInstanceId()); toMsg = std::move(to); break; } case api::MessageType::STATBUCKET_ID: { - api::StatBucketCommand& from(static_cast<api::StatBucketCommand&>(fromMsg)); + auto & from(static_cast<api::StatBucketCommand&>(fromMsg)); auto statMsg = std::make_unique<documentapi::StatBucketMessage>(from.getBucket().getBucketId(), from.getDocumentSelection()); statMsg->setBucketSpace(bucketResolver()->nameFromBucketSpace(from.getBucket().getBucketSpace())); toMsg = std::move(statMsg); diff --git a/storage/src/vespa/storage/storageserver/rpc/slime_cluster_state_bundle_codec.cpp b/storage/src/vespa/storage/storageserver/rpc/slime_cluster_state_bundle_codec.cpp index 1c5b38967cc..8c994991b9b 100644 --- a/storage/src/vespa/storage/storageserver/rpc/slime_cluster_state_bundle_codec.cpp +++ b/storage/src/vespa/storage/storageserver/rpc/slime_cluster_state_bundle_codec.cpp @@ -26,11 +26,12 @@ namespace { class OutputBuf : public vespalib::Output { public: explicit OutputBuf(size_t estimatedSize) : _buf(estimatedSize) { } + ~OutputBuf() override; vespalib::DataBuffer & getBuf() { return _buf; } private: vespalib::WritableMemory reserve(size_t bytes) override { _buf.ensureFree(bytes); - return vespalib::WritableMemory(_buf.getFree(), _buf.getFreeLen()); + return {_buf.getFree(), _buf.getFreeLen()}; } Output &commit(size_t bytes) override { _buf.moveFreeToData(bytes); @@ -39,19 +40,21 @@ private: vespalib::DataBuffer _buf; }; +OutputBuf::~OutputBuf() = default; + vespalib::string serialize_state(const lib::ClusterState& state) { vespalib::asciistream as; state.serialize(as, false); return as.str(); } -static const Memory StatesField("states"); -static const Memory BaselineField("baseline"); -static const Memory SpacesField("spaces"); -static const Memory DeferredActivationField("deferred-activation"); -static const Memory FeedBlockField("feed-block"); -static const Memory BlockFeedInClusterField("block-feed-in-cluster"); -static const Memory DescriptionField("description"); +const Memory StatesField("states"); +const Memory BaselineField("baseline"); +const Memory SpacesField("spaces"); +const Memory DeferredActivationField("deferred-activation"); +const Memory FeedBlockField("feed-block"); +const Memory BlockFeedInClusterField("block-feed-in-cluster"); +const Memory DescriptionField("description"); } diff --git a/storage/src/vespa/storageapi/message/searchresult.h b/storage/src/vespa/storageapi/message/searchresult.h index 4795876102d..b12fa5e1613 100644 --- a/storage/src/vespa/storageapi/message/searchresult.h +++ b/storage/src/vespa/storageapi/message/searchresult.h @@ -4,8 +4,7 @@ #include "visitor.h" #include <vespa/vdslib/container/searchresult.h> -namespace storage { -namespace api { +namespace storage::api { /** * @class SearchResultCommand @@ -33,5 +32,4 @@ public: DECLARE_STORAGEREPLY(SearchResultReply, onSearchResultReply) }; -} // api -} // storage +} diff --git a/streamingvisitors/src/vespa/searchvisitor/indexenvironment.cpp b/streamingvisitors/src/vespa/searchvisitor/indexenvironment.cpp index 0131dc5639c..538f3efe44a 100644 --- a/streamingvisitors/src/vespa/searchvisitor/indexenvironment.cpp +++ b/streamingvisitors/src/vespa/searchvisitor/indexenvironment.cpp @@ -17,6 +17,8 @@ IndexEnvironment::IndexEnvironment(const ITableManager & tableManager) : { } +IndexEnvironment::IndexEnvironment(const IndexEnvironment &) = default; +IndexEnvironment::IndexEnvironment(IndexEnvironment &&) noexcept = default; IndexEnvironment::~IndexEnvironment() = default; bool diff --git a/streamingvisitors/src/vespa/searchvisitor/indexenvironment.h b/streamingvisitors/src/vespa/searchvisitor/indexenvironment.h index 2cc5a450b2e..af037d87076 100644 --- a/streamingvisitors/src/vespa/searchvisitor/indexenvironment.h +++ b/streamingvisitors/src/vespa/searchvisitor/indexenvironment.h @@ -32,7 +32,9 @@ private: public: IndexEnvironment(const search::fef::ITableManager & tableManager); - ~IndexEnvironment(); + IndexEnvironment(IndexEnvironment &&) noexcept; + IndexEnvironment(const IndexEnvironment &); + ~IndexEnvironment() override; const search::fef::Properties & getProperties() const override { return _properties; } diff --git a/streamingvisitors/src/vespa/vsm/vsm/vsm-adapter.cpp b/streamingvisitors/src/vespa/vsm/vsm/vsm-adapter.cpp index 8f3fa8d0d54..8a3226ee52a 100644 --- a/streamingvisitors/src/vespa/vsm/vsm/vsm-adapter.cpp +++ b/streamingvisitors/src/vespa/vsm/vsm/vsm-adapter.cpp @@ -57,7 +57,7 @@ GetDocsumsStateCallback::set_matching_elements_filler(std::unique_ptr<IMatchingE GetDocsumsStateCallback::~GetDocsumsStateCallback() = default; -DocsumTools::FieldSpec::FieldSpec() : +DocsumTools::FieldSpec::FieldSpec() noexcept : _outputName(), _inputNames(), _command(VsmsummaryConfig::Fieldmap::Command::NONE) diff --git a/streamingvisitors/src/vespa/vsm/vsm/vsm-adapter.h b/streamingvisitors/src/vespa/vsm/vsm/vsm-adapter.h index 36bf3a923f6..77ed9573e54 100644 --- a/streamingvisitors/src/vespa/vsm/vsm/vsm-adapter.h +++ b/streamingvisitors/src/vespa/vsm/vsm/vsm-adapter.h @@ -53,7 +53,7 @@ public: VsmsummaryConfig::Fieldmap::Command _command; public: - FieldSpec(); + FieldSpec() noexcept; ~FieldSpec(); const vespalib::string & getOutputName() const { return _outputName; } void setOutputName(const vespalib::string & name) { _outputName = name; } diff --git a/vdslib/src/vespa/vdslib/container/searchresult.cpp b/vdslib/src/vespa/vdslib/container/searchresult.cpp index 6989fbca8e0..3d7ae1631af 100644 --- a/vdslib/src/vespa/vdslib/container/searchresult.cpp +++ b/vdslib/src/vespa/vdslib/container/searchresult.cpp @@ -121,9 +121,11 @@ SearchResult::SearchResult(document::ByteBuffer & buf) : deserialize(buf); } +SearchResult::SearchResult(SearchResult &&) noexcept = default; SearchResult::~SearchResult() = default; -void SearchResult::deserialize(document::ByteBuffer & buf) +void +SearchResult::deserialize(document::ByteBuffer & buf) { int32_t tmp; buf.getIntNetwork(tmp); _totalHits = tmp; diff --git a/vdslib/src/vespa/vdslib/container/searchresult.h b/vdslib/src/vespa/vdslib/container/searchresult.h index 90fb282c8fd..2be2e5ee1c6 100644 --- a/vdslib/src/vespa/vdslib/container/searchresult.h +++ b/vdslib/src/vespa/vdslib/container/searchresult.h @@ -52,6 +52,7 @@ public: * @param buf A byte buffer that contains a serialized message. */ SearchResult(document::ByteBuffer & buf); + SearchResult(SearchResult &&) noexcept; ~SearchResult(); AggregatorList & getGroupingList() { return _groupingList; } diff --git a/vespalib/src/apps/vespa-tsan-digest/tsan_digest.cpp b/vespalib/src/apps/vespa-tsan-digest/tsan_digest.cpp index d015cdb0f89..b5040583d3a 100644 --- a/vespalib/src/apps/vespa-tsan-digest/tsan_digest.cpp +++ b/vespalib/src/apps/vespa-tsan-digest/tsan_digest.cpp @@ -115,9 +115,11 @@ private: bool _is_read; bool _is_write; public: - StackTrace(const vespalib::string &heading) noexcept - : _heading(heading), _frames(), _hash(), _is_read(false), _is_write(false) {} - ~StackTrace() {} + StackTrace(const vespalib::string &heading) noexcept; + StackTrace(const StackTrace &); + StackTrace(StackTrace &&) noexcept; + StackTrace & operator=(StackTrace &&) noexcept; + ~StackTrace(); void add_frame(const vespalib::string &frame) { _frames.push_back(frame); } @@ -149,7 +151,16 @@ public: } }; -std::vector<StackTrace> extract_traces(const std::vector<vespalib::string> &lines, size_t cutoff) { +StackTrace::StackTrace(const vespalib::string &heading) noexcept + : _heading(heading), _frames(), _hash(), _is_read(false), _is_write(false) +{} +StackTrace::StackTrace(const StackTrace &) = default; +StackTrace::StackTrace(StackTrace &&) noexcept = default; +StackTrace & StackTrace::operator=(StackTrace &&) noexcept = default; +StackTrace::~StackTrace() = default; + +std::vector<StackTrace> +extract_traces(const std::vector<vespalib::string> &lines, size_t cutoff) { std::vector<StackTrace> result; for (size_t i = 1; (i < lines.size()) && (result.size() < cutoff); ++i) { auto pos = lines[i].find("#0 "); diff --git a/vespalib/src/tests/coro/async_io/async_io_test.cpp b/vespalib/src/tests/coro/async_io/async_io_test.cpp index f5098e30086..2e783c46e6f 100644 --- a/vespalib/src/tests/coro/async_io/async_io_test.cpp +++ b/vespalib/src/tests/coro/async_io/async_io_test.cpp @@ -21,6 +21,14 @@ using namespace vespalib; using namespace vespalib::coro; using namespace vespalib::test; +vespalib::string impl_spec(AsyncIo &async) { + switch (async.get_impl_tag()) { + case AsyncIo::ImplTag::EPOLL: return "epoll"; + case AsyncIo::ImplTag::URING: return "uring"; + } + abort(); +} + Detached self_exiting_run_loop(AsyncIo::SP async) { for (size_t i = 0; co_await async->schedule(); ++i) { fprintf(stderr, "self_exiting_run_loop -> current value: %zu\n", i); @@ -39,7 +47,7 @@ Work run_loop(AsyncIo &async, int a, int b) { TEST(AsyncIoTest, create_async_io) { auto async = AsyncIo::create(); AsyncIo &api = async; - fprintf(stderr, "async_io impl: %s\n", api.get_impl_spec().c_str()); + fprintf(stderr, "async_io impl: %s\n", impl_spec(api).c_str()); } TEST(AsyncIoTest, run_stuff_in_async_io_context) { @@ -132,10 +140,12 @@ Work async_client(AsyncIo &async, CryptoEngine &engine, ServerSocket &server_soc co_return co_await verify_socket_io(*socket, false); } -void verify_socket_io(CryptoEngine &engine) { +void verify_socket_io(CryptoEngine &engine, AsyncIo::ImplTag prefer_impl = AsyncIo::ImplTag::EPOLL) { ServerSocket server_socket("tcp/0"); server_socket.set_blocking(false); - auto async = AsyncIo::create(); + auto async = AsyncIo::create(prefer_impl); + fprintf(stderr, "verify_socket_io: crypto engine: %s, async impl: %s\n", + getClassName(engine).c_str(), impl_spec(async).c_str()); auto f1 = make_future(async_server(async, engine, server_socket)); auto f2 = make_future(async_client(async, engine, server_socket)); (void) f1.get(); @@ -162,4 +172,14 @@ TEST(AsyncIoTest, maybe_tls_false_socket_io) { verify_socket_io(engine); } +TEST(AsyncIoTest, raw_socket_io_with_io_uring_maybe) { + NullCryptoEngine engine; + verify_socket_io(engine, AsyncIo::ImplTag::URING); +} + +TEST(AsyncIoTest, tls_socket_io_with_io_uring_maybe) { + TlsCryptoEngine engine(make_tls_options_for_testing()); + verify_socket_io(engine, AsyncIo::ImplTag::URING); +} + GTEST_MAIN_RUN_ALL_TESTS() diff --git a/vespalib/src/tests/coro/waiting_for/waiting_for_test.cpp b/vespalib/src/tests/coro/waiting_for/waiting_for_test.cpp index 385d4ad24e3..c6678657c34 100644 --- a/vespalib/src/tests/coro/waiting_for/waiting_for_test.cpp +++ b/vespalib/src/tests/coro/waiting_for/waiting_for_test.cpp @@ -11,29 +11,29 @@ using namespace vespalib::coro; struct AsyncService { std::vector<WaitingFor<int>> pending; auto get_value() { - return awaiter_for<int>([&](WaitingFor<int> handle) - { - pending.push_back(std::move(handle)); - }); + return wait_for<int>([&](WaitingFor<int> handle) + { + pending.push_back(std::move(handle)); + }); } }; struct AsyncVoidService { std::vector<void*> pending; auto get_value() { - return awaiter_for<int>([&](WaitingFor<int> handle) - { - pending.push_back(handle.release()); - }); + return wait_for<int>([&](WaitingFor<int> handle) + { + pending.push_back(handle.release()); + }); } }; struct SyncService { auto get_value() { - return awaiter_for<int>([](WaitingFor<int> handle) + return wait_for<int>([](WaitingFor<int> handle) { handle.set_value(42); - return handle.release_waiter(); // symmetric transfer + return handle.mu(); // symmetric transfer }); } }; @@ -45,8 +45,8 @@ Lazy<int> wait_for_value(Service &service) { } template <typename T> -Lazy<T> wait_any(auto &&fun) { - T result = co_await fun(); +Lazy<T> wait_for_fun(auto &&fun) { + T result = co_await wait_for<T>(fun); co_return std::move(result); } @@ -62,6 +62,23 @@ TEST(WaitingForTest, wait_for_external_async_int) { EXPECT_EQ(res.get(), 42); } +TEST(WaitingForTest, wait_for_external_async_int_calculated_by_coroutine) { + AsyncService service1; + AsyncService service2; + auto res = make_future(wait_for_value(service1)); + ASSERT_EQ(service1.pending.size(), 1); + { + async_wait(wait_for_value(service2), std::move(service1.pending[0])); + service1.pending.clear(); + } + EXPECT_TRUE(res.wait_for(0ms) == std::future_status::timeout); + ASSERT_EQ(service2.pending.size(), 1); + service2.pending[0].set_value(42); + service2.pending.clear(); + EXPECT_TRUE(res.wait_for(0ms) == std::future_status::ready); + EXPECT_EQ(res.get(), 42); +} + TEST(WaitingForTest, wait_for_external_async_int_via_void_ptr) { AsyncVoidService service; auto res = make_future(wait_for_value(service)); @@ -86,7 +103,7 @@ TEST(WaitingForTest, wait_for_external_sync_int) { TEST(WaitingForTest, wait_for_move_only_value) { auto val = std::make_unique<int>(42); auto fun = [&val](auto handle){ handle.set_value(std::move(val)); }; // asymmetric transfer - auto res = make_future(wait_any<decltype(val)>([&fun](){ return awaiter_for<decltype(val)>(fun); })); + auto res = make_future(wait_for_fun<decltype(val)>(fun)); EXPECT_TRUE(res.wait_for(0ms) == std::future_status::ready); EXPECT_EQ(*res.get(), 42); } diff --git a/vespalib/src/tests/datastore/array_store/array_store_test.cpp b/vespalib/src/tests/datastore/array_store/array_store_test.cpp index 69df94f7181..210c18ddb29 100644 --- a/vespalib/src/tests/datastore/array_store/array_store_test.cpp +++ b/vespalib/src/tests/datastore/array_store/array_store_test.cpp @@ -54,12 +54,13 @@ struct ArrayStoreTest : public TestT generation(1), add_using_allocate(add_using_allocate_in) {} - ArrayStoreTest(const ArrayStoreConfig &storeCfg) + explicit ArrayStoreTest(const ArrayStoreConfig &storeCfg) : store(storeCfg, std::make_unique<MemoryAllocatorObserver>(stats)), refStore(), generation(1), add_using_allocate(false) {} + ~ArrayStoreTest() override; void assertAdd(const EntryVector &input) { EntryRef ref = add(input); assertGet(ref, input); @@ -163,6 +164,9 @@ struct ArrayStoreTest : public TestT size_t largeArraySize() const { return sizeof(LargeArray); } }; +template <typename TestT, typename EntryT, typename RefT> +ArrayStoreTest<TestT, EntryT, RefT>::~ArrayStoreTest() = default; + struct TestParam { bool add_using_allocate; TestParam(bool add_using_allocate_in) : add_using_allocate(add_using_allocate_in) {} diff --git a/vespalib/src/tests/unwind_message/unwind_message_test.cpp b/vespalib/src/tests/unwind_message/unwind_message_test.cpp index f9f4824f317..ff0ee060ded 100644 --- a/vespalib/src/tests/unwind_message/unwind_message_test.cpp +++ b/vespalib/src/tests/unwind_message/unwind_message_test.cpp @@ -20,17 +20,21 @@ struct MyObj { UnwindMessage msg1 = UnwindMessage("this SHOULD be printed (1/4)"); UnwindMessage msg2 = UnwindMessage("this should NOT be printed (1)"); UnwindMessage msg3 = UnwindMessage("this SHOULD be printed (2/4)"); - ~MyObj() { - EXPECT_EQ(std::uncaught_exceptions(), 1); - auto not_printed_1 = std::move(msg2); - try { - MyCheck my_check; - auto printed_1 = std::move(msg1); - throw E("next level"); - } catch (const E &) {} - } + MyObj(); + ~MyObj(); }; +MyObj::MyObj() = default; +MyObj::~MyObj() { + EXPECT_EQ(std::uncaught_exceptions(), 1); + auto not_printed_1 = std::move(msg2); + try { + MyCheck my_check; + auto printed_1 = std::move(msg1); + throw E("next level"); + } catch (const E &) {} +} + TEST(UnwindMessageTest, unwind_messages_are_printed_when_appropriate) { auto not_printed_5 = unwind_msg("this should NOT be printed (%d)", 5); UNWIND_MSG("this should NOT be printed (%d)", 4); diff --git a/vespalib/src/vespa/vespalib/coro/async_crypto_socket.cpp b/vespalib/src/vespa/vespalib/coro/async_crypto_socket.cpp index 4f862b48690..81b53102b4e 100644 --- a/vespalib/src/vespa/vespalib/coro/async_crypto_socket.cpp +++ b/vespalib/src/vespa/vespalib/coro/async_crypto_socket.cpp @@ -24,7 +24,9 @@ struct RawSocket : AsyncCryptoSocket { AsyncIo::SP async; SocketHandle handle; RawSocket(AsyncIo &async_in, SocketHandle handle_in) - : async(async_in.shared_from_this()), handle(std::move(handle_in)) {} + : async(async_in.shared_from_this()), handle(std::move(handle_in)) {} + RawSocket(RawSocket &&) noexcept; + ~RawSocket() override; Lazy<ssize_t> read(char *buf, size_t len) override { return async->read(handle, buf, len); } @@ -33,12 +35,16 @@ struct RawSocket : AsyncCryptoSocket { } }; +RawSocket::~RawSocket() = default; + struct SnoopedRawSocket : AsyncCryptoSocket { AsyncIo::SP async; SocketHandle handle; SmartBuffer data; SnoopedRawSocket(AsyncIo &async_in, SocketHandle handle_in) - : async(async_in.shared_from_this()), handle(std::move(handle_in)), data(0) {} + : async(async_in.shared_from_this()), handle(std::move(handle_in)), data(0) {} + SnoopedRawSocket(SnoopedRawSocket &&) noexcept; + ~SnoopedRawSocket() override; void inject_data(const char *buf, size_t len) { if (len > 0) { auto dst = data.reserve(len); @@ -68,6 +74,8 @@ struct SnoopedRawSocket : AsyncCryptoSocket { } }; +SnoopedRawSocket::~SnoopedRawSocket() = default; + struct TlsSocket : AsyncCryptoSocket { AsyncIo::SP async; SocketHandle handle; @@ -78,6 +86,8 @@ struct TlsSocket : AsyncCryptoSocket { TlsSocket(AsyncIo &async_in, SocketHandle handle_in, std::unique_ptr<CryptoCodec> codec_in) : async(async_in.shared_from_this()), handle(std::move(handle_in)), codec(std::move(codec_in)), app_input(0), enc_input(0), enc_output(0) {} + TlsSocket(TlsSocket &&) noexcept; + ~TlsSocket() override; void inject_enc_input(const char *buf, size_t len) { if (len > 0) { auto dst = enc_input.reserve(len); @@ -175,6 +185,8 @@ struct TlsSocket : AsyncCryptoSocket { } }; +TlsSocket::~TlsSocket() = default; + Lazy<AsyncCryptoSocket::UP> try_handshake(std::unique_ptr<TlsSocket> tls_socket) { bool hs_ok = co_await tls_socket->handshake(); if (hs_ok) { diff --git a/vespalib/src/vespa/vespalib/coro/async_io.cpp b/vespalib/src/vespa/vespalib/coro/async_io.cpp index 8d628fd7887..86eddfbd035 100644 --- a/vespalib/src/vespa/vespalib/coro/async_io.cpp +++ b/vespalib/src/vespa/vespalib/coro/async_io.cpp @@ -4,12 +4,28 @@ #include "detached.h" #include <vespa/vespalib/net/selector.h> #include <vespa/vespalib/util/require.h> +#include <vespa/vespalib/util/time.h> +#include <vespa/config.h> +#include <thread> #include <atomic> #include <vector> #include <map> #include <set> +#ifdef VESPA_HAS_IO_URING +#include "io_uring_thread.hpp" +namespace { +bool can_use_io_uring() { return vespalib::coro::UringProbe::check_support(); } +vespalib::coro::AsyncIo::SP create_io_uring_thread() { return std::make_shared<vespalib::coro::IoUringThread>(); } +} +#else +namespace { +bool can_use_io_uring() { return false; } +vespalib::coro::AsyncIo::SP create_io_uring_thread() { abort(); } +} +#endif + namespace vespalib::coro { namespace { @@ -195,15 +211,17 @@ struct SelectorThread : AsyncIo { writer.resume(); } } - vespalib::string get_impl_spec() override { - return "selector-thread"; - } + ImplTag get_impl_tag() override { return ImplTag::EPOLL; } Lazy<SocketHandle> accept(ServerSocket &server_socket) override { bool in_thread = co_await enter_thread(); if (in_thread) { bool can_read = co_await readable(server_socket.get_fd()); if (can_read) { - co_return server_socket.accept(); + auto res = server_socket.accept(); + if (res.valid()) { + res.set_blocking(false); + } + co_return res; } } co_return SocketHandle(-ECANCELED); @@ -325,7 +343,7 @@ SelectorThread::~SelectorThread() REQUIRE(_queue.empty()); } -} +} // <unnamed> AsyncIo::~AsyncIo() = default; AsyncIo::AsyncIo() = default; @@ -367,8 +385,11 @@ AsyncIo::Owner::~Owner() } AsyncIo::Owner -AsyncIo::create() { +AsyncIo::create(ImplTag prefer_impl) { + if (prefer_impl == ImplTag::URING && can_use_io_uring()) { + return Owner(create_io_uring_thread()); + } return Owner(std::make_shared<SelectorThread>()); } -} +} // vespalib::coro diff --git a/vespalib/src/vespa/vespalib/coro/async_io.h b/vespalib/src/vespa/vespalib/coro/async_io.h index 56d2aae7fdf..285821fef85 100644 --- a/vespalib/src/vespa/vespalib/coro/async_io.h +++ b/vespalib/src/vespa/vespalib/coro/async_io.h @@ -26,6 +26,10 @@ struct AsyncIo : std::enable_shared_from_this<AsyncIo> { virtual ~AsyncIo(); using SP = std::shared_ptr<AsyncIo>; + // tag used to separate implementations + enum class ImplTag { EPOLL, URING }; + static constexpr ImplTag default_impl() { return ImplTag::EPOLL; } + // thin wrapper used by the owner to handle lifetime class Owner { private: @@ -46,10 +50,11 @@ struct AsyncIo : std::enable_shared_from_this<AsyncIo> { }; // create an async_io 'runtime' - static Owner create(); + // note that the preferred implementation may not be available + static Owner create(ImplTag prefer_impl = default_impl()); // implementation tag - virtual vespalib::string get_impl_spec() = 0; + virtual ImplTag get_impl_tag() = 0; // api for async io used by coroutines virtual Lazy<SocketHandle> accept(ServerSocket &server_socket) = 0; diff --git a/vespalib/src/vespa/vespalib/coro/io_uring_thread.hpp b/vespalib/src/vespa/vespalib/coro/io_uring_thread.hpp new file mode 100644 index 00000000000..143e65d02d4 --- /dev/null +++ b/vespalib/src/vespa/vespalib/coro/io_uring_thread.hpp @@ -0,0 +1,295 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +// this file is included by async_io.cpp if VESPA_HAS_IO_URING is defined + +#include <liburing.h> +#include <liburing/io_uring.h> +#include <sys/eventfd.h> + +namespace vespalib::coro { +namespace { + +using Handle = std::coroutine_handle<>; +using cqe_res_t = int32_t; + +// Server sockets are always non-blocking. We need to temporarily set +// them to blocking mode to avoid that async accept return -EAGAIN. +struct BlockingGuard { + int fd; + BlockingGuard(int fd_in) : fd(fd_in) { + SocketOptions::set_blocking(fd, true); + } + ~BlockingGuard() { + SocketOptions::set_blocking(fd, false); + } +}; + +struct UringProbe { + io_uring_probe *probe; + UringProbe() : probe(io_uring_get_probe()) {} + ~UringProbe() { free(probe); } + bool check(int opcode) { + return probe && io_uring_opcode_supported(probe, opcode); + } + static bool check_support() { + UringProbe probe; + return probe.check(IORING_OP_ACCEPT) + && probe.check(IORING_OP_CONNECT) + && probe.check(IORING_OP_READ) + && probe.check(IORING_OP_WRITE); + } +}; + +struct Uring { + io_uring uring; + size_t pending; + Uring() : pending(0) { + int res = io_uring_queue_init(4096, &uring, 0); + REQUIRE_EQ(res, 0); + } + auto *get_sqe() { + auto *res = io_uring_get_sqe(&uring); + while (res == nullptr) { + auto submit_res = io_uring_submit(&uring); + REQUIRE(submit_res >= 0); + res = io_uring_get_sqe(&uring); + } + ++pending; + return res; + } + void submit_and_dispatch() { + auto res = io_uring_submit_and_wait(&uring, 1); + REQUIRE(res >= 0); + io_uring_cqe *cqe = nullptr; + while (io_uring_peek_cqe(&uring, &cqe) == 0) { + auto wf = WaitingFor<cqe_res_t>::from_pointer(io_uring_cqe_get_data(cqe)); + wf.set_value(cqe->res); + io_uring_cqe_seen(&uring, cqe); + --pending; + } + } + void drain_pending() { + while (pending > 0) { + auto res = io_uring_submit_and_wait(&uring, 1); + REQUIRE(res >= 0); + io_uring_cqe *cqe = nullptr; + while (io_uring_peek_cqe(&uring, &cqe) == 0) { + auto wf = WaitingFor<cqe_res_t>::from_pointer(io_uring_cqe_get_data(cqe)); + wf.set_value(-ECANCELED); + io_uring_cqe_seen(&uring, cqe); + --pending; + } + } + } + ~Uring() { + REQUIRE_EQ(pending, 0u); + io_uring_queue_exit(&uring); + } +}; + +auto wait_for_sqe(auto *sqe) { + return wait_for<cqe_res_t>([sqe](auto wf) + { + io_uring_sqe_set_data(sqe, wf.release()); + }); +} + +struct IoUringThread : AsyncIo { + + struct RunGuard; + using ThreadId = std::atomic<std::thread::id>; + using RunOp = WaitingFor<bool>; + + Uring _uring; + SocketHandle _event; + std::thread _thread; + ThreadId _thread_id; + std::vector<RunOp> _todo; + std::mutex _lock; + std::vector<RunOp> _queue; + + IoUringThread() + : _uring(), + _event(eventfd(0, 0)), + _thread(), + _thread_id(std::thread::id()), + _todo(), + _lock(), + _queue() + { + static_assert(ThreadId::is_always_lock_free); + REQUIRE(_event.valid()); + } + void start() override; + void main(); + void init_shutdown() override; + void fini_shutdown() override; + ~IoUringThread(); + bool running() const noexcept { + return (_thread_id.load(std::memory_order_relaxed) != std::thread::id()); + } + bool stopped() const noexcept { + return (_thread_id.load(std::memory_order_relaxed) == std::thread::id()); + } + bool in_thread() const noexcept { + return (std::this_thread::get_id() == _thread_id.load(std::memory_order_relaxed)); + } + auto protect() { return std::lock_guard(_lock); } + void wakeup() { + uint64_t value = 1; + int res = ::write(_event.get(), &value, sizeof(value)); + REQUIRE_EQ(res, int(sizeof(value))); + } + auto async_run() { + return wait_for<bool>([this](auto wf) + ->std::coroutine_handle<> + { + bool need_wakeup = false; + { + auto guard = protect(); + if (stopped()) { + wf.set_value(false); + return wf.mu(); + } + need_wakeup = _queue.empty(); + _queue.push_back(std::move(wf)); + } + if (need_wakeup) { + wakeup(); + } + return std::noop_coroutine(); + }); + } + void handle_queue(bool result) { + { + auto guard = protect(); + std::swap(_todo, _queue); + } + for (auto &&item: _todo) { + auto wf = std::move(item); + wf.set_value(result); + } + _todo.clear(); + } + ImplTag get_impl_tag() override { return ImplTag::URING; } + Lazy<SocketHandle> accept(ServerSocket &server_socket) override { + BlockingGuard blocking_guard(server_socket.get_fd()); + int res = -ECANCELED; + bool inside = in_thread() ? true : co_await async_run(); + if (inside) { + auto *sqe = _uring.get_sqe(); + io_uring_prep_accept(sqe, server_socket.get_fd(), nullptr, nullptr, 0); + res = co_await wait_for_sqe(sqe); + } + co_return SocketHandle(res); + } + Lazy<SocketHandle> connect(const SocketAddress &addr) override { + SocketHandle handle = addr.raw_socket(); + if (handle.valid()) { + bool inside = in_thread() ? true : co_await async_run(); + if (inside) { + auto *sqe = _uring.get_sqe(); + io_uring_prep_connect(sqe, handle.get(), addr.raw_addr(), addr.raw_addr_len()); + int res = co_await wait_for_sqe(sqe); + if (res < 0) { + handle.reset(res); + } + } + } + co_return handle; + } + Lazy<ssize_t> read(SocketHandle &socket, char *buf, size_t len) override { + ssize_t res = -ECANCELED; + bool inside = in_thread() ? true : co_await async_run(); + if (inside) { + auto *sqe = _uring.get_sqe(); + io_uring_prep_read(sqe, socket.get(), buf, len, 0); + res = co_await wait_for_sqe(sqe); + } + co_return res; + } + Lazy<ssize_t> write(SocketHandle &socket, const char *buf, size_t len) override { + ssize_t res = -ECANCELED; + bool inside = in_thread() ? true : co_await async_run(); + if (inside) { + auto *sqe = _uring.get_sqe(); + io_uring_prep_write(sqe, socket.get(), buf, len, 0); + res = co_await wait_for_sqe(sqe); + } + co_return res; + } + Lazy<bool> schedule() override { + co_return co_await async_run(); + } + Detached consume_events() { + uint64_t value = 0; + REQUIRE(in_thread()); + int res = sizeof(value); + while (running() && (res == sizeof(value))) { + auto *sqe = _uring.get_sqe(); + io_uring_prep_read(sqe, _event.get(), &value, sizeof(value), 0); + res = co_await wait_for_sqe(sqe); + } + } + Detached async_shutdown() { + bool inside = in_thread() ? true : co_await async_run(); + REQUIRE(inside && "unable to initialize shutdown of internal thread"); + { + auto guard = protect(); + _thread_id = std::thread::id(); + } + } +}; + +void +IoUringThread::start() +{ + _thread = std::thread(&IoUringThread::main, this); + _thread_id.wait(std::thread::id()); +} + +struct IoUringThread::RunGuard { + IoUringThread &self; + RunGuard(IoUringThread &self_in) noexcept : self(self_in) { + self._thread_id = std::this_thread::get_id(); + self._thread_id.notify_all(); + self.consume_events(); + } + ~RunGuard() { + REQUIRE(self.stopped()); + self.wakeup(); + self.handle_queue(false); + self._uring.drain_pending(); + } +}; + +void +IoUringThread::main() +{ + RunGuard guard(*this); + while (running()) { + _uring.submit_and_dispatch(); + handle_queue(true); + } +} + +void +IoUringThread::init_shutdown() +{ + async_shutdown(); +} + +void +IoUringThread::fini_shutdown() +{ + _thread.join(); +} + +IoUringThread::~IoUringThread() +{ + REQUIRE(_todo.empty()); + REQUIRE(_queue.empty()); +} + +} // <unnamed> +} // vespalib::coro diff --git a/vespalib/src/vespa/vespalib/coro/waiting_for.h b/vespalib/src/vespa/vespalib/coro/waiting_for.h index 2e11a9cb38c..d82d0779f41 100644 --- a/vespalib/src/vespa/vespalib/coro/waiting_for.h +++ b/vespalib/src/vespa/vespalib/coro/waiting_for.h @@ -48,9 +48,6 @@ public: void set_done() { _state->result.set_done(); } - std::coroutine_handle<> release_waiter() { - return std::exchange(_state->waiter, std::noop_coroutine()); - } void *release() { return std::exchange(_state, nullptr); } @@ -61,6 +58,18 @@ public: static WaitingFor from_state(PromiseState<T> &state) { return {&state}; } + // Unasking the question. This will drop the internal reference to + // the promise state and return the handle for the waiting + // coroutine. A function responsible for starting an async + // operation may chose to do 'wf.set_value(<result>)' followed by + // 'return wf.mu()' to convert the async operation to a sync + // operation by immediately resuming the waiting coroutine (by + // symmetrically transferring control to itself). + [[nodiscard]] std::coroutine_handle<> mu() { + auto handle = std::exchange(_state->waiter, std::noop_coroutine()); + _state = nullptr; + return handle; + } }; template <typename T> @@ -74,35 +83,54 @@ WaitingFor<T>::~WaitingFor() static_assert(receiver_of<WaitingFor<int>, int>); static_assert(receiver_of<WaitingFor<std::unique_ptr<int>>, std::unique_ptr<int>>); +// concept indicating that F is a function that starts an async +// operation with T as result. The result will eventually be delivered +// to the WaitingFor<T> passed to the function. +template <typename F, typename T> +concept start_async_op = requires(F &&f, std::decay_t<F> cpy, WaitingFor<T> wf) { + std::decay_t<F>(std::forward<F>(f)); + { cpy(std::move(wf)) } -> std::same_as<void>; +}; + +// concept indicating that F is a function that starts an async +// operation with T as result. The result will eventually be delivered +// to the WaitingFor<T> passed to the function. This variant will use +// symmetric transfer to switch to another coroutine as a side-effect +// of starting the async operation (and thus suspending the +// coroutine). This also enables the function to change the operation +// form async to sync by setting the value directly in the function +// and returning wf.mu() +template <typename F, typename T> +concept maybe_start_async_op = requires(F &&f, std::decay_t<F> cpy, WaitingFor<T> wf) { + std::decay_t<F>(std::forward<F>(f)); + { cpy(std::move(wf)) } -> std::same_as<std::coroutine_handle<>>; +}; + // Create a custom awaiter that will return a value of type T when the -// coroutine is resumed. The waiting coroutine will be represented as -// a WaitingFor<T> that is passed as the only parameter to 'f'. The -// return value of 'f' is returned from await_suspend, which means it -// must be void, bool or coroutine handle. If 'f' returns a value -// indicating that the coroutine should be resumed immediately, -// WaitingFor<T>::release_waiter() must be called to avoid resume -// being called as well. Note that await_ready will always return -// false, since the coroutine needs to be suspended in order to create -// the WaitingFor<T> object needed. Also, the WaitingFor<T> api -// implies that the value will be set from the outside and thus cannot -// be ready up-front. Also note that await_resume must return T by -// value, since the awaiter containing the result is a temporary -// object. +// coroutine is resumed. The operation waited upon is represented by +// the function object used to start it. Note that await_ready will +// always return false, since the coroutine needs to be suspended in +// order to create the WaitingFor<T> object needed. Also, the +// WaitingFor<T> api implies that the value will be set from the +// outside and thus cannot be ready up-front. Also note that +// await_resume must return T by value, since the awaiter containing +// the result is a temporary object. template <typename T, typename F> -auto awaiter_for(F &&f) { +requires start_async_op<F,T> || maybe_start_async_op<F,T> +auto wait_for(F &&on_suspend) { struct awaiter final : PromiseState<T> { using PromiseState<T>::result; using PromiseState<T>::waiter; - std::decay_t<F> fun; - awaiter(F &&f) : PromiseState<T>(), fun(std::forward<F>(f)) {} + std::decay_t<F> on_suspend; + awaiter(F &&on_suspend_in) : PromiseState<T>(), on_suspend(std::forward<F>(on_suspend_in)) {} bool await_ready() const noexcept { return false; } T await_resume() { return std::move(result).get_value(); } decltype(auto) await_suspend(std::coroutine_handle<> handle) __attribute__((noinline)) { waiter = handle; - return fun(WaitingFor<T>::from_state(*this)); + return on_suspend(WaitingFor<T>::from_state(*this)); } }; - return awaiter(std::forward<F>(f)); + return awaiter(std::forward<F>(on_suspend)); } } diff --git a/vespalib/src/vespa/vespalib/datastore/buffer_type.cpp b/vespalib/src/vespa/vespalib/datastore/buffer_type.cpp index d1b16e7ae7a..4a9ba2d33a8 100644 --- a/vespalib/src/vespa/vespalib/datastore/buffer_type.cpp +++ b/vespalib/src/vespa/vespalib/datastore/buffer_type.cpp @@ -1,6 +1,8 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "buffer_type.hpp" +#include <vespa/vespalib/stllike/asciistream.h> +#include <vespa/vespalib/util/exceptions.h> #include <algorithm> #include <cassert> #include <cmath> @@ -150,7 +152,24 @@ BufferTypeBase::calcArraysToAlloc(uint32_t bufferId, ElemCount elemsNeeded, bool if (result > _maxArrays) { result = _maxArrays; } - assert(result >= neededArrays); + if (result < neededArrays) { + vespalib::asciistream s; + s << "BufferTypeBase::calcArraysToAlloc(" << + "bufferId=" << bufferId << + ",elemsNeeeded=" << elemsNeeded << + ",resizing=" << (resizing ? "true" : "false") << ")" << + " wantedArrays=" << wantedArrays << + ", _arraySize=" << _arraySize << + ", _maxArrays=" << _maxArrays << + ", reservedElems=" << reservedElems << + ", liveArrays=" << liveArrays << + ", growArrays=" << growArrays << + ", usedArrays=" << usedArrays << + ", typeid(*this).name=\"" << typeid(*this).name() << "\"" << + ", newArrays=" << result << + " < neededArrays=" << neededArrays;; + throw vespalib::OverflowException(s.c_str()); + } return result; } diff --git a/vespalib/src/vespa/vespalib/datastore/unique_store.hpp b/vespalib/src/vespa/vespalib/datastore/unique_store.hpp index b8493017020..d9d5f9fee7f 100644 --- a/vespalib/src/vespa/vespalib/datastore/unique_store.hpp +++ b/vespalib/src/vespa/vespalib/datastore/unique_store.hpp @@ -10,6 +10,7 @@ #include "unique_store_builder.hpp" #include "unique_store_dictionary.hpp" #include "unique_store_enumerator.hpp" +#include "unique_store_remapper.hpp" #include <atomic> #include <algorithm> diff --git a/vespalib/src/vespa/vespalib/datastore/unique_store_remapper.h b/vespalib/src/vespa/vespalib/datastore/unique_store_remapper.h index 174c74a62d2..06c74ab0d4f 100644 --- a/vespalib/src/vespa/vespalib/datastore/unique_store_remapper.h +++ b/vespalib/src/vespa/vespalib/datastore/unique_store_remapper.h @@ -22,33 +22,12 @@ protected: EntryRefFilter _filter; std::vector<std::vector<EntryRef, allocator_large<EntryRef>>> _mapping; public: - UniqueStoreRemapper(EntryRefFilter&& filter) - : _filter(std::move(filter)), - _mapping() - { - } - virtual ~UniqueStoreRemapper() = default; - - EntryRef remap(EntryRef ref) const { - RefType internal_ref(ref); - auto &inner_mapping = _mapping[internal_ref.bufferId()]; - assert(internal_ref.offset() < inner_mapping.size()); - EntryRef mapped_ref = inner_mapping[internal_ref.offset()]; - assert(mapped_ref.valid()); - return mapped_ref; - } - - void remap(vespalib::ArrayRef<AtomicEntryRef> refs) const { - for (auto &atomic_ref : refs) { - auto ref = atomic_ref.load_relaxed(); - if (ref.valid() && _filter.has(ref)) { - atomic_ref.store_release(remap(ref)); - } - } - } + UniqueStoreRemapper(EntryRefFilter&& filter); + virtual ~UniqueStoreRemapper(); + EntryRef remap(EntryRef ref) const; + void remap(vespalib::ArrayRef<AtomicEntryRef> refs) const; const EntryRefFilter& get_entry_ref_filter() const noexcept { return _filter; } - virtual void done() = 0; }; diff --git a/vespalib/src/vespa/vespalib/datastore/unique_store_remapper.hpp b/vespalib/src/vespa/vespalib/datastore/unique_store_remapper.hpp new file mode 100644 index 00000000000..5d8a4d7ec6f --- /dev/null +++ b/vespalib/src/vespa/vespalib/datastore/unique_store_remapper.hpp @@ -0,0 +1,40 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#pragma once + +#include "unique_store_remapper.h" + +namespace vespalib::datastore { + +template <typename RefT> +UniqueStoreRemapper<RefT>::UniqueStoreRemapper(EntryRefFilter&& filter) + : _filter(std::move(filter)), + _mapping() +{ +} +template <typename RefT> +UniqueStoreRemapper<RefT>::~UniqueStoreRemapper() = default; + +template <typename RefT> +EntryRef +UniqueStoreRemapper<RefT>::remap(EntryRef ref) const { + RefType internal_ref(ref); + auto &inner_mapping = _mapping[internal_ref.bufferId()]; + assert(internal_ref.offset() < inner_mapping.size()); + EntryRef mapped_ref = inner_mapping[internal_ref.offset()]; + assert(mapped_ref.valid()); + return mapped_ref; +} + +template <typename RefT> +void +UniqueStoreRemapper<RefT>::remap(vespalib::ArrayRef<AtomicEntryRef> refs) const { + for (auto &atomic_ref : refs) { + auto ref = atomic_ref.load_relaxed(); + if (ref.valid() && _filter.has(ref)) { + atomic_ref.store_release(remap(ref)); + } + } +} + +} diff --git a/vespalib/src/vespa/vespalib/hwaccelrated/CMakeLists.txt b/vespalib/src/vespa/vespalib/hwaccelrated/CMakeLists.txt index d5a3ab429a9..84cd60887b7 100644 --- a/vespalib/src/vespa/vespalib/hwaccelrated/CMakeLists.txt +++ b/vespalib/src/vespa/vespalib/hwaccelrated/CMakeLists.txt @@ -13,5 +13,5 @@ vespa_add_library(vespalib_vespalib_hwaccelrated OBJECT ${ACCEL_FILES} DEPENDS ) -set_source_files_properties(avx2.cpp PROPERTIES COMPILE_FLAGS -march=haswell) -set_source_files_properties(avx512.cpp PROPERTIES COMPILE_FLAGS -march=skylake-avx512) +set_source_files_properties(avx2.cpp PROPERTIES COMPILE_FLAGS "-O3 -march=haswell") +set_source_files_properties(avx512.cpp PROPERTIES COMPILE_FLAGS "-O3 -march=skylake-avx512") diff --git a/vespalib/src/vespa/vespalib/net/http/json_handler_repo.cpp b/vespalib/src/vespa/vespalib/net/http/json_handler_repo.cpp index 07b9306b5dc..3f92cffb4af 100644 --- a/vespalib/src/vespa/vespalib/net/http/json_handler_repo.cpp +++ b/vespalib/src/vespa/vespalib/net/http/json_handler_repo.cpp @@ -46,17 +46,21 @@ JsonHandlerRepo::State::unbind(size_t my_seq) { //----------------------------------------------------------------------------- +JsonHandlerRepo::JsonHandlerRepo() + : _state(std::make_shared<State>()) +{} +JsonHandlerRepo::~JsonHandlerRepo() = default; JsonHandlerRepo::Token::UP JsonHandlerRepo::bind(vespalib::stringref path_prefix, const JsonGetHandler &get_handler) { - return Token::UP(new Unbinder(_state, _state->bind(path_prefix, get_handler))); + return std::make_unique<Unbinder>(_state, _state->bind(path_prefix, get_handler)); } JsonHandlerRepo::Token::UP JsonHandlerRepo::add_root_resource(vespalib::stringref path) { - return Token::UP(new Unbinder(_state, _state->add_root_resource(path))); + return std::make_unique<Unbinder>(_state, _state->add_root_resource(path)); } std::vector<vespalib::string> diff --git a/vespalib/src/vespa/vespalib/net/http/json_handler_repo.h b/vespalib/src/vespa/vespalib/net/http/json_handler_repo.h index 4ca3e061c71..46ef3bd762a 100644 --- a/vespalib/src/vespa/vespalib/net/http/json_handler_repo.h +++ b/vespalib/src/vespa/vespalib/net/http/json_handler_repo.h @@ -25,7 +25,7 @@ class JsonHandlerRepo : public JsonGetHandler public: struct Token { using UP = std::unique_ptr<Token>; - virtual ~Token() {} + virtual ~Token() = default; }; private: @@ -78,14 +78,13 @@ private: std::shared_ptr<State> _state; public: - JsonHandlerRepo() : _state(std::make_shared<State>()) {} - Token::UP bind(vespalib::stringref path_prefix, - const JsonGetHandler &get_handler); + JsonHandlerRepo(); + ~JsonHandlerRepo(); + Token::UP bind(vespalib::stringref path_prefix, const JsonGetHandler &get_handler); Token::UP add_root_resource(vespalib::stringref path); - std::vector<vespalib::string> get_root_resources() const; - vespalib::string get(const vespalib::string &host, - const vespalib::string &path, - const std::map<vespalib::string,vespalib::string> ¶ms) const override; + [[nodiscard]] std::vector<vespalib::string> get_root_resources() const; + [[nodiscard]] vespalib::string get(const vespalib::string &host, const vespalib::string &path, + const std::map<vespalib::string,vespalib::string> ¶ms) const override; }; } // namespace vespalib diff --git a/vespalib/src/vespa/vespalib/net/socket_address.cpp b/vespalib/src/vespa/vespalib/net/socket_address.cpp index c1fb1cb0858..4c552b03de7 100644 --- a/vespalib/src/vespa/vespalib/net/socket_address.cpp +++ b/vespalib/src/vespa/vespalib/net/socket_address.cpp @@ -152,14 +152,21 @@ SocketAddress::spec() const } SocketHandle -SocketAddress::connect(const std::function<bool(SocketHandle&)> &tweak) const +SocketAddress::raw_socket() const { if (valid()) { - SocketHandle handle(socket(_addr.ss_family, SOCK_STREAM, 0)); - if (handle.valid() && tweak(handle)) { - if ((::connect(handle.get(), addr(), _size) == 0) || (errno == EINPROGRESS)) { - return handle; - } + return SocketHandle(::socket(_addr.ss_family, SOCK_STREAM, 0)); + } + return SocketHandle(); +} + +SocketHandle +SocketAddress::connect(const std::function<bool(SocketHandle&)> &tweak) const +{ + SocketHandle handle = raw_socket(); + if (handle.valid() && tweak(handle)) { + if ((::connect(handle.get(), addr(), _size) == 0) || (errno == EINPROGRESS)) { + return handle; } } return SocketHandle(); @@ -168,20 +175,18 @@ SocketAddress::connect(const std::function<bool(SocketHandle&)> &tweak) const SocketHandle SocketAddress::listen(int backlog) const { - if (valid()) { - SocketHandle handle(socket(_addr.ss_family, SOCK_STREAM, 0)); - if (handle.valid()) { - if (is_ipv6()) { - handle.set_ipv6_only(false); - } - if (port() > 0) { - handle.set_reuse_addr(true); - } - if ((bind(handle.get(), addr(), _size) == 0) && - (::listen(handle.get(), backlog) == 0)) - { - return handle; - } + SocketHandle handle = raw_socket(); + if (handle.valid()) { + if (is_ipv6()) { + handle.set_ipv6_only(false); + } + if (port() > 0) { + handle.set_reuse_addr(true); + } + if ((bind(handle.get(), addr(), _size) == 0) && + (::listen(handle.get(), backlog) == 0)) + { + return handle; } } return SocketHandle(); diff --git a/vespalib/src/vespa/vespalib/net/socket_address.h b/vespalib/src/vespa/vespalib/net/socket_address.h index 6dfe843c12e..88da83d902b 100644 --- a/vespalib/src/vespa/vespalib/net/socket_address.h +++ b/vespalib/src/vespa/vespalib/net/socket_address.h @@ -35,6 +35,8 @@ public: memcpy(this, &rhs, sizeof(SocketAddress)); return *this; } + const sockaddr *raw_addr() const { return addr(); } + socklen_t raw_addr_len() const { return _size; } bool valid() const { return (_size >= sizeof(sa_family_t)); } bool is_ipv4() const { return (valid() && (_addr.ss_family == AF_INET)); } bool is_ipv6() const { return (valid() && (_addr.ss_family == AF_INET6)); } @@ -47,6 +49,7 @@ public: vespalib::string path() const; vespalib::string name() const; vespalib::string spec() const; + SocketHandle raw_socket() const; SocketHandle connect(const std::function<bool(SocketHandle&)> &tweak) const; SocketHandle connect() const { return connect([](SocketHandle&) noexcept { return true; }); } SocketHandle connect_async() const { diff --git a/vespalib/src/vespa/vespalib/testkit/test_master.cpp b/vespalib/src/vespa/vespalib/testkit/test_master.cpp index 434d08d6c7a..bacb1051cf3 100644 --- a/vespalib/src/vespa/vespalib/testkit/test_master.cpp +++ b/vespalib/src/vespa/vespalib/testkit/test_master.cpp @@ -31,16 +31,22 @@ TestMaster TestMaster::master; //----------------------------------------------------------------------------- -__thread TestMaster::ThreadState *TestMaster::_threadState = 0; +__thread TestMaster::ThreadState *TestMaster::_threadState = nullptr; //----------------------------------------------------------------------------- - +TestMaster::TraceItem::TraceItem(const std::string &file_in, uint32_t line_in, const std::string &msg_in) + : file(file_in), line(line_in), msg(msg_in) +{} +TestMaster::TraceItem::TraceItem(TraceItem &&) noexcept = default; +TestMaster::TraceItem & TestMaster::TraceItem::operator=(TraceItem &&) noexcept = default; +TestMaster::TraceItem::TraceItem(const TraceItem &) = default; +TestMaster::TraceItem & TestMaster::TraceItem::operator=(const TraceItem &) = default; TestMaster::TraceItem::~TraceItem() = default; TestMaster::ThreadState & TestMaster::threadState(const lock_guard &) { - if (_threadState == 0) { + if (_threadState == nullptr) { std::ostringstream threadName; threadName << "thread-" << _threadStorage.size(); _threadStorage.push_back(std::make_unique<ThreadState>(threadName.str())); @@ -52,7 +58,7 @@ TestMaster::threadState(const lock_guard &) TestMaster::ThreadState & TestMaster::threadState() { - if (_threadState != 0) { + if (_threadState != nullptr) { return *_threadState; } { @@ -87,7 +93,7 @@ TestMaster::printDiff(const lock_guard &guard, const std::string &lhs, const std::string &rhs) { ThreadState &thread = threadState(guard); - if (_state.lhsFile == NULL || _state.rhsFile == NULL) { + if (_state.lhsFile == nullptr || _state.rhsFile == nullptr) { fprintf(stderr, "lhs: %s\n" "rhs: %s\n", @@ -123,13 +129,13 @@ TestMaster::handleFailure(const lock_guard &guard, bool fatal) void TestMaster::closeDebugFiles(const lock_guard &) { - if (_state.lhsFile != NULL) { + if (_state.lhsFile != nullptr) { fclose(_state.lhsFile); - _state.lhsFile = NULL; + _state.lhsFile = nullptr; } - if (_state.rhsFile != NULL) { + if (_state.rhsFile != nullptr) { fclose(_state.rhsFile); - _state.rhsFile = NULL; + _state.rhsFile = nullptr; } } @@ -137,8 +143,8 @@ void TestMaster::importThreads(const lock_guard &) { size_t importCnt = 0; - for (size_t i = 0; i < _threadStorage.size(); ++i) { - ThreadState &thread = *_threadStorage[i]; + for (const auto & i : _threadStorage) { + ThreadState &thread = *i; _state.passCnt += thread.passCnt; importCnt += thread.passCnt; thread.passCnt = 0; @@ -245,7 +251,7 @@ void TestMaster::awaitThreadBarrier(const char *file, uint32_t line) { ThreadState &thread = threadState(); - if (thread.barrier == 0) { + if (thread.barrier == nullptr) { return; } if (!thread.barrier->await()) { @@ -275,7 +281,7 @@ TestMaster::Progress TestMaster::getProgress() { lock_guard guard(_lock); - return Progress(_state.passCnt, _state.failCnt); + return {_state.passCnt, _state.failCnt}; } void @@ -286,7 +292,7 @@ TestMaster::openDebugFiles(const std::string &lhsFile, closeDebugFiles(guard); _state.lhsFile = fopen(lhsFile.c_str(), "w"); _state.rhsFile = fopen(rhsFile.c_str(), "w"); - if (_state.lhsFile == NULL || _state.rhsFile == NULL) { + if (_state.lhsFile == nullptr || _state.rhsFile == nullptr) { closeDebugFiles(guard); fprintf(stderr, "%s: Warn: could not open debug files (%s, %s)\n", _name.c_str(), lhsFile.c_str(), rhsFile.c_str()); @@ -299,7 +305,7 @@ TestMaster::openDebugFiles(const std::string &lhsFile, void TestMaster::pushState(const char *file, uint32_t line, const char *msg) { - threadState().traceStack.push_back(TraceItem(skip_path(file), line, msg)); + threadState().traceStack.emplace_back(skip_path(file), line, msg); } void diff --git a/vespalib/src/vespa/vespalib/testkit/test_master.h b/vespalib/src/vespa/vespalib/testkit/test_master.h index 01ea94c074d..b841197f2f9 100644 --- a/vespalib/src/vespa/vespalib/testkit/test_master.h +++ b/vespalib/src/vespa/vespalib/testkit/test_master.h @@ -36,9 +36,11 @@ public: std::string file; uint32_t line; std::string msg; - TraceItem(const std::string &file_in, uint32_t line_in, - const std::string &msg_in) - : file(file_in), line(line_in), msg(msg_in) {} + TraceItem(const std::string &file_in, uint32_t line_in, const std::string &msg_in); + TraceItem(TraceItem &&) noexcept; + TraceItem & operator=(TraceItem &&) noexcept; + TraceItem(const TraceItem &); + TraceItem & operator=(const TraceItem &); ~TraceItem(); }; diff --git a/vespalib/src/vespa/vespalib/trace/tracenode.cpp b/vespalib/src/vespa/vespalib/trace/tracenode.cpp index 881b9cc3300..f05953e62df 100644 --- a/vespalib/src/vespa/vespalib/trace/tracenode.cpp +++ b/vespalib/src/vespa/vespalib/trace/tracenode.cpp @@ -67,6 +67,7 @@ TraceNode::TraceNode(const TraceNode &rhs) } TraceNode::TraceNode(TraceNode &&) noexcept = default; +TraceNode & TraceNode::operator =(TraceNode &&) noexcept = default; TraceNode & TraceNode::operator =(const TraceNode &) = default; TraceNode::~TraceNode() = default; diff --git a/vespalib/src/vespa/vespalib/trace/tracenode.h b/vespalib/src/vespa/vespalib/trace/tracenode.h index 4062dc78d3f..c38c5236e35 100644 --- a/vespalib/src/vespa/vespalib/trace/tracenode.h +++ b/vespalib/src/vespa/vespalib/trace/tracenode.h @@ -51,7 +51,7 @@ public: TraceNode & operator =(const TraceNode &); TraceNode(TraceNode &&) noexcept; - TraceNode & operator =(TraceNode &&) noexcept = default; + TraceNode & operator =(TraceNode &&) noexcept; ~TraceNode(); /** diff --git a/vespalib/src/vespa/vespalib/util/cpu_usage.cpp b/vespalib/src/vespa/vespalib/util/cpu_usage.cpp index 97dbedad66f..89ba03fdab9 100644 --- a/vespalib/src/vespa/vespalib/util/cpu_usage.cpp +++ b/vespalib/src/vespa/vespalib/util/cpu_usage.cpp @@ -141,6 +141,8 @@ CpuUsage::CpuUsage() { } +CpuUsage::~CpuUsage() = default; + CpuUsage & CpuUsage::self() { diff --git a/vespalib/src/vespa/vespalib/util/cpu_usage.h b/vespalib/src/vespa/vespalib/util/cpu_usage.h index 31309d88f5f..a5687031522 100644 --- a/vespalib/src/vespa/vespalib/util/cpu_usage.h +++ b/vespalib/src/vespa/vespalib/util/cpu_usage.h @@ -166,6 +166,7 @@ private: CpuUsage(const CpuUsage &) = delete; CpuUsage &operator=(CpuUsage &&) = delete; CpuUsage &operator=(const CpuUsage &) = delete; + ~CpuUsage(); static CpuUsage &self(); diff --git a/vespalib/src/vespa/vespalib/util/jsonstream.cpp b/vespalib/src/vespa/vespalib/util/jsonstream.cpp index 135611e975f..e8814ee95f6 100644 --- a/vespalib/src/vespa/vespalib/util/jsonstream.cpp +++ b/vespalib/src/vespa/vespalib/util/jsonstream.cpp @@ -8,6 +8,19 @@ namespace vespalib { +JsonStream::StateEntry::StateEntry() noexcept + : state(State::ROOT), object_key(""), array_index(size_t(0)) +{} +JsonStream::StateEntry::StateEntry(State s) noexcept + : state(s), object_key(""), array_index(size_t(0)) +{} +JsonStream::StateEntry::StateEntry(State s, stringref key) noexcept + : state(s), object_key(key), array_index(size_t(0)) +{} +JsonStream::StateEntry::StateEntry(const StateEntry &) noexcept = default; +JsonStream::StateEntry & JsonStream::StateEntry::operator =(const StateEntry &) noexcept = default; +JsonStream::StateEntry::~StateEntry() = default; + const char* JsonStream::getStateName(const State& s) { switch (s) { diff --git a/vespalib/src/vespa/vespalib/util/jsonstream.h b/vespalib/src/vespa/vespalib/util/jsonstream.h index e41c4238b09..b85ad7f5486 100644 --- a/vespalib/src/vespa/vespalib/util/jsonstream.h +++ b/vespalib/src/vespa/vespalib/util/jsonstream.h @@ -44,12 +44,12 @@ class JsonStream : public JsonStreamTypes { string object_key; size_t array_index; - StateEntry() noexcept - : state(State::ROOT), object_key(""), array_index(size_t(0)) {} - StateEntry(State s) - : state(s), object_key(""), array_index(size_t(0)) {} - StateEntry(State s, stringref key) - : state(s), object_key(key), array_index(size_t(0)) {} + StateEntry() noexcept; + StateEntry(State s) noexcept; + StateEntry(State s, stringref key) noexcept; + StateEntry(const StateEntry &) noexcept; + StateEntry & operator =(const StateEntry &) noexcept; + ~StateEntry(); }; std::vector<StateEntry> _state; diff --git a/vespalib/src/vespa/vespalib/util/memory_trap.cpp b/vespalib/src/vespa/vespalib/util/memory_trap.cpp index 1403fc5d222..0c0b4a29363 100644 --- a/vespalib/src/vespa/vespalib/util/memory_trap.cpp +++ b/vespalib/src/vespa/vespalib/util/memory_trap.cpp @@ -7,7 +7,6 @@ #include <cstdint> #include <cstdlib> #include <cstring> -#include <malloc.h> #include <unistd.h> #include <sys/mman.h> @@ -153,7 +152,7 @@ void MemoryRangeTrapper::unprotect_buffer_to_read_and_write() { /* no-op */ } #endif HeapMemoryTrap::HeapMemoryTrap(size_t trap_4k_pages) - : _trap_buf(static_cast<char*>(memalign(4096, trap_4k_pages * 4096))), + : _trap_buf(static_cast<char*>(aligned_alloc(4096, trap_4k_pages * 4096))), _trapper(_trap_buf, _trap_buf ? trap_4k_pages * 4096 : 0) { } diff --git a/vespalog/src/vespa/log/log_message.cpp b/vespalog/src/vespa/log/log_message.cpp index 10fb77fcf26..e1356ce5f75 100644 --- a/vespalog/src/vespa/log/log_message.cpp +++ b/vespalog/src/vespa/log/log_message.cpp @@ -120,9 +120,10 @@ LogMessage::LogMessage(int64_t time_nanos_in, { } +LogMessage::LogMessage(LogMessage &&) noexcept = default; +LogMessage & LogMessage::operator=(LogMessage &&) noexcept = default; LogMessage::~LogMessage() = default; - /* * Parse log line to populate log message class. The parsing is based on * LegacyForwarder in logd. diff --git a/vespalog/src/vespa/log/log_message.h b/vespalog/src/vespa/log/log_message.h index 12a0ac09443..6966ad04f74 100644 --- a/vespalog/src/vespa/log/log_message.h +++ b/vespalog/src/vespa/log/log_message.h @@ -23,6 +23,8 @@ class LogMessage { public: LogMessage(); + LogMessage(LogMessage &&) noexcept; + LogMessage & operator=(LogMessage &&) noexcept; LogMessage(int64_t time_nanos_in, const std::string& hostname_in, int32_t process_id_in, diff --git a/vespamalloc/src/tests/test1/new_test.cpp b/vespamalloc/src/tests/test1/new_test.cpp index dfa67f2aa7c..d41258fb115 100644 --- a/vespamalloc/src/tests/test1/new_test.cpp +++ b/vespamalloc/src/tests/test1/new_test.cpp @@ -1,6 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include <vespa/vespalib/testkit/testapp.h> #include <vespa/vespalib/util/size_literals.h> +#include <vespa/vespalib/util/optimized.h> #include <vespa/log/log.h> #include <malloc.h> #include <dlfcn.h> @@ -8,6 +9,20 @@ LOG_SETUP("new_test"); +void *wrap_memalign_real(size_t alignment, size_t size) +{ + return memalign(alignment, size); +} + +void* (*wrap_memalign)(size_t alignment, size_t size) = wrap_memalign_real; + +void *wrap_aligned_alloc_real(size_t alignment, size_t size) +{ + return aligned_alloc(alignment, size); +} + +void* (*wrap_aligned_alloc)(size_t alignment, size_t size) = wrap_aligned_alloc_real; + void cmp(const void *a, const void *b) { EXPECT_EQUAL(a, b); } @@ -250,4 +265,43 @@ TEST("test realloc large buffers") { EXPECT_EQUAL(1, mallopt(M_MMAP_THRESHOLD, 1_Gi)); } +void verify_alignment(void * ptr, size_t align, size_t min_sz) { + EXPECT_NOT_EQUAL(ptr, nullptr); + EXPECT_EQUAL(0u, size_t(ptr) & (align-1)); + assert(0ul == (size_t(ptr) & (align-1))); + EXPECT_GREATER_EQUAL(malloc_usable_size(ptr), min_sz); + free(ptr); +} + +TEST("test memalign") { + verify_alignment(wrap_memalign(0, 0), 1, 1); + verify_alignment(wrap_memalign(0, 1), 1, 1); + verify_alignment(wrap_memalign(1, 0), 1, 1); + + for (size_t align : {3,7,19}) { + // According to man pages these should fail, but it seems it rounds up and does best effort + verify_alignment(wrap_memalign(align, 73), 1ul << vespalib::Optimized::msbIdx(align), 73); + } + for (size_t align : {1, 2, 4, 8, 16, 32, 64, 128, 256, 512, 1024, 2048, 4096, 8192, 16384, 32768, 65536}) { + verify_alignment(wrap_memalign(align, 1), align, 1); + } +} + +TEST("test aligned_alloc") { + verify_alignment(wrap_aligned_alloc(0, 0), 1, 1); + verify_alignment(wrap_aligned_alloc(0, 1), 1, 1); + verify_alignment(wrap_aligned_alloc(1, 0), 1, 1); + for (size_t align : {3,7,19}) { + // According to man pages these should fail, but it seems it rounds up and does best effort + verify_alignment(wrap_aligned_alloc(align, align*7), 1ul << vespalib::Optimized::msbIdx(align), align*7); + } + for (size_t align : {1, 2, 4, 8, 16, 32, 64, 128, 256, 512, 1024, 2048, 4096, 8192, 16384, 32768, 65536}) { + verify_alignment(wrap_aligned_alloc(align, align*7), align, align*7); + } + for (size_t sz : {31,33,63}) { + // According to man pages these should fail, but it seems it rounds up and does best effort + verify_alignment(wrap_aligned_alloc(32, sz), 32, sz); + } +} + TEST_MAIN() { TEST_RUN_ALL(); } diff --git a/vespamalloc/src/vespamalloc/malloc/malloc.h b/vespamalloc/src/vespamalloc/malloc/malloc.h index 0ecdb8c58e1..5867852db31 100644 --- a/vespamalloc/src/vespamalloc/malloc/malloc.h +++ b/vespamalloc/src/vespamalloc/malloc/malloc.h @@ -73,7 +73,7 @@ public: size_t getMinSizeForAlignment(size_t align, size_t sz) const { return MemBlockPtrT::getMinSizeForAlignment(align, sz); } size_t sizeClass(const void *ptr) const { return _segment.sizeClass(ptr); } size_t usable_size(void *ptr) const { - return MemBlockPtrT::usable_size(ptr, _segment.getMaxSize<MemBlockPtrT>(ptr)); + return MemBlockPtrT::usable_size(ptr, _segment); } void * calloc(size_t nelm, size_t esz) { diff --git a/vespamalloc/src/vespamalloc/malloc/memblock.h b/vespamalloc/src/vespamalloc/malloc/memblock.h index f7b5923ecff..ee090de6f3d 100644 --- a/vespamalloc/src/vespamalloc/malloc/memblock.h +++ b/vespamalloc/src/vespamalloc/malloc/memblock.h @@ -18,7 +18,7 @@ public: SizeClassSpan = (MaxSizeClassMultiAllocC-MinSizeClassC) }; MemBlockT() : _ptr(nullptr) { } - MemBlockT(void * p) : _ptr(p) { } + explicit MemBlockT(void * p) : _ptr(p) { } MemBlockT(void * p, size_t /*sz*/) : _ptr(p) { } MemBlockT(void * p, size_t, bool) : _ptr(p) { } template<typename T> @@ -34,8 +34,9 @@ public: void setThreadId(uint32_t ) { } void free() { } size_t size() const { return 0; } - static size_t usable_size(void *, size_t classSize) { - return classSize; + template<typename T> + static size_t usable_size(void * ptr, const T & segment) { + return segment.template getMaxSize<MemBlockT>(ptr); } bool allocated() const { return false; } uint32_t threadId() const { return 0; } diff --git a/vespamalloc/src/vespamalloc/malloc/memblockboundscheck.h b/vespamalloc/src/vespamalloc/malloc/memblockboundscheck.h index 9d46dbce378..67b98701d8e 100644 --- a/vespamalloc/src/vespamalloc/malloc/memblockboundscheck.h +++ b/vespamalloc/src/vespamalloc/malloc/memblockboundscheck.h @@ -103,8 +103,11 @@ public: && ((p[3] == ALLOC_MAGIC) || (p[3] == FREE_MAGIC)) && *(reinterpret_cast<const unsigned *> ((const char*)_ptr + size() + alignment() + StackTraceLen*sizeof(void *))) == TAIL_MAGIC; } - static size_t usable_size(void *ptr, size_t ) { - return MemBlockBoundsCheckBaseT(ptr).size(); + template<typename T> + static size_t usable_size(void *ptr, const T & segment) { + MemBlockBoundsCheckBaseT mem(ptr); + mem.readjustAlignment(segment); + return mem.size(); } bool validAlloc1() const { unsigned *p((unsigned*)_ptr); diff --git a/vespamalloc/src/vespamalloc/malloc/overload.h b/vespamalloc/src/vespamalloc/malloc/overload.h index 6650f107ca9..e209d1b6df5 100644 --- a/vespamalloc/src/vespamalloc/malloc/overload.h +++ b/vespamalloc/src/vespamalloc/malloc/overload.h @@ -1,8 +1,9 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #pragma once +#include "common.h" #include <dlfcn.h> -#include <errno.h> +#include <cerrno> #include <new> #include <cstdlib> #include <malloc.h> @@ -174,15 +175,22 @@ void * reallocarray(void * ptr, size_t nemb, size_t elemSize) __THROW } void* memalign(size_t align, size_t sz) __THROW __attribute__((visibility ("default"))); -void* memalign(size_t align, size_t sz) __THROW +void* memalign(size_t align_in, size_t sz) __THROW { - void *ptr(nullptr); + size_t align = (align_in == 0) ? 1 : 1ul << vespamalloc::msbIdx(align_in*2 - 1); size_t align_1(align - 1); - if ((align & (align_1)) == 0) { - ptr = vespamalloc::_GmemP->malloc(vespamalloc::_GmemP->getMinSizeForAlignment(align, sz)); - ptr = (void *) ((size_t(ptr) + align_1) & ~align_1); - } - return ptr; + void * ptr = vespamalloc::_GmemP->malloc(vespamalloc::_GmemP->getMinSizeForAlignment(align, sz)); + return (void *) ((size_t(ptr) + align_1) & ~align_1); +} + +void *aligned_alloc (size_t align, size_t sz) __THROW __attribute__((visibility ("default"))); +void *aligned_alloc (size_t align_in, size_t sz_in) __THROW +{ + size_t align = (align_in == 0) ? 1 : 1ul << vespamalloc::msbIdx(align_in*2 - 1); + size_t align_1(align - 1); + size_t sz = ((sz_in - 1) + align) & ~align_1; + void * ptr = vespamalloc::_GmemP->malloc(vespamalloc::_GmemP->getMinSizeForAlignment(align, sz)); + return (void *) ((size_t(ptr) + align_1) & ~align_1); } int posix_memalign(void** ptr, size_t align, size_t sz) __THROW __nonnull((1)) __attribute__((visibility ("default"))); |