diff options
56 files changed, 837 insertions, 102 deletions
diff --git a/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java b/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java index 638a66db03f..19daa43de13 100644 --- a/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java +++ b/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java @@ -109,6 +109,7 @@ public interface ModelContext { @ModelFeatureFlag(owners = {"vekterli"}) default String mergeThrottlingPolicy() { throw new UnsupportedOperationException("TODO specify default value"); } @ModelFeatureFlag(owners = {"vekterli"}) default double persistenceThrottlingWsDecrementFactor() { throw new UnsupportedOperationException("TODO specify default value"); } @ModelFeatureFlag(owners = {"vekterli"}) default double persistenceThrottlingWsBackoff() { throw new UnsupportedOperationException("TODO specify default value"); } + @ModelFeatureFlag(owners = {"geirst", "vekterli"}) default boolean inhibitDefaultMergesWhenGlobalMergesPending() { return false; } @ModelFeatureFlag(owners = {"arnej"}) default boolean useQrserverServiceName() { return true; } @ModelFeatureFlag(owners = {"bjorncs", "baldersheim"}) default boolean enableJdiscPreshutdownCommand() { return true; } } diff --git a/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java b/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java index c9ddcdf38eb..0cde8c361a7 100644 --- a/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java +++ b/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java @@ -72,6 +72,7 @@ public class TestProperties implements ModelContext.Properties, ModelContext.Fea private String mergeThrottlingPolicy = "STATIC"; private double persistenceThrottlingWsDecrementFactor = 1.2; private double persistenceThrottlingWsBackoff = 0.95; + private boolean inhibitDefaultMergesWhenGlobalMergesPending = false; private boolean useV8GeoPositions = false; private List<String> environmentVariables = List.of(); @@ -126,6 +127,7 @@ public class TestProperties implements ModelContext.Properties, ModelContext.Fea @Override public String mergeThrottlingPolicy() { return mergeThrottlingPolicy; } @Override public double persistenceThrottlingWsDecrementFactor() { return persistenceThrottlingWsDecrementFactor; } @Override public double persistenceThrottlingWsBackoff() { return persistenceThrottlingWsBackoff; } + @Override public boolean inhibitDefaultMergesWhenGlobalMergesPending() { return inhibitDefaultMergesWhenGlobalMergesPending; } @Override public boolean useV8GeoPositions() { return useV8GeoPositions; } @Override public List<String> environmentVariables() { return environmentVariables; } @@ -330,6 +332,11 @@ public class TestProperties implements ModelContext.Properties, ModelContext.Fea return this; } + public TestProperties inhibitDefaultMergesWhenGlobalMergesPending(boolean value) { + this.inhibitDefaultMergesWhenGlobalMergesPending = value; + return this; + } + public TestProperties setUseV8GeoPositions(boolean value) { this.useV8GeoPositions = value; return this; diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/AddExtraFieldsToDocument.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/AddExtraFieldsToDocument.java index 9d933b8439d..51defffa00b 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/AddExtraFieldsToDocument.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/AddExtraFieldsToDocument.java @@ -26,11 +26,6 @@ public class AddExtraFieldsToDocument extends Processor { super(schema, deployLogger, rankProfileRegistry, queryProfiles); } - //TODO This is a tempoarry hack to avoid producing illegal code for fields not wanted anyway. - private boolean dirtyLegalFieldNameCheck(String fieldName) { - return ! fieldName.contains(".") && !"rankfeatures".equals(fieldName) && !"summaryfeatures".equals(fieldName); - } - @Override public void process(boolean validate, boolean documentsOnly) { SDDocumentType document = schema.getDocument(); @@ -38,10 +33,21 @@ public class AddExtraFieldsToDocument extends Processor { for (SDField field : schema.extraFieldList()) { addSdField(schema, document, field, validate); } - //TODO Vespa 8 or sooner we should avoid the dirty addition of fields from dirty 'default' summary to document at all - for (SummaryField field : schema.getSummary("default").getSummaryFields().values()) { - if (dirtyLegalFieldNameCheck(field.getName())) { - addSummaryField(schema, document, field, validate); + for (var docsum : schema.getSummaries().values()) { + for (var summaryField : docsum.getSummaryFields().values()) { + switch (summaryField.getTransform()) { + case NONE: + case BOLDED: + case DYNAMICBOLDED: + case DYNAMICTEASER: + case TEXTEXTRACTOR: + addSummaryField(schema, document, summaryField, validate); + break; + default: + // skip: generated from attribute or similar, + // so does not need to be included as an extra + // field in the document type + } } } } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/DistributorCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/content/DistributorCluster.java index b8d2a4f91fe..3f01f5610f1 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/content/DistributorCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/content/DistributorCluster.java @@ -43,6 +43,7 @@ public class DistributorCluster extends AbstractConfigProducer<Distributor> impl private final boolean useThreePhaseUpdates; private final int maxActivationInhibitedOutOfSyncGroups; private final boolean unorderedMergeChaining; + private final boolean inhibitDefaultMergesWhenGlobalMergesPending; public static class Builder extends VespaDomBuilder.DomConfigProducerBuilder<DistributorCluster> { @@ -106,11 +107,12 @@ public class DistributorCluster extends AbstractConfigProducer<Distributor> impl boolean useThreePhaseUpdates = deployState.getProperties().featureFlags().useThreePhaseUpdates(); int maxInhibitedGroups = deployState.getProperties().featureFlags().maxActivationInhibitedOutOfSyncGroups(); boolean unorderedMergeChaining = deployState.getProperties().featureFlags().unorderedMergeChaining(); + boolean inhibitDefaultMerges = deployState.getProperties().featureFlags().inhibitDefaultMergesWhenGlobalMergesPending(); return new DistributorCluster(parent, new BucketSplitting.Builder().build(new ModelElement(producerSpec)), gc, hasIndexedDocumentType, useThreePhaseUpdates, - maxInhibitedGroups, unorderedMergeChaining); + maxInhibitedGroups, unorderedMergeChaining, inhibitDefaultMerges); } } @@ -118,7 +120,8 @@ public class DistributorCluster extends AbstractConfigProducer<Distributor> impl GcOptions gc, boolean hasIndexedDocumentType, boolean useThreePhaseUpdates, int maxActivationInhibitedOutOfSyncGroups, - boolean unorderedMergeChaining) + boolean unorderedMergeChaining, + boolean inhibitDefaultMergesWhenGlobalMergesPending) { super(parent, "distributor"); this.parent = parent; @@ -128,6 +131,7 @@ public class DistributorCluster extends AbstractConfigProducer<Distributor> impl this.useThreePhaseUpdates = useThreePhaseUpdates; this.maxActivationInhibitedOutOfSyncGroups = maxActivationInhibitedOutOfSyncGroups; this.unorderedMergeChaining = unorderedMergeChaining; + this.inhibitDefaultMergesWhenGlobalMergesPending = inhibitDefaultMergesWhenGlobalMergesPending; } @Override @@ -142,6 +146,7 @@ public class DistributorCluster extends AbstractConfigProducer<Distributor> impl builder.enable_metadata_only_fetch_phase_for_inconsistent_updates(useThreePhaseUpdates); builder.max_activation_inhibited_out_of_sync_groups(maxActivationInhibitedOutOfSyncGroups); builder.use_unordered_merge_chaining(unorderedMergeChaining); + builder.inhibit_default_merges_when_global_merges_pending(inhibitDefaultMergesWhenGlobalMergesPending); bucketSplitting.getConfig(builder); } diff --git a/config-model/src/test/derived/multiplesummaries/ilscripts.cfg b/config-model/src/test/derived/multiplesummaries/ilscripts.cfg index 64d4bd3ba0a..5434b0770f7 100644 --- a/config-model/src/test/derived/multiplesummaries/ilscripts.cfg +++ b/config-model/src/test/derived/multiplesummaries/ilscripts.cfg @@ -12,14 +12,16 @@ ilscript[].docfield[7] "f" ilscript[].docfield[8] "g" ilscript[].docfield[9] "h" ilscript[].docfield[10] "loc" +ilscript[].docfield[11] "mytags" ilscript[].content[0] "clear_state | guard { input loc | to_pos | zcurve | attribute loc_pos_zcurve; }" ilscript[].content[1] "clear_state | guard { input a | tokenize normalize stem:\"BEST\" | summary abolded2 | summary aboldeddynamic | summary adynamic2 | attribute a; }" ilscript[].content[2] "clear_state | guard { input adynamic | tokenize normalize stem:\"BEST\" | summary adynamic | attribute adynamic; }" ilscript[].content[3] "clear_state | guard { input abolded | tokenize normalize stem:\"BEST\" | summary abolded | attribute abolded; }" -ilscript[].content[4] "clear_state | guard { input b | summary b; }" +ilscript[].content[4] "clear_state | guard { input b | summary anotherb | summary b; }" ilscript[].content[5] "clear_state | guard { input c | summary c | attribute c; }" ilscript[].content[6] "clear_state | guard { input d | tokenize normalize stem:\"BEST\" | summary d; }" ilscript[].content[7] "clear_state | guard { input e | tokenize normalize stem:\"BEST\" | summary dynamice | summary e; }" ilscript[].content[8] "clear_state | guard { input f | summary f; }" ilscript[].content[9] "clear_state | guard { input g | summary g; }" ilscript[].content[10] "clear_state | guard { input h | summary h; }" +ilscript[].content[11] "clear_state | guard { input mytags | for_each { tokenize normalize stem:\"BEST\" } | index mytags; }" diff --git a/config-model/src/test/derived/multiplesummaries/index-info.cfg b/config-model/src/test/derived/multiplesummaries/index-info.cfg index 9c53a66549c..d5002535761 100644 --- a/config-model/src/test/derived/multiplesummaries/index-info.cfg +++ b/config-model/src/test/derived/multiplesummaries/index-info.cfg @@ -71,6 +71,20 @@ indexinfo[].command[].indexname "loc" indexinfo[].command[].command "index" indexinfo[].command[].indexname "loc" indexinfo[].command[].command "type string" +indexinfo[].command[].indexname "mytags" +indexinfo[].command[].command "index" +indexinfo[].command[].indexname "mytags" +indexinfo[].command[].command "lowercase" +indexinfo[].command[].indexname "mytags" +indexinfo[].command[].command "multivalue" +indexinfo[].command[].indexname "mytags" +indexinfo[].command[].command "stem:BEST" +indexinfo[].command[].indexname "mytags" +indexinfo[].command[].command "normalize" +indexinfo[].command[].indexname "mytags" +indexinfo[].command[].command "plain-tokens" +indexinfo[].command[].indexname "mytags" +indexinfo[].command[].command "type Array<string>" indexinfo[].command[].indexname "abolded2" indexinfo[].command[].command "index" indexinfo[].command[].indexname "abolded2" @@ -83,6 +97,16 @@ indexinfo[].command[].indexname "adynamic2" indexinfo[].command[].command "index" indexinfo[].command[].indexname "adynamic2" indexinfo[].command[].command "type string" +indexinfo[].command[].indexname "alltags" +indexinfo[].command[].command "index" +indexinfo[].command[].indexname "alltags" +indexinfo[].command[].command "multivalue" +indexinfo[].command[].indexname "alltags" +indexinfo[].command[].command "type Array<string>" +indexinfo[].command[].indexname "anotherb" +indexinfo[].command[].command "index" +indexinfo[].command[].indexname "anotherb" +indexinfo[].command[].command "type string" indexinfo[].command[].indexname "dynamice" indexinfo[].command[].command "index" indexinfo[].command[].indexname "dynamice" diff --git a/config-model/src/test/derived/multiplesummaries/multiplesummaries.sd b/config-model/src/test/derived/multiplesummaries/multiplesummaries.sd index a1454a8d8a4..ae0e2fe92bc 100644 --- a/config-model/src/test/derived/multiplesummaries/multiplesummaries.sd +++ b/config-model/src/test/derived/multiplesummaries/multiplesummaries.sd @@ -67,6 +67,10 @@ search multiplesummaries { field loc type string { } + + field mytags type array<string> { + indexing: index + } } field loc_pos type position { @@ -120,7 +124,7 @@ search multiplesummaries { } # Since a here is a dynamic summary, it will be fetched from disk - document-summary notattributesonly2 { + document-summary anothernotattributesonly2 { summary adynamic2 type string { # Should still be dynamic here source: a @@ -130,6 +134,19 @@ search multiplesummaries { summary c type string { } + summary alltags type array<string> { + source: mytags + } + summary sometags type array<string> { + source: mytags + matched-elements-only + } + summary anothera type string { + source: a + } + summary anotherb type string { + source: b + } } # Not attributes only because d is bolded diff --git a/config-model/src/test/derived/multiplesummaries/summary.cfg b/config-model/src/test/derived/multiplesummaries/summary.cfg index ec5e0610385..1c8fc47878b 100644 --- a/config-model/src/test/derived/multiplesummaries/summary.cfg +++ b/config-model/src/test/derived/multiplesummaries/summary.cfg @@ -1,6 +1,6 @@ -defaultsummaryid 2038247029 +defaultsummaryid 456145241 usev8geopositions false -classes[].id 2038247029 +classes[].id 456145241 classes[].name "default" classes[].omitsummaryfeatures false classes[].fields[].name "loc_pos" @@ -37,6 +37,12 @@ classes[].fields[].name "e" classes[].fields[].type "longstring" classes[].fields[].name "adynamic2" classes[].fields[].type "longstring" +classes[].fields[].name "alltags" +classes[].fields[].type "jsonstring" +classes[].fields[].name "sometags" +classes[].fields[].type "jsonstring" +classes[].fields[].name "anotherb" +classes[].fields[].type "longstring" classes[].fields[].name "abolded2" classes[].fields[].type "longstring" classes[].fields[].name "aboldeddynamic" @@ -86,13 +92,21 @@ classes[].fields[].name "rankfeatures" classes[].fields[].type "featuredata" classes[].fields[].name "summaryfeatures" classes[].fields[].type "featuredata" -classes[].id 1527097108 -classes[].name "notattributesonly2" +classes[].id 1609068631 +classes[].name "anothernotattributesonly2" classes[].omitsummaryfeatures false classes[].fields[].name "adynamic2" classes[].fields[].type "longstring" classes[].fields[].name "c" classes[].fields[].type "longstring" +classes[].fields[].name "alltags" +classes[].fields[].type "jsonstring" +classes[].fields[].name "sometags" +classes[].fields[].type "jsonstring" +classes[].fields[].name "anothera" +classes[].fields[].type "longstring" +classes[].fields[].name "anotherb" +classes[].fields[].type "longstring" classes[].fields[].name "rankfeatures" classes[].fields[].type "featuredata" classes[].fields[].name "summaryfeatures" diff --git a/config-model/src/test/derived/multiplesummaries/summarymap.cfg b/config-model/src/test/derived/multiplesummaries/summarymap.cfg index adf0770a835..94adc250c54 100644 --- a/config-model/src/test/derived/multiplesummaries/summarymap.cfg +++ b/config-model/src/test/derived/multiplesummaries/summarymap.cfg @@ -20,6 +20,12 @@ override[].arguments "c" override[].field "adynamic2" override[].command "dynamicteaser" override[].arguments "adynamic2" +override[].field "sometags" +override[].command "matchedelementsfilter" +override[].arguments "mytags" +override[].field "anothera" +override[].command "attribute" +override[].arguments "a" override[].field "anotdynamic" override[].command "attribute" override[].arguments "adynamic" diff --git a/config-model/src/test/examples/nextgen/summaryfield.sd b/config-model/src/test/examples/nextgen/summaryfield.sd index 9b3cc6862b9..99c73d1be53 100644 --- a/config-model/src/test/examples/nextgen/summaryfield.sd +++ b/config-model/src/test/examples/nextgen/summaryfield.sd @@ -5,11 +5,21 @@ search summaryfield { indexing: index | summary summary bar: full } + field mytags type array<string> { + indexing: index + } } document-summary baz { summary cox type string { source: bar } + summary alltags type array<string> { + source: mytags + } + summary sometags type array<string> { + source: mytags + matched-elements-only + } } } diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/ImplicitSchemaFieldsTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/processing/ImplicitSchemaFieldsTestCase.java index b0b9ce81cc7..833a6effe4a 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/processing/ImplicitSchemaFieldsTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/processing/ImplicitSchemaFieldsTestCase.java @@ -38,7 +38,9 @@ public class ImplicitSchemaFieldsTestCase extends AbstractSchemaTestCase { assertNotNull(docType.getField("foo")); assertNotNull(docType.getField("bar")); assertNotNull(docType.getField("cox")); - assertEquals(3, docType.getFieldCount()); + assertNotNull(docType.getField("mytags")); + assertNotNull(docType.getField("alltags")); + assertEquals(5, docType.getFieldCount()); } @Test diff --git a/config-model/src/test/java/com/yahoo/vespa/model/content/ContentClusterTest.java b/config-model/src/test/java/com/yahoo/vespa/model/content/ContentClusterTest.java index 88073d281c5..9f571167d8c 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/content/ContentClusterTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/content/ContentClusterTest.java @@ -1159,6 +1159,24 @@ public class ContentClusterTest extends ContentBaseTest { } @Test + public void inhibit_default_merges_when_global_merges_pending_controlled_by_properties() throws Exception { + assertFalse(resolveInhibitDefaultMergesConfig(Optional.empty())); + assertFalse(resolveInhibitDefaultMergesConfig(Optional.of(false))); + assertTrue(resolveInhibitDefaultMergesConfig(Optional.of(true))); + } + + private boolean resolveInhibitDefaultMergesConfig(Optional<Boolean> inhibitDefaultMerges) throws Exception { + var props = new TestProperties(); + if (inhibitDefaultMerges.isPresent()) { + props.inhibitDefaultMergesWhenGlobalMergesPending(inhibitDefaultMerges.get()); + } + var cluster = createOneNodeCluster(props); + var builder = new StorDistributormanagerConfig.Builder(); + cluster.getDistributorNodes().getConfig(builder); + return (new StorDistributormanagerConfig(builder)).inhibit_default_merges_when_global_merges_pending(); + } + + @Test public void testDedicatedClusterControllers() { VespaModel noContentModel = createEnd2EndOneNode(new TestProperties().setHostedVespa(true) .setMultitenant(true), diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java index 7cc5f5edd14..6028d559473 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java @@ -203,6 +203,7 @@ public class ModelContextImpl implements ModelContext { private final String mergeThrottlingPolicy; private final double persistenceThrottlingWsDecrementFactor; private final double persistenceThrottlingWsBackoff; + private final boolean inhibitDefaultMergesWhenGlobalMergesPending; private final boolean useQrserverServiceName; public FeatureFlags(FlagSource source, ApplicationId appId) { @@ -245,6 +246,7 @@ public class ModelContextImpl implements ModelContext { this.mergeThrottlingPolicy = flagValue(source, appId, Flags.MERGE_THROTTLING_POLICY); this.persistenceThrottlingWsDecrementFactor = flagValue(source, appId, Flags.PERSISTENCE_THROTTLING_WS_DECREMENT_FACTOR); this.persistenceThrottlingWsBackoff = flagValue(source, appId, Flags.PERSISTENCE_THROTTLING_WS_BACKOFF); + this.inhibitDefaultMergesWhenGlobalMergesPending = flagValue(source, appId, Flags.INHIBIT_DEFAULT_MERGES_WHEN_GLOBAL_MERGES_PENDING); this.useQrserverServiceName = flagValue(source, appId, Flags.USE_QRSERVER_SERVICE_NAME); } @@ -289,6 +291,7 @@ public class ModelContextImpl implements ModelContext { @Override public String mergeThrottlingPolicy() { return mergeThrottlingPolicy; } @Override public double persistenceThrottlingWsDecrementFactor() { return persistenceThrottlingWsDecrementFactor; } @Override public double persistenceThrottlingWsBackoff() { return persistenceThrottlingWsBackoff; } + @Override public boolean inhibitDefaultMergesWhenGlobalMergesPending() { return inhibitDefaultMergesWhenGlobalMergesPending; } @Override public boolean useQrserverServiceName() { return useQrserverServiceName; } private static <V> V flagValue(FlagSource source, ApplicationId appId, UnboundFlag<? extends V, ?, ?> flag) { diff --git a/eval/CMakeLists.txt b/eval/CMakeLists.txt index eed4fa5ce66..e6669e3fde8 100644 --- a/eval/CMakeLists.txt +++ b/eval/CMakeLists.txt @@ -12,6 +12,7 @@ vespa_define_module( TESTS src/tests/ann + src/tests/apps/analyze_onnx_model src/tests/apps/eval_expr src/tests/eval/addr_to_symbol src/tests/eval/aggr diff --git a/eval/src/apps/analyze_onnx_model/CMakeLists.txt b/eval/src/apps/analyze_onnx_model/CMakeLists.txt index e2ed64cd8cc..dc89213f9eb 100644 --- a/eval/src/apps/analyze_onnx_model/CMakeLists.txt +++ b/eval/src/apps/analyze_onnx_model/CMakeLists.txt @@ -1,7 +1,8 @@ # Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -vespa_add_executable(vespa-analyze-onnx-model +vespa_add_executable(eval_analyze_onnx_model_app SOURCES analyze_onnx_model.cpp + OUTPUT_NAME vespa-analyze-onnx-model INSTALL bin DEPENDS vespaeval diff --git a/eval/src/apps/analyze_onnx_model/analyze_onnx_model.cpp b/eval/src/apps/analyze_onnx_model/analyze_onnx_model.cpp index 2f22f903f2e..506073ae8b3 100644 --- a/eval/src/apps/analyze_onnx_model/analyze_onnx_model.cpp +++ b/eval/src/apps/analyze_onnx_model/analyze_onnx_model.cpp @@ -4,6 +4,7 @@ #include <vespa/eval/eval/tensor_spec.h> #include <vespa/eval/eval/value_codec.h> #include <vespa/eval/eval/fast_value.h> +#include <vespa/eval/eval/test/test_io.h> #include <vespa/vespalib/util/benchmark_timer.h> #include <vespa/vespalib/util/require.h> #include <vespa/vespalib/util/guard.h> @@ -11,8 +12,13 @@ using vespalib::make_string_short::fmt; +using vespalib::Slime; +using vespalib::slime::JsonFormat; +using vespalib::slime::Inspector; +using vespalib::slime::Cursor; using vespalib::FilePointer; using namespace vespalib::eval; +using namespace vespalib::eval::test; bool read_line(FilePointer &file, vespalib::string &line) { char line_buffer[1024]; @@ -169,14 +175,50 @@ int usage(const char *self) { fprintf(stderr, " load onnx model and report memory usage\n"); fprintf(stderr, " options are used to specify unknown values, like dimension sizes\n"); fprintf(stderr, " options are accepted in the order in which they are needed\n"); - fprintf(stderr, " tip: run without options first, to see which you need\n"); + fprintf(stderr, " tip: run without options first, to see which you need\n\n"); + fprintf(stderr, "usage: %s --probe-types\n", self); + fprintf(stderr, " use onnx model to infer/probe output types based on input types\n"); + fprintf(stderr, " parameters are read from stdin and results are written to stdout\n"); + fprintf(stderr, " input format (json): {model:<filename>, inputs:{<name>:vespa-type-string}}\n"); + fprintf(stderr, " output format (json): {outputs:{<name>:vespa-type-string}}\n"); return 1; } +int probe_types() { + StdIn std_in; + StdOut std_out; + Slime params; + if (!JsonFormat::decode(std_in, params)) { + return 3; + } + Slime result; + auto &root = result.setObject(); + auto &types = root.setObject("outputs"); + Onnx model(params["model"].asString().make_string(), Onnx::Optimize::DISABLE); + Onnx::WirePlanner planner; + for (size_t i = 0; i < model.inputs().size(); ++i) { + auto spec = params["inputs"][model.inputs()[i].name].asString().make_string(); + auto input_type = ValueType::from_spec(spec); + REQUIRE(!input_type.is_error()); + REQUIRE(planner.bind_input_type(input_type, model.inputs()[i])); + } + planner.prepare_output_types(model); + for (const auto &output: model.outputs()) { + auto output_type = planner.make_output_type(output); + REQUIRE(!output_type.is_error()); + types.setString(output.name, output_type.to_spec()); + } + write_compact(result, std_out); + return 0; +} + int my_main(int argc, char **argv) { if (argc < 2) { return usage(argv[0]); } + if ((argc == 2) && (vespalib::string(argv[1]) == "--probe-types")) { + return probe_types(); + } Options opts; for (int i = 2; i < argc; ++i) { opts.add_option(argv[i]); diff --git a/eval/src/tests/apps/analyze_onnx_model/CMakeLists.txt b/eval/src/tests/apps/analyze_onnx_model/CMakeLists.txt new file mode 100644 index 00000000000..7b70360a622 --- /dev/null +++ b/eval/src/tests/apps/analyze_onnx_model/CMakeLists.txt @@ -0,0 +1,9 @@ +# Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +vespa_add_executable(eval_analyze_onnx_model_test_app TEST + SOURCES + analyze_onnx_model_test.cpp + DEPENDS + vespaeval +) +vespa_add_test(NAME eval_analyze_onnx_model_test_app COMMAND eval_analyze_onnx_model_test_app + DEPENDS eval_analyze_onnx_model_test_app eval_analyze_onnx_model_app) diff --git a/eval/src/tests/apps/analyze_onnx_model/analyze_onnx_model_test.cpp b/eval/src/tests/apps/analyze_onnx_model/analyze_onnx_model_test.cpp new file mode 100644 index 00000000000..2c1b2b21b9e --- /dev/null +++ b/eval/src/tests/apps/analyze_onnx_model/analyze_onnx_model_test.cpp @@ -0,0 +1,137 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include <vespa/vespalib/testkit/test_kit.h> +#include <vespa/vespalib/testkit/time_bomb.h> +#include <vespa/vespalib/util/stringfmt.h> +#include <vespa/vespalib/data/slime/slime.h> +#include <vespa/vespalib/util/child_process.h> +#include <vespa/vespalib/data/input.h> +#include <vespa/vespalib/data/output.h> +#include <vespa/vespalib/data/simple_buffer.h> +#include <vespa/vespalib/util/size_literals.h> +#include <vespa/eval/eval/test/test_io.h> + +using namespace vespalib; +using namespace vespalib::eval::test; +using vespalib::make_string_short::fmt; +using vespalib::slime::JsonFormat; +using vespalib::slime::Inspector; + +vespalib::string module_build_path("../../../../"); +vespalib::string binary = module_build_path + "src/apps/analyze_onnx_model/vespa-analyze-onnx-model"; +vespalib::string probe_cmd = binary + " --probe-types"; + +std::string get_source_dir() { + const char *dir = getenv("SOURCE_DIRECTORY"); + return (dir ? dir : "."); +} +std::string source_dir = get_source_dir(); +std::string guess_batch_model = source_dir + "/../../tensor/onnx_wrapper/guess_batch.onnx"; + +//----------------------------------------------------------------------------- + +void read_until_eof(Input &input) { + for (auto mem = input.obtain(); mem.size > 0; mem = input.obtain()) { + input.evict(mem.size); + } +} + +// Output adapter used to write to stdin of a child process +class ChildIn : public Output { + ChildProcess &_child; + SimpleBuffer _output; +public: + ChildIn(ChildProcess &child) : _child(child) {} + WritableMemory reserve(size_t bytes) override { + return _output.reserve(bytes); + } + Output &commit(size_t bytes) override { + _output.commit(bytes); + Memory buf = _output.obtain(); + ASSERT_TRUE(_child.write(buf.data, buf.size)); + _output.evict(buf.size); + return *this; + } +}; + +// Input adapter used to read from stdout of a child process +class ChildOut : public Input { + ChildProcess &_child; + SimpleBuffer _input; +public: + ChildOut(ChildProcess &child) + : _child(child) + { + EXPECT_TRUE(_child.running()); + EXPECT_TRUE(!_child.failed()); + } + Memory obtain() override { + if ((_input.get().size == 0) && !_child.eof()) { + WritableMemory buf = _input.reserve(4_Ki); + uint32_t res = _child.read(buf.data, buf.size); + ASSERT_TRUE((res > 0) || _child.eof()); + _input.commit(res); + } + return _input.obtain(); + } + Input &evict(size_t bytes) override { + _input.evict(bytes); + return *this; + } +}; + +//----------------------------------------------------------------------------- + +void dump_message(const char *prefix, const Slime &slime) { + SimpleBuffer buf; + slime::JsonFormat::encode(slime, buf, true); + auto str = buf.get().make_string(); + fprintf(stderr, "%s%s\n", prefix, str.c_str()); +} + +class Server { +private: + TimeBomb _bomb; + ChildProcess _child; + ChildIn _child_stdin; + ChildOut _child_stdout; +public: + Server(vespalib::string cmd) + : _bomb(60), + _child(cmd.c_str()), + _child_stdin(_child), + _child_stdout(_child) {} + ~Server(); + Slime invoke(const Slime &req) { + dump_message("request --> ", req); + write_compact(req, _child_stdin); + Slime reply; + ASSERT_TRUE(JsonFormat::decode(_child_stdout, reply)); + dump_message(" reply <-- ", reply); + return reply; + } +}; +Server::~Server() { + _child.close(); + read_until_eof(_child_stdout); + ASSERT_TRUE(_child.wait()); + ASSERT_TRUE(!_child.running()); + ASSERT_TRUE(!_child.failed()); +} + +//----------------------------------------------------------------------------- + +TEST_F("require that output types can be probed", Server(probe_cmd)) { + Slime params; + params.setObject(); + params.get().setString("model", guess_batch_model); + params.get().setObject("inputs"); + params["inputs"].setString("in1", "tensor<float>(x[3])"); + params["inputs"].setString("in2", "tensor<float>(x[3])"); + Slime result = f1.invoke(params); + EXPECT_EQUAL(result["outputs"]["out"].asString().make_string(), vespalib::string("tensor<float>(d0[3])")); +} + +//----------------------------------------------------------------------------- + +TEST_MAIN_WITH_PROCESS_PROXY() { TEST_RUN_ALL(); } diff --git a/eval/src/tests/tensor/onnx_wrapper/onnx_wrapper_test.cpp b/eval/src/tests/tensor/onnx_wrapper/onnx_wrapper_test.cpp index da957673f95..e50c41e2e09 100644 --- a/eval/src/tests/tensor/onnx_wrapper/onnx_wrapper_test.cpp +++ b/eval/src/tests/tensor/onnx_wrapper/onnx_wrapper_test.cpp @@ -2,6 +2,7 @@ #include <vespa/eval/eval/tensor_spec.h> #include <vespa/eval/eval/int8float.h> +#include <vespa/eval/eval/test/eval_onnx.h> #include <vespa/eval/onnx/onnx_wrapper.h> #include <vespa/eval/onnx/onnx_model_cache.h> #include <vespa/vespalib/util/bfloat16.h> @@ -28,6 +29,7 @@ std::string int_types_model = source_dir + "/int_types.onnx"; std::string guess_batch_model = source_dir + "/guess_batch.onnx"; std::string unstable_types_model = source_dir + "/unstable_types.onnx"; std::string float_to_int8_model = source_dir + "/float_to_int8.onnx"; +std::string probe_model = source_dir + "/probe_model.onnx"; void dump_info(const char *ctx, const std::vector<TensorInfo> &info) { fprintf(stderr, "%s:\n", ctx); @@ -504,4 +506,24 @@ TEST(OnnxModelCacheTest, share_and_evict_onnx_models) { EXPECT_EQ(OnnxModelCache::count_refs(), 0); } +TensorSpec val(const vespalib::string &expr) { + auto result = TensorSpec::from_expr(expr); + EXPECT_FALSE(ValueType::from_spec(result.type()).is_error()); + return result; +} + +TEST(OnnxTest, eval_onnx_with_probe_model) { + Onnx model(probe_model, Onnx::Optimize::ENABLE); + auto in1 = val("tensor<float>( x[2], y[3]):[[ 1, 2, 3],[ 4, 5, 6]]"); + auto in2 = val("tensor<float>( x[2], y[3]):[[ 7, 8, 9],[ 4, 5, 6]]"); + auto out1 = val("tensor<float>(d0[2],d1[3]):[[ 8,10,12],[ 8,10,12]]"); + auto out2 = val("tensor<float>(d0[2],d1[3]):[[-6,-6,-6],[ 0, 0, 0]]"); + auto out3 = val("tensor<float>(d0[2],d1[3]):[[ 7,16,27],[16,25,36]]"); + auto result = test::eval_onnx(model, {in1, in2}); + ASSERT_EQ(result.size(), 3); + EXPECT_EQ(result[0], out1); + EXPECT_EQ(result[1], out2); + EXPECT_EQ(result[2], out3); +} + GTEST_MAIN_RUN_ALL_TESTS() diff --git a/eval/src/tests/tensor/onnx_wrapper/probe_model.onnx b/eval/src/tests/tensor/onnx_wrapper/probe_model.onnx new file mode 100644 index 00000000000..89dab2e7c4c --- /dev/null +++ b/eval/src/tests/tensor/onnx_wrapper/probe_model.onnx @@ -0,0 +1,30 @@ +probe_model.py:’ + +in1 +in2out1"Add + +in1 +in2out2"Sub + +in1 +in2out3"Mul probe_modelZ# +in1 + + ÿÿÿÿÿÿÿÿÿ +innerZ# +in2 + +outer + ÿÿÿÿÿÿÿÿÿb$ +out1 + + ÿÿÿÿÿÿÿÿÿ +innerb$ +out2 + +outer + ÿÿÿÿÿÿÿÿÿb( +out3 + + ÿÿÿÿÿÿÿÿÿ + ÿÿÿÿÿÿÿÿÿB
\ No newline at end of file diff --git a/eval/src/tests/tensor/onnx_wrapper/probe_model.py b/eval/src/tests/tensor/onnx_wrapper/probe_model.py new file mode 100755 index 00000000000..529fa23b2b1 --- /dev/null +++ b/eval/src/tests/tensor/onnx_wrapper/probe_model.py @@ -0,0 +1,35 @@ +# Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +import onnx +from onnx import helper, TensorProto + +IN1 = helper.make_tensor_value_info('in1', TensorProto.FLOAT, [-1, 'inner']) +IN2 = helper.make_tensor_value_info('in2', TensorProto.FLOAT, ['outer', -1]) +OUT1 = helper.make_tensor_value_info('out1', TensorProto.FLOAT, [-1, 'inner']) +OUT2 = helper.make_tensor_value_info('out2', TensorProto.FLOAT, ['outer', -1]) +OUT3 = helper.make_tensor_value_info('out3', TensorProto.FLOAT, [-1, -1]) + +nodes = [ + helper.make_node( + 'Add', + ['in1', 'in2'], + ['out1'], + ), + helper.make_node( + 'Sub', + ['in1', 'in2'], + ['out2'], + ), + helper.make_node( + 'Mul', + ['in1', 'in2'], + ['out3'], + ), +] +graph_def = helper.make_graph( + nodes, + 'probe_model', + [IN1, IN2], + [OUT1, OUT2, OUT3], +) +model_def = helper.make_model(graph_def, producer_name='probe_model.py', opset_imports=[onnx.OperatorSetIdProto(version=12)]) +onnx.save(model_def, 'probe_model.onnx') diff --git a/eval/src/vespa/eval/eval/test/CMakeLists.txt b/eval/src/vespa/eval/eval/test/CMakeLists.txt index e8a291adf2a..ff1505a4010 100644 --- a/eval/src/vespa/eval/eval/test/CMakeLists.txt +++ b/eval/src/vespa/eval/eval/test/CMakeLists.txt @@ -3,6 +3,7 @@ vespa_add_library(eval_eval_test OBJECT SOURCES cell_type_space.cpp eval_fixture.cpp + eval_onnx.cpp eval_spec.cpp gen_spec.cpp reference_evaluation.cpp diff --git a/eval/src/vespa/eval/eval/test/eval_onnx.cpp b/eval/src/vespa/eval/eval/test/eval_onnx.cpp new file mode 100644 index 00000000000..74a83b130c2 --- /dev/null +++ b/eval/src/vespa/eval/eval/test/eval_onnx.cpp @@ -0,0 +1,54 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include "eval_onnx.h" +#include <vespa/eval/eval/fast_value.h> +#include <vespa/eval/eval/value_codec.h> + +#include <vespa/log/log.h> +LOG_SETUP(".eval.eval.test.eval_onnx"); + +namespace vespalib::eval::test { + +std::vector<TensorSpec> eval_onnx(const Onnx &model, const std::vector<TensorSpec> ¶ms) { + if (params.size() != model.inputs().size()) { + LOG(error, "model with %zu inputs run with %zu parameters", model.inputs().size(), params.size()); + return {}; // wrong number of parameters + } + Onnx::WirePlanner planner; + for (size_t i = 0; i < model.inputs().size(); ++i) { + if (!planner.bind_input_type(ValueType::from_spec(params[i].type()), model.inputs()[i])) { + LOG(error, "unable to bind input type: %s -> %s", params[i].type().c_str(), model.inputs()[i].type_as_string().c_str()); + return {}; // inconsistent input types + } + } + planner.prepare_output_types(model); + for (size_t i = 0; i < model.outputs().size(); ++i) { + if (planner.make_output_type(model.outputs()[i]).is_error()) { + LOG(error, "unable to make output type: %s -> error", model.outputs()[i].type_as_string().c_str()); + return {}; // unable to infer/probe output type + } + } + planner.prepare_output_types(model); + auto wire_info = planner.get_wire_info(model); + try { + Onnx::EvalContext context(model, wire_info); + std::vector<Value::UP> inputs; + for (const auto ¶m: params) { + inputs.push_back(value_from_spec(param, FastValueBuilderFactory::get())); + } + for (size_t i = 0; i < model.inputs().size(); ++i) { + context.bind_param(i, *inputs[i]); + } + context.eval(); + std::vector<TensorSpec> results; + for (size_t i = 0; i < model.outputs().size(); ++i) { + results.push_back(spec_from_value(context.get_result(i))); + } + return results; + } catch (const Ort::Exception &ex) { + LOG(error, "model run failed: %s", ex.what()); + return {}; // evaluation failed + } +} + +} // namespace diff --git a/eval/src/vespa/eval/eval/test/eval_onnx.h b/eval/src/vespa/eval/eval/test/eval_onnx.h new file mode 100644 index 00000000000..bb346b7f21e --- /dev/null +++ b/eval/src/vespa/eval/eval/test/eval_onnx.h @@ -0,0 +1,13 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#pragma once + +#include <vespa/eval/eval/tensor_spec.h> +#include <vespa/eval/onnx/onnx_wrapper.h> +#include <vector> + +namespace vespalib::eval::test { + +std::vector<TensorSpec> eval_onnx(const Onnx &model, const std::vector<TensorSpec> ¶ms); + +} // namespace diff --git a/flags/src/main/java/com/yahoo/vespa/flags/Flags.java b/flags/src/main/java/com/yahoo/vespa/flags/Flags.java index 9c1e1776894..2e985a4fca0 100644 --- a/flags/src/main/java/com/yahoo/vespa/flags/Flags.java +++ b/flags/src/main/java/com/yahoo/vespa/flags/Flags.java @@ -328,6 +328,14 @@ public class Flags { "Takes effect on redeployment", ZONE_ID, APPLICATION_ID); + public static final UnboundBooleanFlag INHIBIT_DEFAULT_MERGES_WHEN_GLOBAL_MERGES_PENDING = defineFeatureFlag( + "inhibit-default-merges-when-global-merges-pending", false, + List.of("geirst", "vekterli"), "2022-02-11", "2022-06-01", + "Inhibits all merges to buckets in the default bucket space if the current " + + "cluster state bundle indicates that global merges are pending in the cluster", + "Takes effect on redeployment", + ZONE_ID, APPLICATION_ID); + public static final UnboundBooleanFlag CHECK_CONFIG_CONVERGENCE_BEFORE_RESTARTING = defineFeatureFlag( "check-config-convergence-before-restart", true, List.of("hmusum"), "2022-01-16", "2022-02-16", diff --git a/searchlib/src/tests/attribute/enumstore/enumstore_test.cpp b/searchlib/src/tests/attribute/enumstore/enumstore_test.cpp index 5346cc7f764..99fdd9f4b0a 100644 --- a/searchlib/src/tests/attribute/enumstore/enumstore_test.cpp +++ b/searchlib/src/tests/attribute/enumstore/enumstore_test.cpp @@ -1,6 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include <vespa/searchlib/attribute/enumstore.hpp> +#include <vespa/vespalib/test/memory_allocator_observer.h> #include <vespa/vespalib/gtest/gtest.h> #include <vespa/log/log.h> @@ -11,6 +12,8 @@ using vespalib::datastore::CompactionStrategy; using vespalib::datastore::EntryRef; using vespalib::datastore::EntryRefFilter; using RefT = vespalib::datastore::EntryRefT<22>; +using vespalib::alloc::test::MemoryAllocatorObserver; +using AllocStats = MemoryAllocatorObserver::Stats; namespace vespalib::datastore { @@ -374,6 +377,13 @@ TEST(EnumStoreTest, address_space_usage_is_reported) EXPECT_EQ(AddressSpace(3, 3, ADDRESS_LIMIT + 2), store.get_values_address_space_usage()); } +TEST(EnumStoreTest, provided_memory_allocator_is_used) +{ + AllocStats stats; + NumericEnumStore ses(false, DictionaryConfig::Type::BTREE, std::make_unique<MemoryAllocatorObserver>(stats)); + EXPECT_EQ(AllocStats(1, 0), stats); +} + class BatchUpdaterTest : public ::testing::Test { public: NumericEnumStore store; diff --git a/searchlib/src/tests/attribute/multi_value_mapping/multi_value_mapping_test.cpp b/searchlib/src/tests/attribute/multi_value_mapping/multi_value_mapping_test.cpp index bddaa4f4e31..29af989d484 100644 --- a/searchlib/src/tests/attribute/multi_value_mapping/multi_value_mapping_test.cpp +++ b/searchlib/src/tests/attribute/multi_value_mapping/multi_value_mapping_test.cpp @@ -6,6 +6,7 @@ #include <vespa/vespalib/gtest/gtest.h> #include <vespa/vespalib/stllike/hash_set.h> #include <vespa/vespalib/test/insertion_operators.h> +#include <vespa/vespalib/test/memory_allocator_observer.h> #include <vespa/vespalib/util/generationhandler.h> #include <vespa/vespalib/util/rand48.h> #include <vespa/vespalib/util/size_literals.h> @@ -16,6 +17,8 @@ LOG_SETUP("multivaluemapping_test"); using vespalib::datastore::ArrayStoreConfig; using vespalib::datastore::CompactionSpec; using vespalib::datastore::CompactionStrategy; +using vespalib::alloc::test::MemoryAllocatorObserver; +using AllocStats = MemoryAllocatorObserver::Stats; template <typename EntryT> void @@ -69,6 +72,7 @@ class MappingTestBase : public ::testing::Test { protected: using MvMapping = search::attribute::MultiValueMapping<EntryT>; using AttributeType = MyAttribute<MvMapping>; + AllocStats _stats; std::unique_ptr<MvMapping> _mvMapping; std::unique_ptr<AttributeType> _attr; uint32_t _maxSmallArraySize; @@ -78,7 +82,8 @@ protected: public: using ConstArrayRef = vespalib::ConstArrayRef<EntryT>; MappingTestBase() - : _mvMapping(), + : _stats(), + _mvMapping(), _attr(), _maxSmallArraySize() { @@ -87,7 +92,7 @@ public: ArrayStoreConfig config(maxSmallArraySize, ArrayStoreConfig::AllocSpec(0, RefType::offsetSize(), 8_Ki, ALLOC_GROW_FACTOR)); config.enable_free_lists(enable_free_lists); - _mvMapping = std::make_unique<MvMapping>(config); + _mvMapping = std::make_unique<MvMapping>(config, vespalib::GrowStrategy(), std::make_unique<MemoryAllocatorObserver>(_stats)); _attr = std::make_unique<AttributeType>(*_mvMapping); _maxSmallArraySize = maxSmallArraySize; } @@ -95,7 +100,7 @@ public: ArrayStoreConfig config(maxSmallArraySize, ArrayStoreConfig::AllocSpec(minArrays, maxArrays, numArraysForNewBuffer, ALLOC_GROW_FACTOR)); config.enable_free_lists(enable_free_lists); - _mvMapping = std::make_unique<MvMapping>(config); + _mvMapping = std::make_unique<MvMapping>(config, vespalib::GrowStrategy(), std::make_unique<MemoryAllocatorObserver>(_stats)); _attr = std::make_unique<AttributeType>(*_mvMapping); _maxSmallArraySize = maxSmallArraySize; } @@ -129,6 +134,7 @@ public: _mvMapping->clearDocs(lidLow, lidLimit, [this](uint32_t docId) { _attr->clearDoc(docId); }); } size_t getTotalValueCnt() const { return _mvMapping->getTotalValueCnt(); } + const AllocStats &get_stats() const noexcept { return _stats; } uint32_t countBuffers() { using RefVector = typename MvMapping::RefCopyVector; @@ -326,6 +332,12 @@ TEST_F(IntMappingTest, test_that_free_lists_can_be_disabled) EXPECT_FALSE(_mvMapping->has_free_lists_enabled()); } +TEST_F(IntMappingTest, provided_memory_allocator_is_used) +{ + setup(3, 64, 512, 129, true); + EXPECT_EQ(AllocStats(5, 0), get_stats()); +} + TEST_F(CompactionIntMappingTest, test_that_compaction_works) { setup(3, 64, 512, 129); diff --git a/searchlib/src/vespa/searchlib/attribute/attributevector.cpp b/searchlib/src/vespa/searchlib/attribute/attributevector.cpp index a2ac482ebf3..08721b7302e 100644 --- a/searchlib/src/vespa/searchlib/attribute/attributevector.cpp +++ b/searchlib/src/vespa/searchlib/attribute/attributevector.cpp @@ -23,6 +23,7 @@ #include <vespa/searchlib/util/file_settings.h> #include <vespa/searchlib/util/logutil.h> #include <vespa/vespalib/util/exceptions.h> +#include <vespa/vespalib/util/mmap_file_allocator_factory.h> #include <vespa/vespalib/util/size_literals.h> #include <thread> @@ -107,6 +108,34 @@ AttributeVector::ValueModifier::~ValueModifier() { } } +namespace { + +bool +allow_paged(const search::attribute::Config& config) +{ + if (!config.paged()) { + return false; + } + using Type = search::attribute::BasicType::Type; + if (config.basicType() == Type::REFERENCE || config.basicType() == Type::PREDICATE) { + return false; + } + if (config.basicType() == Type::TENSOR) { + return (!config.tensorType().is_error() && config.tensorType().is_dense()); + } + return true; +} + +std::unique_ptr<vespalib::alloc::MemoryAllocator> +make_memory_allocator(const vespalib::string& name, const search::attribute::Config& config) +{ + if (allow_paged(config)) { + return vespalib::alloc::MmapFileAllocatorFactory::instance().make_memory_allocator(name); + } + return {}; +} + +} AttributeVector::AttributeVector(vespalib::stringref baseFileName, const Config &c) : _baseFileName(baseFileName), @@ -124,7 +153,9 @@ AttributeVector::AttributeVector(vespalib::stringref baseFileName, const Config _compactLidSpaceGeneration(0u), _hasEnum(false), _loaded(false), - _isUpdateableInMemoryOnly(attribute::isUpdateableInMemoryOnly(getName(), getConfig())) + _isUpdateableInMemoryOnly(attribute::isUpdateableInMemoryOnly(getName(), getConfig())), + _nextStatUpdateTime(), + _memory_allocator(make_memory_allocator(_baseFileName.getAttributeName(), c)) { } diff --git a/searchlib/src/vespa/searchlib/attribute/attributevector.h b/searchlib/src/vespa/searchlib/attribute/attributevector.h index acd00413568..2bf2f3a6ed6 100644 --- a/searchlib/src/vespa/searchlib/attribute/attributevector.h +++ b/searchlib/src/vespa/searchlib/attribute/attributevector.h @@ -377,6 +377,7 @@ protected: virtual vespalib::MemoryUsage getEnumStoreValuesMemoryUsage() const; virtual void populate_address_space_usage(AddressSpaceUsage& usage) const; + const std::shared_ptr<vespalib::alloc::MemoryAllocator>& get_memory_allocator() const noexcept { return _memory_allocator; } public: DECLARE_IDENTIFIABLE_ABSTRACT(AttributeVector); bool isLoaded() const { return _loaded; } @@ -584,6 +585,7 @@ private: bool _loaded; bool _isUpdateableInMemoryOnly; vespalib::steady_time _nextStatUpdateTime; + std::shared_ptr<vespalib::alloc::MemoryAllocator> _memory_allocator; ////// Locking strategy interface. only available from the Guards. /** diff --git a/searchlib/src/vespa/searchlib/attribute/enumstore.h b/searchlib/src/vespa/searchlib/attribute/enumstore.h index 7fe586b8ccc..73378311bec 100644 --- a/searchlib/src/vespa/searchlib/attribute/enumstore.h +++ b/searchlib/src/vespa/searchlib/attribute/enumstore.h @@ -77,6 +77,7 @@ private: std::unique_ptr<EntryComparator> allocate_optionally_folded_comparator(bool folded) const; ComparatorType make_optionally_folded_comparator(bool folded) const; public: + EnumStoreT(bool has_postings, const search::DictionaryConfig& dict_cfg, std::shared_ptr<vespalib::alloc::MemoryAllocator> memory_allocator); EnumStoreT(bool has_postings, const search::DictionaryConfig & dict_cfg); ~EnumStoreT() override; diff --git a/searchlib/src/vespa/searchlib/attribute/enumstore.hpp b/searchlib/src/vespa/searchlib/attribute/enumstore.hpp index 5baa7cc9df8..fa0b8977c8a 100644 --- a/searchlib/src/vespa/searchlib/attribute/enumstore.hpp +++ b/searchlib/src/vespa/searchlib/attribute/enumstore.hpp @@ -72,8 +72,8 @@ EnumStoreT<EntryT>::load_unique_value(const void* src, size_t available, Index& } template <typename EntryT> -EnumStoreT<EntryT>::EnumStoreT(bool has_postings, const DictionaryConfig & dict_cfg) - : _store({}), +EnumStoreT<EntryT>::EnumStoreT(bool has_postings, const DictionaryConfig& dict_cfg, std::shared_ptr<vespalib::alloc::MemoryAllocator> memory_allocator) + : _store(std::move(memory_allocator)), _dict(), _is_folded(dict_cfg.getMatch() == DictionaryConfig::Match::UNCASED), _comparator(_store.get_data_store()), @@ -87,6 +87,12 @@ EnumStoreT<EntryT>::EnumStoreT(bool has_postings, const DictionaryConfig & dict_ } template <typename EntryT> +EnumStoreT<EntryT>::EnumStoreT(bool has_postings, const DictionaryConfig& dict_cfg) + : EnumStoreT<EntryT>(has_postings, dict_cfg, {}) +{ +} + +template <typename EntryT> EnumStoreT<EntryT>::~EnumStoreT() = default; template <typename EntryT> diff --git a/searchlib/src/vespa/searchlib/attribute/multi_value_mapping.h b/searchlib/src/vespa/searchlib/attribute/multi_value_mapping.h index 81abaa05a45..f5f2950a59c 100644 --- a/searchlib/src/vespa/searchlib/attribute/multi_value_mapping.h +++ b/searchlib/src/vespa/searchlib/attribute/multi_value_mapping.h @@ -27,7 +27,8 @@ public: MultiValueMapping(const MultiValueMapping &) = delete; MultiValueMapping & operator = (const MultiValueMapping &) = delete; MultiValueMapping(const vespalib::datastore::ArrayStoreConfig &storeCfg, - const vespalib::GrowStrategy &gs = vespalib::GrowStrategy()); + const vespalib::GrowStrategy &gs, + std::shared_ptr<vespalib::alloc::MemoryAllocator> memory_allocator); ~MultiValueMapping() override; ConstArrayRef get(uint32_t docId) const { return _store.get(_indices[docId]); } ConstArrayRef getDataForIdx(EntryRef idx) const { return _store.get(idx); } diff --git a/searchlib/src/vespa/searchlib/attribute/multi_value_mapping.hpp b/searchlib/src/vespa/searchlib/attribute/multi_value_mapping.hpp index 339f562757d..16b29bf33cd 100644 --- a/searchlib/src/vespa/searchlib/attribute/multi_value_mapping.hpp +++ b/searchlib/src/vespa/searchlib/attribute/multi_value_mapping.hpp @@ -10,16 +10,17 @@ namespace search::attribute { template <typename EntryT, typename RefT> MultiValueMapping<EntryT,RefT>::MultiValueMapping(const vespalib::datastore::ArrayStoreConfig &storeCfg, - const vespalib::GrowStrategy &gs) + const vespalib::GrowStrategy &gs, + std::shared_ptr<vespalib::alloc::MemoryAllocator> memory_allocator) #ifdef __clang__ #pragma clang diagnostic push #pragma clang diagnostic ignored "-Wuninitialized" #endif - : MultiValueMappingBase(gs, _store.getGenerationHolder()), + : MultiValueMappingBase(gs, _store.getGenerationHolder(), memory_allocator), #ifdef __clang__ #pragma clang diagnostic pop #endif - _store(storeCfg, {}) + _store(storeCfg, std::move(memory_allocator)) { } diff --git a/searchlib/src/vespa/searchlib/attribute/multi_value_mapping_base.cpp b/searchlib/src/vespa/searchlib/attribute/multi_value_mapping_base.cpp index b0d50c129c6..7ad61ccedc5 100644 --- a/searchlib/src/vespa/searchlib/attribute/multi_value_mapping_base.cpp +++ b/searchlib/src/vespa/searchlib/attribute/multi_value_mapping_base.cpp @@ -10,8 +10,10 @@ namespace search::attribute { using vespalib::datastore::CompactionStrategy; MultiValueMappingBase::MultiValueMappingBase(const vespalib::GrowStrategy &gs, - vespalib::GenerationHolder &genHolder) - : _indices(gs, genHolder), + vespalib::GenerationHolder &genHolder, + std::shared_ptr<vespalib::alloc::MemoryAllocator> memory_allocator) + : _memory_allocator(std::move(memory_allocator)), + _indices(gs, genHolder, _memory_allocator ? vespalib::alloc::Alloc::alloc_with_allocator(_memory_allocator.get()) : vespalib::alloc::Alloc::alloc()), _totalValues(0u), _compaction_spec() { diff --git a/searchlib/src/vespa/searchlib/attribute/multi_value_mapping_base.h b/searchlib/src/vespa/searchlib/attribute/multi_value_mapping_base.h index f27a9f1667c..2b2b4d5f8a3 100644 --- a/searchlib/src/vespa/searchlib/attribute/multi_value_mapping_base.h +++ b/searchlib/src/vespa/searchlib/attribute/multi_value_mapping_base.h @@ -27,11 +27,12 @@ public: using RefVector = vespalib::RcuVectorBase<EntryRef>; protected: + std::shared_ptr<vespalib::alloc::MemoryAllocator> _memory_allocator; RefVector _indices; size_t _totalValues; CompactionSpec _compaction_spec; - MultiValueMappingBase(const vespalib::GrowStrategy &gs, vespalib::GenerationHolder &genHolder); + MultiValueMappingBase(const vespalib::GrowStrategy &gs, vespalib::GenerationHolder &genHolder, std::shared_ptr<vespalib::alloc::MemoryAllocator> memory_allocator); virtual ~MultiValueMappingBase(); void updateValueCount(size_t oldValues, size_t newValues) { diff --git a/searchlib/src/vespa/searchlib/attribute/multivalueattribute.hpp b/searchlib/src/vespa/searchlib/attribute/multivalueattribute.hpp index b9ef16c6adf..c0524a4d043 100644 --- a/searchlib/src/vespa/searchlib/attribute/multivalueattribute.hpp +++ b/searchlib/src/vespa/searchlib/attribute/multivalueattribute.hpp @@ -28,7 +28,7 @@ MultiValueAttribute(const vespalib::string &baseFileName, 8 * 1024, cfg.getGrowStrategy().getMultiValueAllocGrowFactor(), multivalueattribute::enable_free_lists), - cfg.getGrowStrategy().to_generic_strategy()) + cfg.getGrowStrategy().to_generic_strategy(), {}) { } diff --git a/searchlib/src/vespa/searchlib/tensor/dense_tensor_attribute.cpp b/searchlib/src/vespa/searchlib/tensor/dense_tensor_attribute.cpp index 6dd630a6426..d376fb020be 100644 --- a/searchlib/src/vespa/searchlib/tensor/dense_tensor_attribute.cpp +++ b/searchlib/src/vespa/searchlib/tensor/dense_tensor_attribute.cpp @@ -99,17 +99,6 @@ BlobSequenceReader::is_present() { return true; } - - -std::unique_ptr<vespalib::alloc::MemoryAllocator> -make_memory_allocator(const vespalib::string& name, bool swappable) -{ - if (swappable) { - return vespalib::alloc::MmapFileAllocatorFactory::instance().make_memory_allocator(name); - } - return {}; -} - } void @@ -161,7 +150,7 @@ DenseTensorAttribute::populate_address_space_usage(AddressSpaceUsage& usage) con DenseTensorAttribute::DenseTensorAttribute(vespalib::stringref baseFileName, const Config& cfg, const NearestNeighborIndexFactory& index_factory) : TensorAttribute(baseFileName, cfg, _denseTensorStore), - _denseTensorStore(cfg.tensorType(), make_memory_allocator(getName(), cfg.paged())), + _denseTensorStore(cfg.tensorType(), get_memory_allocator()), _index() { if (cfg.hnsw_index_params().has_value()) { diff --git a/searchlib/src/vespa/searchlib/tensor/dense_tensor_store.cpp b/searchlib/src/vespa/searchlib/tensor/dense_tensor_store.cpp index ed3fb737b7d..6435ba6f27c 100644 --- a/searchlib/src/vespa/searchlib/tensor/dense_tensor_store.cpp +++ b/searchlib/src/vespa/searchlib/tensor/dense_tensor_store.cpp @@ -45,7 +45,7 @@ DenseTensorStore::TensorSizeCalc::TensorSizeCalc(const ValueType &type) _aligned_size = my_align(buf_size, alignment); } -DenseTensorStore::BufferType::BufferType(const TensorSizeCalc &tensorSizeCalc, std::unique_ptr<vespalib::alloc::MemoryAllocator> allocator) +DenseTensorStore::BufferType::BufferType(const TensorSizeCalc &tensorSizeCalc, std::shared_ptr<vespalib::alloc::MemoryAllocator> allocator) : vespalib::datastore::BufferType<char>(tensorSizeCalc.alignedSize(), MIN_BUFFER_ARRAYS, RefType::offsetSize()), _allocator(std::move(allocator)) {} @@ -65,7 +65,7 @@ DenseTensorStore::BufferType::get_memory_allocator() const return _allocator.get(); } -DenseTensorStore::DenseTensorStore(const ValueType &type, std::unique_ptr<vespalib::alloc::MemoryAllocator> allocator) +DenseTensorStore::DenseTensorStore(const ValueType &type, std::shared_ptr<vespalib::alloc::MemoryAllocator> allocator) : TensorStore(_concreteStore), _concreteStore(), _tensorSizeCalc(type), diff --git a/searchlib/src/vespa/searchlib/tensor/dense_tensor_store.h b/searchlib/src/vespa/searchlib/tensor/dense_tensor_store.h index 47932fbff7e..7176edbcf08 100644 --- a/searchlib/src/vespa/searchlib/tensor/dense_tensor_store.h +++ b/searchlib/src/vespa/searchlib/tensor/dense_tensor_store.h @@ -37,9 +37,9 @@ public: class BufferType : public vespalib::datastore::BufferType<char> { using CleanContext = vespalib::datastore::BufferType<char>::CleanContext; - std::unique_ptr<vespalib::alloc::MemoryAllocator> _allocator; + std::shared_ptr<vespalib::alloc::MemoryAllocator> _allocator; public: - BufferType(const TensorSizeCalc &tensorSizeCalc, std::unique_ptr<vespalib::alloc::MemoryAllocator> allocator); + BufferType(const TensorSizeCalc &tensorSizeCalc, std::shared_ptr<vespalib::alloc::MemoryAllocator> allocator); ~BufferType() override; void cleanHold(void *buffer, size_t offset, ElemCount numElems, CleanContext cleanCtx) override; const vespalib::alloc::MemoryAllocator* get_memory_allocator() const override; @@ -55,7 +55,7 @@ private: TensorStore::EntryRef setDenseTensor(const TensorType &tensor); public: - DenseTensorStore(const ValueType &type, std::unique_ptr<vespalib::alloc::MemoryAllocator> allocator); + DenseTensorStore(const ValueType &type, std::shared_ptr<vespalib::alloc::MemoryAllocator> allocator); ~DenseTensorStore() override; const ValueType &type() const { return _type; } diff --git a/storage/src/vespa/storage/config/stor-distributormanager.def b/storage/src/vespa/storage/config/stor-distributormanager.def index a01bd30546e..1d2d4babf74 100644 --- a/storage/src/vespa/storage/config/stor-distributormanager.def +++ b/storage/src/vespa/storage/config/stor-distributormanager.def @@ -293,3 +293,9 @@ implicitly_clear_bucket_priority_on_schedule bool default=true ## involved in a given merge have previously reported (as part of bucket info fetching) ## that they support the unordered merge feature. use_unordered_merge_chaining bool default=true + +## If true, inhibits _all_ merges to buckets in the default bucket space if the current +## cluster state bundle indicates that global merges are pending in the cluster, i.e. +## one or more nodes is in maintenance mode in the default bucket space but marked up in +## the global bucket space. +inhibit_default_merges_when_global_merges_pending bool default=false diff --git a/vespalib/src/vespa/vespalib/datastore/unique_store.h b/vespalib/src/vespa/vespalib/datastore/unique_store.h index 281b719deea..81034ab4210 100644 --- a/vespalib/src/vespa/vespalib/datastore/unique_store.h +++ b/vespalib/src/vespa/vespalib/datastore/unique_store.h @@ -50,7 +50,7 @@ private: public: UniqueStore(std::shared_ptr<alloc::MemoryAllocator> memory_allocator); - UniqueStore(std::shared_ptr<alloc::MemoryAllocator> memory_allocator, std::unique_ptr<IUniqueStoreDictionary> dict); + UniqueStore(std::unique_ptr<IUniqueStoreDictionary> dict, std::shared_ptr<alloc::MemoryAllocator> memory_allocator); ~UniqueStore(); void set_dictionary(std::unique_ptr<IUniqueStoreDictionary> dict); UniqueStoreAddResult add(EntryConstRefType value); diff --git a/vespalib/src/vespa/vespalib/datastore/unique_store.hpp b/vespalib/src/vespa/vespalib/datastore/unique_store.hpp index 06a2f288673..b1a7db56545 100644 --- a/vespalib/src/vespa/vespalib/datastore/unique_store.hpp +++ b/vespalib/src/vespa/vespalib/datastore/unique_store.hpp @@ -28,12 +28,12 @@ using DefaultUniqueStoreDictionary = UniqueStoreDictionary<DefaultDictionary>; template <typename EntryT, typename RefT, typename Compare, typename Allocator> UniqueStore<EntryT, RefT, Compare, Allocator>::UniqueStore(std::shared_ptr<alloc::MemoryAllocator> memory_allocator) - : UniqueStore<EntryT, RefT, Compare, Allocator>(std::move(memory_allocator), std::make_unique<uniquestore::DefaultUniqueStoreDictionary>(std::unique_ptr<EntryComparator>())) + : UniqueStore<EntryT, RefT, Compare, Allocator>(std::make_unique<uniquestore::DefaultUniqueStoreDictionary>(std::unique_ptr<EntryComparator>()), std::move(memory_allocator)) { } template <typename EntryT, typename RefT, typename Compare, typename Allocator> -UniqueStore<EntryT, RefT, Compare, Allocator>::UniqueStore(std::shared_ptr<alloc::MemoryAllocator> memory_allocator, std::unique_ptr<IUniqueStoreDictionary> dict) +UniqueStore<EntryT, RefT, Compare, Allocator>::UniqueStore(std::unique_ptr<IUniqueStoreDictionary> dict, std::shared_ptr<alloc::MemoryAllocator> memory_allocator) : _allocator(std::move(memory_allocator)), _store(_allocator.get_data_store()), _dict(std::move(dict)) diff --git a/vespamalloc/src/tests/stacktrace/stacktrace.cpp b/vespamalloc/src/tests/stacktrace/stacktrace.cpp index 2f0d2eb2277..40d77b20e27 100644 --- a/vespamalloc/src/tests/stacktrace/stacktrace.cpp +++ b/vespamalloc/src/tests/stacktrace/stacktrace.cpp @@ -28,7 +28,7 @@ void verify_that_vespamalloc_datasegment_size_exists() { assert(info.keepcost == 0); assert(info.ordblks == 0); assert(info.smblks == 0); - assert(info.uordblks == 0); + assert(info.uordblks > 0); assert(info.usmblks == 0); #else struct mallinfo info = mallinfo(); @@ -42,7 +42,7 @@ void verify_that_vespamalloc_datasegment_size_exists() { assert(info.keepcost == 0); assert(info.ordblks == 0); assert(info.smblks == 0); - assert(info.uordblks == 0); + assert(info.uordblks > 0); assert(info.usmblks == 0); #endif } diff --git a/vespamalloc/src/tests/test1/new_test.cpp b/vespamalloc/src/tests/test1/new_test.cpp index 0723f8cca85..77c07a52918 100644 --- a/vespamalloc/src/tests/test1/new_test.cpp +++ b/vespamalloc/src/tests/test1/new_test.cpp @@ -1,5 +1,6 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include <vespa/vespalib/testkit/testapp.h> +#include <vespa/vespalib/util/size_literals.h> #include <vespa/log/log.h> #include <malloc.h> #include <dlfcn.h> @@ -170,7 +171,19 @@ TEST("verify mallopt") { if (env == MallocLibrary::UNKNOWN) return; EXPECT_EQUAL(0, mallopt(M_MMAP_MAX, 0x1000000)); EXPECT_EQUAL(1, mallopt(M_MMAP_THRESHOLD, 0x1000000)); - EXPECT_EQUAL(1, mallopt(M_MMAP_THRESHOLD, -1)); + EXPECT_EQUAL(1, mallopt(M_MMAP_THRESHOLD, 1_Gi)); +} + +TEST("verify mmap_limit") { + MallocLibrary env = detectLibrary(); + if (env == MallocLibrary::UNKNOWN) return; + EXPECT_EQUAL(1, mallopt(M_MMAP_THRESHOLD, 0x100000)); + auto small = std::make_unique<char[]>(16_Ki); + auto large_1 = std::make_unique<char[]>(1200_Ki); + EXPECT_GREATER(size_t(labs(small.get() - large_1.get())), 1_Ti); + EXPECT_EQUAL(1, mallopt(M_MMAP_THRESHOLD, 1_Gi)); + auto large_2 = std::make_unique<char[]>(1200_Ki); + EXPECT_LESS(size_t(labs(small.get() - large_2.get())), 1_Ti); } diff --git a/vespamalloc/src/vespamalloc/malloc/CMakeLists.txt b/vespamalloc/src/vespamalloc/malloc/CMakeLists.txt index cfa8018e25f..acac1aa5b85 100644 --- a/vespamalloc/src/vespamalloc/malloc/CMakeLists.txt +++ b/vespamalloc/src/vespamalloc/malloc/CMakeLists.txt @@ -4,6 +4,7 @@ vespa_add_library(vespamalloc_malloc OBJECT malloc.cpp allocchunk.cpp common.cpp + mmappool.cpp threadproxy.cpp memblock.cpp datasegment.cpp @@ -17,6 +18,7 @@ vespa_add_library(vespamalloc_mallocd OBJECT mallocd.cpp allocchunk.cpp common.cpp + mmappool.cpp threadproxy.cpp memblockboundscheck.cpp memblockboundscheck_d.cpp @@ -31,6 +33,7 @@ vespa_add_library(vespamalloc_mallocdst16 OBJECT mallocdst16.cpp allocchunk.cpp common.cpp + mmappool.cpp threadproxy.cpp memblockboundscheck.cpp memblockboundscheck_dst.cpp @@ -46,6 +49,7 @@ vespa_add_library(vespamalloc_mallocdst16_nl OBJECT mallocdst16_nl.cpp allocchunk.cpp common.cpp + mmappool.cpp threadproxy.cpp memblockboundscheck.cpp memblockboundscheck_dst.cpp diff --git a/vespamalloc/src/vespamalloc/malloc/common.h b/vespamalloc/src/vespamalloc/malloc/common.h index 4b0af3199a1..65a86b89bf6 100644 --- a/vespamalloc/src/vespamalloc/malloc/common.h +++ b/vespamalloc/src/vespamalloc/malloc/common.h @@ -55,7 +55,7 @@ static constexpr uint32_t NUM_THREADS = 16384; using OSMemory = MmapMemory; using SizeClassT = int; -constexpr size_t ALWAYS_REUSE_LIMIT = 0x200000ul; +constexpr size_t ALWAYS_REUSE_LIMIT = 0x100000ul; inline constexpr int msbIdx(uint64_t v) { return (sizeof(v)*8 - 1) - __builtin_clzl(v); diff --git a/vespamalloc/src/vespamalloc/malloc/datasegment.h b/vespamalloc/src/vespamalloc/malloc/datasegment.h index dc81178150a..d03a585ccc2 100644 --- a/vespamalloc/src/vespamalloc/malloc/datasegment.h +++ b/vespamalloc/src/vespamalloc/malloc/datasegment.h @@ -21,13 +21,14 @@ public: void * getBlock(size_t & oldBlockSize, SizeClassT sc) __attribute__((noinline)); void returnBlock(void *ptr) __attribute__((noinline)); SizeClassT sizeClass(const void * ptr) const { return _blockList[blockId(ptr)].sizeClass(); } + bool containsPtr(const void * ptr) const { return blockId(ptr) < BlockCount; } size_t getMaxSize(const void * ptr) const { return _blockList[blockId(ptr)].getMaxSize(); } const void * start() const { return _osMemory.getStart(); } const void * end() const { return _osMemory.getEnd(); } static SizeClassT adjustedSizeClass(size_t sz) { return (sz >> 16) + 0x400; } static size_t adjustedClassSize(SizeClassT sc) { return (sc > 0x400) ? (sc - 0x400) << 16 : sc; } size_t dataSize() const { return (const char*)end() - (const char*)start(); } - size_t textSize() const { return size_t(start()); } + size_t freeSize() const; size_t infoThread(FILE * os, int level, uint32_t thread, SizeClassT sct, uint32_t maxThreadId=0) const __attribute__((noinline)); void info(FILE * os, size_t level) __attribute__((noinline)); void setupLog(size_t bigMemLogLevel, size_t bigLimit, size_t bigIncrement, size_t allocs2Show) @@ -98,7 +99,8 @@ private: _count--; } } - size_t info(FILE * os, int level) __attribute__((noinline)); + size_t numFreeBlocks() const; + void info(FILE * os) __attribute__((noinline)); private: void * linkOut(size_t findex, size_t left) __attribute__((noinline)); BlockT *_blockList; diff --git a/vespamalloc/src/vespamalloc/malloc/datasegment.hpp b/vespamalloc/src/vespamalloc/malloc/datasegment.hpp index 75ca07a5cf4..80a70b6b5bc 100644 --- a/vespamalloc/src/vespamalloc/malloc/datasegment.hpp +++ b/vespamalloc/src/vespamalloc/malloc/datasegment.hpp @@ -39,6 +39,12 @@ DataSegment<MemBlockPtrT>::DataSegment() : } template<typename MemBlockPtrT> +size_t +DataSegment<MemBlockPtrT>::freeSize() const { + return _freeList.numFreeBlocks() * BlockSize; +} + +template<typename MemBlockPtrT> void * DataSegment<MemBlockPtrT>::getBlock(size_t & oldBlockSize, SizeClassT sc) { const size_t minBlockSize = std::max(size_t(BlockSize), _osMemory.getMinBlockSize()); @@ -273,12 +279,10 @@ void DataSegment<MemBlockPtrT>::info(FILE * os, size_t level) { fprintf(os, "Start at %p, End at %p(%p) size(%ld) partialExtension(%ld) NextLogLimit(%lx) logLevel(%ld)\n", _osMemory.getStart(), _osMemory.getEnd(), sbrk(0), dataSize(), _partialExtension, _nextLogLimit, level); - size_t numFreeBlocks(0), numAllocatedBlocks(0); - { - // Guard sync(_mutex); - numFreeBlocks = _freeList.info(os, level); - _unMappedList.info(os, level); - } + size_t numAllocatedBlocks(0); + size_t numFreeBlocks = _freeList.numFreeBlocks(); + _freeList.info(os); + _unMappedList.info(os); if (level >= 1) { #ifdef PRINT_ALOT SizeClassT oldSc(-17); @@ -430,16 +434,26 @@ size_t DataSegment<MemBlockPtrT>::FreeListT<MaxCount>::lastBlock(size_t nextBloc template<typename MemBlockPtrT> template <int MaxCount> -size_t DataSegment<MemBlockPtrT>::FreeListT<MaxCount>::info(FILE * os, int UNUSED(level)) +void DataSegment<MemBlockPtrT>::FreeListT<MaxCount>::info(FILE * os) { - size_t freeBlockCount(0); for (size_t i=0; i < _count; i++) { size_t index(_freeStartIndex[i]); const BlockT & b = _blockList[index]; - freeBlockCount += b.freeChainLength(); fprintf(os, "Free #%3ld block #%5ld chainlength %5d size %10lu\n", i, index, b.freeChainLength(), size_t(b.freeChainLength())*BlockSize); } +} + +template<typename MemBlockPtrT> +template <int MaxCount> +size_t DataSegment<MemBlockPtrT>::FreeListT<MaxCount>::numFreeBlocks() const +{ + size_t freeBlockCount(0); + for (size_t i=0; i < _count; i++) { + size_t index(_freeStartIndex[i]); + const BlockT & b = _blockList[index]; + freeBlockCount += b.freeChainLength(); + } return freeBlockCount; } diff --git a/vespamalloc/src/vespamalloc/malloc/malloc.h b/vespamalloc/src/vespamalloc/malloc/malloc.h index 2fb0f81826d..f35184cc581 100644 --- a/vespamalloc/src/vespamalloc/malloc/malloc.h +++ b/vespamalloc/src/vespamalloc/malloc/malloc.h @@ -30,13 +30,25 @@ public: void *malloc(size_t sz, std::align_val_t); void *realloc(void *oldPtr, size_t sz); void free(void *ptr) { - freeSC(ptr, _segment.sizeClass(ptr)); + if (_segment.containsPtr(ptr)) { + freeSC(ptr, _segment.sizeClass(ptr)); + } else { + _mmapPool.unmap(MemBlockPtrT(ptr).rawPtr()); + } } void free(void *ptr, size_t sz) { - freeSC(ptr, MemBlockPtrT::sizeClass(MemBlockPtrT::adjustSize(sz))); + if (_segment.containsPtr(ptr)) { + freeSC(ptr, MemBlockPtrT::sizeClass(MemBlockPtrT::adjustSize(sz))); + } else { + _mmapPool.unmap(MemBlockPtrT(ptr).rawPtr()); + } } void free(void *ptr, size_t sz, std::align_val_t alignment) { - freeSC(ptr, MemBlockPtrT::sizeClass(MemBlockPtrT::adjustSize(sz, alignment))); + if (_segment.containsPtr(ptr)) { + freeSC(ptr, MemBlockPtrT::sizeClass(MemBlockPtrT::adjustSize(sz, alignment))); + } else { + _mmapPool.unmap(MemBlockPtrT(ptr).rawPtr()); + } } size_t getMinSizeForAlignment(size_t align, size_t sz) const { return MemBlockPtrT::getMinSizeForAlignment(align, sz); } size_t sizeClass(const void *ptr) const { return _segment.sizeClass(ptr); } @@ -65,6 +77,7 @@ public: _allocPool.setParams(threadCacheLimit); } const DataSegment<MemBlockPtrT> & dataSegment() const { return _segment; } + const MMapPool & mmapPool() const { return _mmapPool; } private: void freeSC(void *ptr, SizeClassT sc); void crash() __attribute__((noinline));; @@ -73,6 +86,7 @@ private: size_t _prAllocLimit; DataSegment<MemBlockPtrT> _segment; AllocPool _allocPool; + MMapPool _mmapPool; ThreadListT _threadList; }; @@ -82,7 +96,8 @@ MemoryManager<MemBlockPtrT, ThreadListT>::MemoryManager(size_t logLimitAtStart) _prAllocLimit(logLimitAtStart), _segment(), _allocPool(_segment), - _threadList(_allocPool) + _mmapPool(), + _threadList(_allocPool, _mmapPool) { setAllocatorForThreads(this); initThisThread(); diff --git a/vespamalloc/src/vespamalloc/malloc/mmappool.cpp b/vespamalloc/src/vespamalloc/malloc/mmappool.cpp new file mode 100644 index 00000000000..296471e54a2 --- /dev/null +++ b/vespamalloc/src/vespamalloc/malloc/mmappool.cpp @@ -0,0 +1,103 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include <vespamalloc/malloc/mmappool.h> +#include <vespamalloc/malloc/common.h> +#include <cassert> +#include <sys/mman.h> + +namespace vespamalloc { + +MMapPool::MMapPool() + : _page_size(getpagesize()), + _huge_flags((getenv("VESPA_USE_HUGEPAGES") != nullptr) ? MAP_HUGETLB : 0), + _count(0), + _mutex(), + _mappings() +{ + +} + +MMapPool::~MMapPool() { + assert(_mappings.empty()); +} + +size_t +MMapPool::getNumMappings() const { + std::lock_guard guard(_mutex); + return _mappings.size(); +} + +size_t +MMapPool::getMmappedBytes() const { + std::lock_guard guard(_mutex); + size_t sum(0); + std::for_each(_mappings.begin(), _mappings.end(), [&sum](const auto & e){ sum += e.second._sz; }); + return sum; +} + +void * +MMapPool::mmap(size_t sz) { + void * buf(nullptr); + assert((sz & (_page_size - 1)) == 0); + if (sz > 0) { + const int flags(MAP_ANON | MAP_PRIVATE); + const int prot(PROT_READ | PROT_WRITE); + size_t mmapId = _count.fetch_add(1); + if (sz >= _G_bigBlockLimit) { + fprintf(_G_logFile, "mmap %ld of size %ld from : ", mmapId, sz); + logStackTrace(); + } + buf = ::mmap(nullptr, sz, prot, flags | _huge_flags, -1, 0); + if (buf == MAP_FAILED) { + if (!_has_hugepage_failure_just_happened) { + _has_hugepage_failure_just_happened = true; + } + buf = ::mmap(nullptr, sz, prot, flags, -1, 0); + if (buf == MAP_FAILED) { + fprintf(_G_logFile, "Failed mmaping anonymous of size %ld errno(%d) from : ", sz, errno); + logStackTrace(); + abort(); + } + } else { + if (_has_hugepage_failure_just_happened) { + _has_hugepage_failure_just_happened = false; + } + } +#ifdef __linux__ + if (sz >= _G_bigBlockLimit) { + if (madvise(buf, sz, MADV_DONTDUMP) != 0) { + std::error_code ec(errno, std::system_category()); + fprintf(_G_logFile, "Failed madvise(%p, %ld, MADV_DONTDUMP) = '%s'\n", buf, sz, + ec.message().c_str()); + } + } +#endif + std::lock_guard guard(_mutex); + auto [it, inserted] = _mappings.insert(std::make_pair(buf, MMapInfo(mmapId, sz))); + assert(inserted); + if (sz >= _G_bigBlockLimit) { + size_t sum(0); + std::for_each(_mappings.begin(), _mappings.end(), [&sum](const auto & e){ sum += e.second._sz; }); + fprintf(_G_logFile, "%ld mappings of accumulated size %ld\n", _mappings.size(), sum); + } + } + return buf; +} + +void +MMapPool::unmap(void * ptr) { + size_t sz; + { + std::lock_guard guard(_mutex); + auto found = _mappings.find(ptr); + if (found == _mappings.end()) { + fprintf(_G_logFile, "Not able to unmap %p as it is not registered: ", ptr); + logStackTrace(); + abort(); + } + sz = found->second._sz; + } + int munmap_ok = ::munmap(ptr, sz); + assert(munmap_ok == 0); +} + +} diff --git a/vespamalloc/src/vespamalloc/malloc/mmappool.h b/vespamalloc/src/vespamalloc/malloc/mmappool.h new file mode 100644 index 00000000000..c07ef9dce15 --- /dev/null +++ b/vespamalloc/src/vespamalloc/malloc/mmappool.h @@ -0,0 +1,33 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#pragma once + +#include <atomic> +#include <unordered_map> + +namespace vespamalloc { + +class MMapPool { +public: + MMapPool(); + MMapPool(const MMapPool &) = delete; + MMapPool & operator =(const MMapPool &) = delete; + ~MMapPool(); + void * mmap(size_t sz); + void unmap(void *); + size_t getNumMappings() const; + size_t getMmappedBytes() const; +private: + struct MMapInfo { + MMapInfo(size_t id, size_t sz) : _id(id), _sz(sz) { } + size_t _id; + size_t _sz; + }; + const size_t _page_size; + const int _huge_flags; + std::atomic<size_t> _count; + std::atomic<bool> _has_hugepage_failure_just_happened; + mutable std::mutex _mutex; + std::unordered_map<const void *, MMapInfo> _mappings; +}; + +} diff --git a/vespamalloc/src/vespamalloc/malloc/overload.h b/vespamalloc/src/vespamalloc/malloc/overload.h index 7d9c2b9c72e..6650f107ca9 100644 --- a/vespamalloc/src/vespamalloc/malloc/overload.h +++ b/vespamalloc/src/vespamalloc/malloc/overload.h @@ -113,12 +113,12 @@ struct mallinfo2 mallinfo2() __THROW { info.arena = vespamalloc::_GmemP->dataSegment().dataSize(); info.ordblks = 0; info.smblks = 0; - info.hblks = 0; - info.hblkhd = 0; + info.hblkhd = vespamalloc::_GmemP->mmapPool().getNumMappings(); + info.hblks = vespamalloc::_GmemP->mmapPool().getMmappedBytes(); info.usmblks = 0; info.fsmblks = 0; - info.uordblks = 0; - info.fordblks = 0; + info.fordblks = vespamalloc::_GmemP->dataSegment().freeSize(); + info.uordblks = info.arena + info.hblks - info.fordblks; info.keepcost = 0; return info; } @@ -129,12 +129,12 @@ struct mallinfo mallinfo() __THROW { info.arena = (vespamalloc::_GmemP->dataSegment().dataSize() >> 20); // Note reporting in 1M blocks info.ordblks = 0; info.smblks = 0; - info.hblks = 0; - info.hblkhd = 0; + info.hblkhd = vespamalloc::_GmemP->mmapPool().getNumMappings(); + info.hblks = (vespamalloc::_GmemP->mmapPool().getMmappedBytes() >> 20); info.usmblks = 0; info.fsmblks = 0; - info.uordblks = 0; - info.fordblks = 0; + info.fordblks = (vespamalloc::_GmemP->dataSegment().freeSize() >> 20); + info.uordblks = info.arena + info.hblks - info.fordblks; info.keepcost = 0; return info; } diff --git a/vespamalloc/src/vespamalloc/malloc/threadlist.h b/vespamalloc/src/vespamalloc/malloc/threadlist.h index c95760dc015..ca3a58483c9 100644 --- a/vespamalloc/src/vespamalloc/malloc/threadlist.h +++ b/vespamalloc/src/vespamalloc/malloc/threadlist.h @@ -17,7 +17,9 @@ class ThreadListT public: using ThreadPool = ThreadPoolT<MemBlockPtrT, ThreadStatT >; using AllocPool = AllocPoolT<MemBlockPtrT>; - ThreadListT(AllocPool & pool); + ThreadListT(AllocPool & allocPool, MMapPool & mmapPool); + ThreadListT(const ThreadListT & tl) = delete; + ThreadListT & operator = (const ThreadListT & tl) = delete; ~ThreadListT(); void setParams(size_t threadCacheLimit) { ThreadPool::setParams(threadCacheLimit); @@ -33,13 +35,12 @@ public: void info(FILE * os, size_t level=0); size_t getMaxNumThreads() const { return NELEMS(_threadVector); } private: - ThreadListT(const ThreadListT & tl); - ThreadListT & operator = (const ThreadListT & tl); std::atomic_flag _isThreaded; std::atomic<uint32_t> _threadCount; std::atomic<uint32_t> _threadCountAccum; ThreadPool _threadVector[NUM_THREADS]; AllocPoolT<MemBlockPtrT> & _allocPool; + MMapPool & _mmapPool; static thread_local ThreadPool * _myPool TLS_LINKAGE; }; diff --git a/vespamalloc/src/vespamalloc/malloc/threadlist.hpp b/vespamalloc/src/vespamalloc/malloc/threadlist.hpp index 8a2cb1de879..5f65e98d0ac 100644 --- a/vespamalloc/src/vespamalloc/malloc/threadlist.hpp +++ b/vespamalloc/src/vespamalloc/malloc/threadlist.hpp @@ -6,14 +6,15 @@ namespace vespamalloc { template <typename MemBlockPtrT, typename ThreadStatT> -ThreadListT<MemBlockPtrT, ThreadStatT>::ThreadListT(AllocPool & pool) : +ThreadListT<MemBlockPtrT, ThreadStatT>::ThreadListT(AllocPool & allocPool, MMapPool & mmapPool) : _isThreaded(false), _threadCount(0), _threadCountAccum(0), - _allocPool(pool) + _allocPool(allocPool), + _mmapPool(mmapPool) { for (size_t i = 0; i < getMaxNumThreads(); i++) { - _threadVector[i].setPool(_allocPool); + _threadVector[i].setPool(_allocPool, _mmapPool); } } diff --git a/vespamalloc/src/vespamalloc/malloc/threadpool.h b/vespamalloc/src/vespamalloc/malloc/threadpool.h index ec89079a415..1276d0129b5 100644 --- a/vespamalloc/src/vespamalloc/malloc/threadpool.h +++ b/vespamalloc/src/vespamalloc/malloc/threadpool.h @@ -5,6 +5,7 @@ #include <vespamalloc/malloc/common.h> #include <vespamalloc/malloc/allocchunk.h> #include <vespamalloc/malloc/globalpool.h> +#include <vespamalloc/malloc/mmappool.h> namespace vespamalloc { @@ -12,12 +13,13 @@ template <typename MemBlockPtrT, typename ThreadStatT > class ThreadPoolT { public: - typedef AFList<MemBlockPtrT> ChunkSList; - typedef AllocPoolT<MemBlockPtrT> AllocPool; + using ChunkSList = AFList<MemBlockPtrT>; + using AllocPool = AllocPoolT<MemBlockPtrT>; ThreadPoolT(); ~ThreadPoolT(); - void setPool(AllocPool & pool) { - _allocPool = & pool; + void setPool(AllocPool & allocPool, MMapPool & mmapPool) { + _allocPool = & allocPool; + _mmapPool = & mmapPool; } int mallopt(int param, int value); void malloc(size_t sz, MemBlockPtrT & mem); @@ -66,7 +68,8 @@ private: static constexpr bool alwaysReuse(SizeClassT sc) { return sc > ALWAYS_REUSE_SC_LIMIT; } AllocPool * _allocPool; - ssize_t _mmapLimit; + MMapPool * _mmapPool; + size_t _mmapLimit; AllocFree _memList[NUM_SIZE_CLASSES]; ThreadStatT _stat[NUM_SIZE_CLASSES]; uint32_t _threadId; diff --git a/vespamalloc/src/vespamalloc/malloc/threadpool.hpp b/vespamalloc/src/vespamalloc/malloc/threadpool.hpp index e9b9fabebdc..31ed1296a9e 100644 --- a/vespamalloc/src/vespamalloc/malloc/threadpool.hpp +++ b/vespamalloc/src/vespamalloc/malloc/threadpool.hpp @@ -6,11 +6,17 @@ namespace vespamalloc { +namespace { + constexpr size_t MMAP_LIMIT_MIN = 0x100000; // 1M + constexpr size_t MMAP_LIMIT_MAX = 0x40000000; // 1G +} + template <typename MemBlockPtrT, typename ThreadStatT> size_t ThreadPoolT<MemBlockPtrT, ThreadStatT>::_threadCacheLimit __attribute__((visibility("hidden"))) = 0x10000; template <typename MemBlockPtrT, typename ThreadStatT> -void ThreadPoolT<MemBlockPtrT, ThreadStatT>::info(FILE * os, size_t level, const DataSegment<MemBlockPtrT> & ds) const { +void +ThreadPoolT<MemBlockPtrT, ThreadStatT>::info(FILE * os, size_t level, const DataSegment<MemBlockPtrT> & ds) const { if (level > 0) { for (size_t i=0; i < NELEMS(_stat); i++) { const ThreadStatT & s = _stat[i]; @@ -50,7 +56,8 @@ void ThreadPoolT<MemBlockPtrT, ThreadStatT>::info(FILE * os, size_t level, const } template <typename MemBlockPtrT, typename ThreadStatT > -void ThreadPoolT<MemBlockPtrT, ThreadStatT>:: +void +ThreadPoolT<MemBlockPtrT, ThreadStatT>:: mallocHelper(size_t exactSize, SizeClassT sc, typename ThreadPoolT<MemBlockPtrT, ThreadStatT>::AllocFree & af, @@ -71,13 +78,20 @@ mallocHelper(size_t exactSize, PARANOID_CHECK2( *(int *)2 = 2; ); } } else { - af._allocFrom = _allocPool->exactAlloc(exactSize, sc, af._allocFrom); - _stat[sc].incExactAlloc(); - if (af._allocFrom) { - af._allocFrom->sub(mem); - PARANOID_CHECK2( if (!mem.ptr()) { *(int *)3 = 3; } ); + if (exactSize > _mmapLimit) { + mem = MemBlockPtrT(_mmapPool->mmap(MemBlockPtrT::classSize(sc)), MemBlockPtrT::classSize(sc)); + // The below settings are to allow the sanity checks conducted at the call site to succeed + mem.setExact(exactSize); + mem.free(); } else { - PARANOID_CHECK2( *(int *)4 = 4; ); + af._allocFrom = _allocPool->exactAlloc(exactSize, sc, af._allocFrom); + _stat[sc].incExactAlloc(); + if (af._allocFrom) { + af._allocFrom->sub(mem); + PARANOID_CHECK2(if (!mem.ptr()) { *(int *) 3 = 3; }); + } else { + PARANOID_CHECK2(*(int *) 4 = 4;); + } } } } @@ -86,7 +100,8 @@ mallocHelper(size_t exactSize, template <typename MemBlockPtrT, typename ThreadStatT > ThreadPoolT<MemBlockPtrT, ThreadStatT>::ThreadPoolT() : _allocPool(nullptr), - _mmapLimit(0x40000000), + _mmapPool(nullptr), + _mmapLimit(MMAP_LIMIT_MAX), _threadId(0), _osThreadId(0) { @@ -97,8 +112,9 @@ ThreadPoolT<MemBlockPtrT, ThreadStatT>::~ThreadPoolT() = default; template <typename MemBlockPtrT, typename ThreadStatT > int ThreadPoolT<MemBlockPtrT, ThreadStatT>::mallopt(int param, int value) { + size_t limit = value; if (param == M_MMAP_THRESHOLD) { - _mmapLimit = value; + _mmapLimit = std::min(MMAP_LIMIT_MAX, std::max(MMAP_LIMIT_MIN, limit)); return 1; } return 0; |