diff options
108 files changed, 1599 insertions, 1245 deletions
diff --git a/config-model/src/main/java/com/yahoo/config/model/producer/AbstractConfigProducer.java b/config-model/src/main/java/com/yahoo/config/model/producer/AbstractConfigProducer.java index f64d71ee364..23aa7e80d74 100644 --- a/config-model/src/main/java/com/yahoo/config/model/producer/AbstractConfigProducer.java +++ b/config-model/src/main/java/com/yahoo/config/model/producer/AbstractConfigProducer.java @@ -203,15 +203,14 @@ public abstract class AbstractConfigProducer<CHILD extends AbstractConfigProduce @Override public final boolean cascadeConfig(ConfigInstance.Builder builder) { - boolean found=false; + boolean found = false; if (parent != null) found = parent.cascadeConfig(builder); boolean foundHere = builder.dispatchGetConfig(this); - if (log.isLoggable(Level.FINE)) { - log.log(Level.FINE, "cascadeconfig in " + this + ", getting config " + - builder.getClass().getDeclaringClass().getName() + " for config id '" + configId + "' found here=" + foundHere); - } + log.log(Level.FINE, () -> "cascadeconfig in " + this + ", getting config " + + builder.getClass().getDeclaringClass().getName() + " for config id '" + configId + + "' found here=" + foundHere); found = found || foundHere; return found; } diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/parser/ParsedType.java b/config-model/src/main/java/com/yahoo/searchdefinition/parser/ParsedType.java index a5f00b1ce45..bcf8d8c9172 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/parser/ParsedType.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/parser/ParsedType.java @@ -153,6 +153,32 @@ class ParsedType { } static ParsedType wsetOf(ParsedType vt) { assert(vt != null); + if (vt.getVariant() != Variant.BUILTIN) { + throw new IllegalArgumentException("weightedset of complex type '" + vt + "' is not supported"); + } + switch (vt.name()) { + // allowed types: + case "bool": + case "byte": + case "int": + case "long": + case "string": + case "uri": + break; + case "predicate": + case "raw": + case "tag": + throw new IllegalArgumentException("weightedset of complex type '" + vt + "' is not supported"); + case "float16": + case "float": + case "double": + /* TODO Vespa 8: + throw new IllegalArgumentException("weightedset of inexact type '" + vt + "' is not supported"); + */ + break; + default: + throw new IllegalArgumentException("weightedset of unknown type '" + vt + "' is not supported"); + } return new ParsedType("weightedset<" + vt.name() + ">", Variant.WSET, vt); } static ParsedType documentRef(ParsedType docType) { diff --git a/config-model/src/main/java/com/yahoo/vespa/model/Affinity.java b/config-model/src/main/java/com/yahoo/vespa/model/Affinity.java index 68696c5ead0..e882f6a2bb5 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/Affinity.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/Affinity.java @@ -6,7 +6,6 @@ package com.yahoo.vespa.model; * CPU socket affinity. * * @author Ulf Lilleengen - * @since 5.12 */ public class Affinity { diff --git a/config-model/src/main/java/com/yahoo/vespa/model/VespaModel.java b/config-model/src/main/java/com/yahoo/vespa/model/VespaModel.java index 205a397178b..c671ad5bd17 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/VespaModel.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/VespaModel.java @@ -643,7 +643,6 @@ public final class VespaModel extends AbstractConfigProducerRoot implements Seri return configModelRepo.getClients(); } - /** Returns all search clusters, both in Search and Content */ public List<SearchCluster> getSearchClusters() { return Content.getSearchClusters(configModelRepo()); } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/VespaMetricSet.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/VespaMetricSet.java index 1b6b43625f7..f66e987bea6 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/VespaMetricSet.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/VespaMetricSet.java @@ -196,6 +196,10 @@ public class VespaMetricSet { metrics.add(new Metric("mem.direct.used.average")); metrics.add(new Metric("mem.direct.used.max")); metrics.add(new Metric("mem.direct.count.max")); + metrics.add(new Metric("mem.native.total.average")); + metrics.add(new Metric("mem.native.free.average")); + metrics.add(new Metric("mem.native.used.average")); + metrics.add(new Metric("mem.native.used.max")); metrics.add(new Metric("jdisc.gc.count.average")); metrics.add(new Metric("jdisc.gc.count.max")); diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerCluster.java index 6295418ca38..04301ff6380 100755 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerCluster.java @@ -29,6 +29,7 @@ import com.yahoo.metrics.simple.runtime.MetricProperties; import com.yahoo.osgi.provider.model.ComponentModel; import com.yahoo.prelude.semantics.SemanticRulesConfig; import com.yahoo.search.config.IndexInfoConfig; +import com.yahoo.search.config.SchemaInfoConfig; import com.yahoo.search.config.QrStartConfig; import com.yahoo.search.pagetemplates.PageTemplatesConfig; import com.yahoo.search.query.profile.config.QueryProfilesConfig; @@ -99,7 +100,8 @@ public abstract class ContainerCluster<CONTAINER extends Container> DocprocConfig.Producer, ClusterInfoConfig.Producer, ConfigserverConfig.Producer, - CuratorConfig.Producer + CuratorConfig.Producer, + SchemaInfoConfig.Producer { /** @@ -537,6 +539,11 @@ public abstract class ContainerCluster<CONTAINER extends Container> if (containerSearch != null) containerSearch.getConfig(builder); } + @Override + public void getConfig(SchemaInfoConfig.Builder builder) { + if (containerSearch != null) containerSearch.getConfig(builder); + } + public void initialize(Map<String, SearchCluster> clusterMap) { if (containerSearch != null) containerSearch.connectSearchClusters(clusterMap); } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/search/ContainerSearch.java b/config-model/src/main/java/com/yahoo/vespa/model/container/search/ContainerSearch.java index 2788b29ac0a..789b0fc6fb7 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/search/ContainerSearch.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/search/ContainerSearch.java @@ -4,6 +4,7 @@ package com.yahoo.vespa.model.container.search; import com.yahoo.container.QrSearchersConfig; import com.yahoo.prelude.semantics.SemanticRulesConfig; import com.yahoo.search.config.IndexInfoConfig; +import com.yahoo.search.config.SchemaInfoConfig; import com.yahoo.search.pagetemplates.PageTemplatesConfig; import com.yahoo.search.query.profile.compiled.CompiledQueryProfileRegistry; import com.yahoo.search.query.profile.config.QueryProfilesConfig; @@ -35,11 +36,12 @@ public class ContainerSearch extends ContainerSubsystem<SearchChains> QrSearchersConfig.Producer, QueryProfilesConfig.Producer, SemanticRulesConfig.Producer, - PageTemplatesConfig.Producer { + PageTemplatesConfig.Producer, + SchemaInfoConfig.Producer { public static final String QUERY_PROFILE_REGISTRY_CLASS = CompiledQueryProfileRegistry.class.getName(); - private ApplicationContainerCluster owningCluster; + private final ApplicationContainerCluster owningCluster; private final List<SearchCluster> searchClusters = new LinkedList<>(); private final Options options; @@ -119,6 +121,36 @@ public class ContainerSearch extends ContainerSubsystem<SearchChains> } @Override + public void getConfig(SchemaInfoConfig.Builder builder) { + Map<String, SearchCluster.SchemaInfo> allSchemas = new LinkedHashMap<>(); + for (SearchCluster sc : searchClusters) + allSchemas.putAll(sc.schemas()); + + for (var schemaEntry : allSchemas.entrySet()) { + var schemaBuilder = new SchemaInfoConfig.Schema.Builder(); + schemaBuilder.name(schemaEntry.getKey()); + addRankProfilesConfig(schemaEntry.getValue(), schemaBuilder); + builder.schema(schemaBuilder); + } + } + + protected void addRankProfilesConfig(SearchCluster.SchemaInfo schema, SchemaInfoConfig.Schema.Builder schemaBuilder) { + for (SearchCluster.RankProfileInfo rankProfile : schema.rankProfiles().values()) { + var rankProfileConfig = new SchemaInfoConfig.Schema.Rankprofile.Builder(); + rankProfileConfig.name(rankProfile.name()); + rankProfileConfig.hasSummaryFeatures(rankProfile.hasSummaryFeatures()); + rankProfileConfig.hasRankFeatures(rankProfile.hasRankFeatures()); + for (var input : rankProfile.inputs().entrySet()) { + var inputConfig = new SchemaInfoConfig.Schema.Rankprofile.Input.Builder(); + inputConfig.name(input.getKey().toString()); + inputConfig.type(input.getValue().toString()); + rankProfileConfig.input(inputConfig); + } + schemaBuilder.rankprofile(rankProfileConfig); + } + } + + @Override public void getConfig(QrSearchersConfig.Builder builder) { for (int i = 0; i < searchClusters.size(); i++) { SearchCluster sys = findClusterWithId(searchClusters, i); diff --git a/config-model/src/test/derived/attributeprefetch/attributeprefetch.sd b/config-model/src/test/derived/attributeprefetch/attributeprefetch.sd index e778c590aac..296a1fd828d 100644 --- a/config-model/src/test/derived/attributeprefetch/attributeprefetch.sd +++ b/config-model/src/test/derived/attributeprefetch/attributeprefetch.sd @@ -34,18 +34,12 @@ search prefetch { field multifloat type array<float> { indexing: attribute } - field wsfloat type weightedset<float> { - indexing: attribute - } field singledouble type double { indexing: attribute } field multidouble type array<double> { indexing: attribute } - field wsdouble type weightedset<double> { - indexing: attribute - } field singlestring type string { indexing: attribute match { diff --git a/config-model/src/test/derived/attributeprefetch/attributes.cfg b/config-model/src/test/derived/attributeprefetch/attributes.cfg index b6f37a216a7..8b49dc1dc23 100644 --- a/config-model/src/test/derived/attributeprefetch/attributes.cfg +++ b/config-model/src/test/derived/attributeprefetch/attributes.cfg @@ -350,38 +350,6 @@ attribute[].index.hnsw.maxlinkspernode 16 attribute[].index.hnsw.neighborstoexploreatinsert 200 attribute[].index.hnsw.distancemetric EUCLIDEAN attribute[].index.hnsw.multithreadedindexing true -attribute[].name "wsfloat" -attribute[].datatype FLOAT -attribute[].collectiontype WEIGHTEDSET -attribute[].dictionary.type BTREE -attribute[].dictionary.match UNCASED -attribute[].match UNCASED -attribute[].removeifzero false -attribute[].createifnonexistent false -attribute[].fastsearch false -attribute[].huge false -attribute[].paged false -attribute[].ismutable false -attribute[].sortascending true -attribute[].sortfunction UCA -attribute[].sortstrength PRIMARY -attribute[].sortlocale "" -attribute[].enablebitvectors false -attribute[].enableonlybitvector false -attribute[].fastaccess false -attribute[].arity 8 -attribute[].lowerbound -9223372036854775808 -attribute[].upperbound 9223372036854775807 -attribute[].densepostinglistthreshold 0.4 -attribute[].tensortype "" -attribute[].imported false -attribute[].maxuncommittedmemory 77777 -attribute[].distancemetric EUCLIDEAN -attribute[].index.hnsw.enabled false -attribute[].index.hnsw.maxlinkspernode 16 -attribute[].index.hnsw.neighborstoexploreatinsert 200 -attribute[].index.hnsw.distancemetric EUCLIDEAN -attribute[].index.hnsw.multithreadedindexing true attribute[].name "singledouble" attribute[].datatype DOUBLE attribute[].collectiontype SINGLE @@ -446,38 +414,6 @@ attribute[].index.hnsw.maxlinkspernode 16 attribute[].index.hnsw.neighborstoexploreatinsert 200 attribute[].index.hnsw.distancemetric EUCLIDEAN attribute[].index.hnsw.multithreadedindexing true -attribute[].name "wsdouble" -attribute[].datatype DOUBLE -attribute[].collectiontype WEIGHTEDSET -attribute[].dictionary.type BTREE -attribute[].dictionary.match UNCASED -attribute[].match UNCASED -attribute[].removeifzero false -attribute[].createifnonexistent false -attribute[].fastsearch false -attribute[].huge false -attribute[].paged false -attribute[].ismutable false -attribute[].sortascending true -attribute[].sortfunction UCA -attribute[].sortstrength PRIMARY -attribute[].sortlocale "" -attribute[].enablebitvectors false -attribute[].enableonlybitvector false -attribute[].fastaccess false -attribute[].arity 8 -attribute[].lowerbound -9223372036854775808 -attribute[].upperbound 9223372036854775807 -attribute[].densepostinglistthreshold 0.4 -attribute[].tensortype "" -attribute[].imported false -attribute[].maxuncommittedmemory 77777 -attribute[].distancemetric EUCLIDEAN -attribute[].index.hnsw.enabled false -attribute[].index.hnsw.maxlinkspernode 16 -attribute[].index.hnsw.neighborstoexploreatinsert 200 -attribute[].index.hnsw.distancemetric EUCLIDEAN -attribute[].index.hnsw.multithreadedindexing true attribute[].name "singlestring" attribute[].datatype STRING attribute[].collectiontype SINGLE diff --git a/config-model/src/test/derived/attributeprefetch/documentmanager.cfg b/config-model/src/test/derived/attributeprefetch/documentmanager.cfg index 373e0211253..9d91ad3f5ab 100644 --- a/config-model/src/test/derived/attributeprefetch/documentmanager.cfg +++ b/config-model/src/test/derived/attributeprefetch/documentmanager.cfg @@ -56,8 +56,6 @@ doctype[].fieldsets{[document]}.fields[] "singleint" doctype[].fieldsets{[document]}.fields[] "singlelong" doctype[].fieldsets{[document]}.fields[] "singlestring" doctype[].fieldsets{[document]}.fields[] "wsbyte" -doctype[].fieldsets{[document]}.fields[] "wsdouble" -doctype[].fieldsets{[document]}.fields[] "wsfloat" doctype[].fieldsets{[document]}.fields[] "wsint" doctype[].fieldsets{[document]}.fields[] "wslong" doctype[].fieldsets{[document]}.fields[] "wsstring" @@ -69,9 +67,9 @@ doctype[].arraytype[].idx 10021 doctype[].arraytype[].elementtype 10008 doctype[].arraytype[].idx 10023 doctype[].arraytype[].elementtype 10005 -doctype[].arraytype[].idx 10025 +doctype[].arraytype[].idx 10024 doctype[].arraytype[].elementtype 10004 -doctype[].arraytype[].idx 10027 +doctype[].arraytype[].idx 10025 doctype[].arraytype[].elementtype 10012 doctype[].wsettype[].idx 10018 doctype[].wsettype[].elementtype 10003 @@ -85,15 +83,7 @@ doctype[].wsettype[].idx 10022 doctype[].wsettype[].elementtype 10008 doctype[].wsettype[].createifnonexistent false doctype[].wsettype[].removeifzero false -doctype[].wsettype[].idx 10024 -doctype[].wsettype[].elementtype 10005 -doctype[].wsettype[].createifnonexistent false -doctype[].wsettype[].removeifzero false doctype[].wsettype[].idx 10026 -doctype[].wsettype[].elementtype 10004 -doctype[].wsettype[].createifnonexistent false -doctype[].wsettype[].removeifzero false -doctype[].wsettype[].idx 10028 doctype[].wsettype[].elementtype 10012 doctype[].wsettype[].createifnonexistent false doctype[].wsettype[].removeifzero false @@ -132,24 +122,18 @@ doctype[].structtype[].field[].type 10005 doctype[].structtype[].field[].name "multifloat" doctype[].structtype[].field[].internalid 1028626753 doctype[].structtype[].field[].type 10023 -doctype[].structtype[].field[].name "wsfloat" -doctype[].structtype[].field[].internalid 2087992058 -doctype[].structtype[].field[].type 10024 doctype[].structtype[].field[].name "singledouble" doctype[].structtype[].field[].internalid 1982688634 doctype[].structtype[].field[].type 10004 doctype[].structtype[].field[].name "multidouble" doctype[].structtype[].field[].internalid 1316159002 -doctype[].structtype[].field[].type 10025 -doctype[].structtype[].field[].name "wsdouble" -doctype[].structtype[].field[].internalid 995331392 -doctype[].structtype[].field[].type 10026 +doctype[].structtype[].field[].type 10024 doctype[].structtype[].field[].name "singlestring" doctype[].structtype[].field[].internalid 289408547 doctype[].structtype[].field[].type 10012 doctype[].structtype[].field[].name "multistring" doctype[].structtype[].field[].internalid 862051908 -doctype[].structtype[].field[].type 10027 +doctype[].structtype[].field[].type 10025 doctype[].structtype[].field[].name "wsstring" doctype[].structtype[].field[].internalid 447961272 -doctype[].structtype[].field[].type 10028 +doctype[].structtype[].field[].type 10026 diff --git a/config-model/src/test/derived/attributeprefetch/ilscripts.cfg b/config-model/src/test/derived/attributeprefetch/ilscripts.cfg index 61f4cde6e66..771a54359b9 100644 --- a/config-model/src/test/derived/attributeprefetch/ilscripts.cfg +++ b/config-model/src/test/derived/attributeprefetch/ilscripts.cfg @@ -12,10 +12,8 @@ ilscript[].docfield[] "multilong" ilscript[].docfield[] "wslong" ilscript[].docfield[] "singlefloat" ilscript[].docfield[] "multifloat" -ilscript[].docfield[] "wsfloat" ilscript[].docfield[] "singledouble" ilscript[].docfield[] "multidouble" -ilscript[].docfield[] "wsdouble" ilscript[].docfield[] "singlestring" ilscript[].docfield[] "multistring" ilscript[].docfield[] "wsstring" @@ -30,10 +28,8 @@ ilscript[].content[] "clear_state | guard { input multilong | attribute multilon ilscript[].content[] "clear_state | guard { input wslong | attribute wslong; }" ilscript[].content[] "clear_state | guard { input singlefloat | attribute singlefloat; }" ilscript[].content[] "clear_state | guard { input multifloat | attribute multifloat; }" -ilscript[].content[] "clear_state | guard { input wsfloat | attribute wsfloat; }" ilscript[].content[] "clear_state | guard { input singledouble | attribute singledouble; }" ilscript[].content[] "clear_state | guard { input multidouble | attribute multidouble; }" -ilscript[].content[] "clear_state | guard { input wsdouble | attribute wsdouble; }" ilscript[].content[] "clear_state | guard { input singlestring | attribute singlestring; }" ilscript[].content[] "clear_state | guard { input multistring | attribute multistring; }" ilscript[].content[] "clear_state | guard { input wsstring | attribute wsstring; }" diff --git a/config-model/src/test/derived/attributeprefetch/index-info.cfg b/config-model/src/test/derived/attributeprefetch/index-info.cfg index e1b20b33e17..28e669d3571 100644 --- a/config-model/src/test/derived/attributeprefetch/index-info.cfg +++ b/config-model/src/test/derived/attributeprefetch/index-info.cfg @@ -105,16 +105,6 @@ indexinfo[].command[].indexname "multifloat" indexinfo[].command[].command "numerical" indexinfo[].command[].indexname "multifloat" indexinfo[].command[].command "type Array<float>" -indexinfo[].command[].indexname "wsfloat" -indexinfo[].command[].command "index" -indexinfo[].command[].indexname "wsfloat" -indexinfo[].command[].command "multivalue" -indexinfo[].command[].indexname "wsfloat" -indexinfo[].command[].command "attribute" -indexinfo[].command[].indexname "wsfloat" -indexinfo[].command[].command "numerical" -indexinfo[].command[].indexname "wsfloat" -indexinfo[].command[].command "type WeightedSet<float>" indexinfo[].command[].indexname "singledouble" indexinfo[].command[].command "index" indexinfo[].command[].indexname "singledouble" @@ -133,16 +123,6 @@ indexinfo[].command[].indexname "multidouble" indexinfo[].command[].command "numerical" indexinfo[].command[].indexname "multidouble" indexinfo[].command[].command "type Array<double>" -indexinfo[].command[].indexname "wsdouble" -indexinfo[].command[].command "index" -indexinfo[].command[].indexname "wsdouble" -indexinfo[].command[].command "multivalue" -indexinfo[].command[].indexname "wsdouble" -indexinfo[].command[].command "attribute" -indexinfo[].command[].indexname "wsdouble" -indexinfo[].command[].command "numerical" -indexinfo[].command[].indexname "wsdouble" -indexinfo[].command[].command "type WeightedSet<double>" indexinfo[].command[].indexname "singlestring" indexinfo[].command[].command "index" indexinfo[].command[].indexname "singlestring" diff --git a/config-model/src/test/derived/attributes/attributes.cfg b/config-model/src/test/derived/attributes/attributes.cfg index cbc3280f213..cf473ad4854 100644 --- a/config-model/src/test/derived/attributes/attributes.cfg +++ b/config-model/src/test/derived/attributes/attributes.cfg @@ -351,7 +351,7 @@ attribute[].index.hnsw.neighborstoexploreatinsert 200 attribute[].index.hnsw.distancemetric EUCLIDEAN attribute[].index.hnsw.multithreadedindexing true attribute[].name "b7" -attribute[].datatype DOUBLE +attribute[].datatype INT32 attribute[].collectiontype WEIGHTEDSET attribute[].dictionary.type BTREE attribute[].dictionary.match UNCASED diff --git a/config-model/src/test/derived/attributes/attributes.sd b/config-model/src/test/derived/attributes/attributes.sd index f38087fbc6f..df647c833f4 100644 --- a/config-model/src/test/derived/attributes/attributes.sd +++ b/config-model/src/test/derived/attributes/attributes.sd @@ -74,7 +74,7 @@ search attributes { field b6 type array<long> { indexing: summary | attribute } - field b7 type weightedset<double> { + field b7 type weightedset<int> { indexing: summary | attribute } diff --git a/config-model/src/test/derived/attributes/index-info.cfg b/config-model/src/test/derived/attributes/index-info.cfg index 9aeb79a7e2d..560da741ad9 100644 --- a/config-model/src/test/derived/attributes/index-info.cfg +++ b/config-model/src/test/derived/attributes/index-info.cfg @@ -132,7 +132,7 @@ indexinfo[].command[].command "attribute" indexinfo[].command[].indexname "b7" indexinfo[].command[].command "numerical" indexinfo[].command[].indexname "b7" -indexinfo[].command[].command "type WeightedSet<double>" +indexinfo[].command[].command "type WeightedSet<int>" indexinfo[].command[].indexname "a9" indexinfo[].command[].command "index" indexinfo[].command[].indexname "a9" diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/TensorTransformTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/processing/TensorTransformTestCase.java index c71fd3dd489..e3ac5da8955 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/processing/TensorTransformTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/processing/TensorTransformTestCase.java @@ -150,7 +150,7 @@ public class TensorTransformTestCase extends AbstractSchemaTestCase { " field double_array_field type array<double> { \n" + " indexing: summary | attribute \n" + " }\n" + - " field weightedset_field type weightedset<double> { \n" + + " field weightedset_field type weightedset<int> { \n" + " indexing: summary | attribute \n" + " }\n" + " field tensor_field_1 type tensor(x{}) { \n" + diff --git a/config-model/src/test/java/com/yahoo/vespa/model/search/test/DocumentDatabaseTestCase.java b/config-model/src/test/java/com/yahoo/vespa/model/search/test/DocumentDatabaseTestCase.java index 2cdd6521661..3cff04198bd 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/search/test/DocumentDatabaseTestCase.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/search/test/DocumentDatabaseTestCase.java @@ -4,6 +4,7 @@ package com.yahoo.vespa.model.search.test; import com.google.common.collect.ImmutableMap; import com.yahoo.config.model.deploy.DeployState; import com.yahoo.config.model.deploy.TestProperties; +import com.yahoo.search.config.SchemaInfoConfig; import com.yahoo.vespa.config.search.IndexschemaConfig; import com.yahoo.vespa.config.search.core.ProtonConfig; import com.yahoo.vespa.config.search.RankProfilesConfig; @@ -344,6 +345,53 @@ public class DocumentDatabaseTestCase { } } + /** Schema-info should contain all schemas, independent of clusters. */ + @Test + public void requireThatSchemaInfoIsAvailable() { + List<String> schemas = Arrays.asList("type1", "type2"); + VespaModel model = new VespaModelCreatorWithMockPkg(vespaHosts, createVespaServices(schemas, "index"), + ApplicationPackageUtils.generateSchemas(schemas)).create(); + assertSchemaInfo("container/searchchains/chain/test/component/com.yahoo.prelude.cluster.ClusterSearcher", model); + assertSchemaInfo("container", model); + } + + private void assertSchemaInfo(String configId, VespaModel model) { + { // schema-info config + SchemaInfoConfig dcfg = model.getConfig(SchemaInfoConfig.class, configId); + assertEquals(2, dcfg.schema().size()); + + { // type1 + SchemaInfoConfig.Schema schema = dcfg.schema(0); + assertEquals("type1", schema.name()); + + assertEquals(7, schema.rankprofile().size()); + assertRankProfile(schema, 0, "default", false, false); + assertRankProfile(schema, 1, "unranked", false, false); + assertRankProfile(schema, 2, "staticrank", false, false); + assertRankProfile(schema, 3, "summaryfeatures", true, false); + assertRankProfile(schema, 4, "inheritedsummaryfeatures", true, false); + assertRankProfile(schema, 5, "rankfeatures", false, true); + var inputs = assertRankProfile(schema, 6, "inputs", false, false); + + assertEquals(2, inputs.input().size()); + assertEquals("query(foo)", inputs.input(0).name()); + assertEquals("tensor<float>(x[10])", inputs.input(0).type()); + assertEquals("query(bar)", inputs.input(1).name()); + assertEquals("tensor(key{},x[1000])", inputs.input(1).type()); + + // assertEquals(2, schema.summaryclass().size()); + // assertEquals("default", schema.summaryclass(0).name()); + // assertEquals("attributeprefetch", schema.summaryclass(1).name()); + // assertSummaryField(schema, 0, 0, "f1", "longstring", true); + // assertSummaryField(schema, 0, 1, "f2", "integer", false); + } + { // type2 + SchemaInfoConfig.Schema schema = dcfg.schema(1); + assertEquals("type2", schema.name()); + } + } + } + private DocumentdbInfoConfig.Documentdb.Rankprofile assertRankProfile(DocumentdbInfoConfig.Documentdb db, int index, String name, @@ -356,6 +404,18 @@ public class DocumentDatabaseTestCase { return rankProfile; } + private SchemaInfoConfig.Schema.Rankprofile assertRankProfile(SchemaInfoConfig.Schema schema, + int index, + String name, + boolean hasSummaryFeatures, + boolean hasRankFeatures) { + SchemaInfoConfig.Schema.Rankprofile rankProfile = schema.rankprofile(index); + assertEquals(name, rankProfile.name()); + assertEquals(hasSummaryFeatures, rankProfile.hasSummaryFeatures()); + assertEquals(hasRankFeatures, rankProfile.hasRankFeatures()); + return rankProfile; + } + private void assertSummaryField(DocumentdbInfoConfig.Documentdb db, int summaryClassIndex, int fieldIndex, String name, String type, boolean dynamic) { DocumentdbInfoConfig.Documentdb.Summaryclass.Fields field = db.summaryclass(summaryClassIndex).fields(fieldIndex); diff --git a/config/src/vespa/config/subscription/configsubscriptionset.cpp b/config/src/vespa/config/subscription/configsubscriptionset.cpp index 1d5db778dda..48e2bdc2615 100644 --- a/config/src/vespa/config/subscription/configsubscriptionset.cpp +++ b/config/src/vespa/config/subscription/configsubscriptionset.cpp @@ -42,7 +42,7 @@ ConfigSubscriptionSet::acquireSnapshot(duration timeout, bool ignoreChange) steady_time now = steady_clock::now(); const steady_time deadline = now + timeout; - int64_t lastGeneration = _currentGeneration; + int64_t lastGeneration = getGeneration(); bool inSync = false; LOG(spam, "Going into nextConfig loop, time left is %f", vespalib::to_s(deadline - now)); @@ -55,7 +55,7 @@ ConfigSubscriptionSet::acquireSnapshot(duration timeout, bool ignoreChange) // Run nextUpdate on all subscribers to get them in sync. for (const auto & subscription : _subscriptionList) { - if (!subscription->nextUpdate(_currentGeneration, deadline) && !subscription->hasGenerationChanged()) { + if (!subscription->nextUpdate(getGeneration(), deadline) && !subscription->hasGenerationChanged()) { subscription->reset(); continue; } @@ -68,7 +68,7 @@ ConfigSubscriptionSet::acquireSnapshot(duration timeout, bool ignoreChange) LOG(spam, "Config subscription did not change, id(%s), defname(%s)", key.getConfigId().c_str(), key.getDefName().c_str()); } LOG(spam, "Previous generation is %" PRId64 ", updates is %" PRId64, lastGeneration, subscription->getGeneration()); - if (isGenerationNewer(subscription->getGeneration(), _currentGeneration)) { + if (isGenerationNewer(subscription->getGeneration(), getGeneration())) { numGenerationChanged++; } if (generation < 0) { @@ -88,10 +88,10 @@ ConfigSubscriptionSet::acquireSnapshot(duration timeout, bool ignoreChange) } } - bool updated = inSync && isGenerationNewer(lastGeneration, _currentGeneration); + bool updated = inSync && isGenerationNewer(lastGeneration, getGeneration()); if (updated) { - LOG(spam, "Config was updated from %" PRId64 " to %" PRId64, _currentGeneration, lastGeneration); - _currentGeneration = lastGeneration; + LOG(spam, "Config was updated from %" PRId64 " to %" PRId64, getGeneration(), lastGeneration); + _currentGeneration.store(lastGeneration, std::memory_order_relaxed); _state = CONFIGURED; for (const auto & subscription : _subscriptionList) { const ConfigKey & key(subscription->getKey()); diff --git a/config/src/vespa/config/subscription/configsubscriptionset.h b/config/src/vespa/config/subscription/configsubscriptionset.h index 4b6d970770d..8daf7ae91ea 100644 --- a/config/src/vespa/config/subscription/configsubscriptionset.h +++ b/config/src/vespa/config/subscription/configsubscriptionset.h @@ -39,7 +39,7 @@ public: * @return generation number */ int64_t getGeneration() const noexcept { - return _currentGeneration; + return _currentGeneration.load(std::memory_order_relaxed); } /** @@ -69,7 +69,7 @@ private: const vespalib::duration _maxNapTime; std::shared_ptr<IConfigContext> _context; // Context to keep alive managers. IConfigManager & _mgr; // The config manager that we use. - int64_t _currentGeneration; // Holds the current config generation. + std::atomic<int64_t> _currentGeneration; // Holds the current config generation. SubscriptionList _subscriptionList; // List of current subscriptions. std::atomic<SubscriberState> _state; // Current state of this subscriber. }; diff --git a/container-disc/src/main/java/com/yahoo/container/jdisc/metric/MetricUpdater.java b/container-disc/src/main/java/com/yahoo/container/jdisc/metric/MetricUpdater.java index 62982d66978..4659c2acc36 100644 --- a/container-disc/src/main/java/com/yahoo/container/jdisc/metric/MetricUpdater.java +++ b/container-disc/src/main/java/com/yahoo/container/jdisc/metric/MetricUpdater.java @@ -5,6 +5,8 @@ import com.google.inject.Inject; import com.yahoo.component.AbstractComponent; import com.yahoo.jdisc.Metric; import com.yahoo.jdisc.statistics.ContainerWatchdogMetrics; +import com.yahoo.nativec.NativeHeap; + import java.lang.management.BufferPoolMXBean; import java.lang.management.ManagementFactory; import java.nio.file.DirectoryStream; @@ -24,6 +26,9 @@ import java.util.TimerTask; */ public class MetricUpdater extends AbstractComponent { + private static final String NATIVE_FREE_MEMORY_BYTES = "mem.native.free"; + private static final String NATIVE_USED_MEMORY_BYTES = "mem.native.used"; + private static final String NATIVE_TOTAL_MEMORY_BYTES = "mem.native.total"; private static final String HEAP_FREE_MEMORY_BYTES = "mem.heap.free"; private static final String HEAP_USED_MEMORY_BYTES = "mem.heap.used"; private static final String HEAP_TOTAL_MEMORY_BYTES = "mem.heap.total"; @@ -116,6 +121,13 @@ public class MetricUpdater extends AbstractComponent { metric.set(DIRECT_COUNT, count, null); } + private void nativeHeapUsed() { + NativeHeap nativeHeap = NativeHeap.sample(); + metric.set(NATIVE_FREE_MEMORY_BYTES, nativeHeap.availableSize(), null); + metric.set(NATIVE_USED_MEMORY_BYTES, nativeHeap.usedSize(), null); + metric.set(NATIVE_TOTAL_MEMORY_BYTES, nativeHeap.totalSize(), null); + } + @Override public void run() { long freeMemory = runtime.freeMemory(); @@ -127,6 +139,7 @@ public class MetricUpdater extends AbstractComponent { metric.set(MEMORY_MAPPINGS_COUNT, count_mappings(), null); metric.set(OPEN_FILE_DESCRIPTORS, count_open_files(), null); directMemoryUsed(); + nativeHeapUsed(); containerWatchdogMetrics.emitMetrics(metric); garbageCollectionMetrics.emitMetrics(metric); diff --git a/container-disc/src/test/java/com/yahoo/container/jdisc/metric/MetricUpdaterTest.java b/container-disc/src/test/java/com/yahoo/container/jdisc/metric/MetricUpdaterTest.java index 68d73a7914a..f49ccf2c2f6 100644 --- a/container-disc/src/test/java/com/yahoo/container/jdisc/metric/MetricUpdaterTest.java +++ b/container-disc/src/test/java/com/yahoo/container/jdisc/metric/MetricUpdaterTest.java @@ -27,7 +27,7 @@ public class MetricUpdaterTest { ContainerWatchdogMetrics containerWatchdogMetrics = mock(ContainerWatchdogMetrics.class); new MetricUpdater(new MockScheduler(), metric, containerWatchdogMetrics); verify(containerWatchdogMetrics, times(1)).emitMetrics(any()); - verify(metric, times(9 + 2 * gcCount)).set(anyString(), any(), any()); + verify(metric, times(12 + 2 * gcCount)).set(anyString(), any(), any()); } private static class MockScheduler implements MetricUpdater.Scheduler { diff --git a/container-search/abi-spec.json b/container-search/abi-spec.json index 2ab51f975a4..b4c4537a708 100644 --- a/container-search/abi-spec.json +++ b/container-search/abi-spec.json @@ -1905,8 +1905,8 @@ "public java.util.Map getEmbedders()", "public com.yahoo.search.Query$Builder setZoneInfo(ai.vespa.cloud.ZoneInfo)", "public ai.vespa.cloud.ZoneInfo getZoneInfo()", - "public com.yahoo.search.Query$Builder setSchemaInfo(com.yahoo.search.config.SchemaInfo)", - "public com.yahoo.search.config.SchemaInfo getSchemaInfo()", + "public com.yahoo.search.Query$Builder setSchemaInfo(com.yahoo.search.schema.SchemaInfo)", + "public com.yahoo.search.schema.SchemaInfo getSchemaInfo()", "public com.yahoo.search.Query build()" ], "fields": [] @@ -2290,580 +2290,6 @@ ], "fields": [] }, - "com.yahoo.search.config.ClusterConfig$Builder": { - "superClass": "java.lang.Object", - "interfaces": [ - "com.yahoo.config.ConfigInstance$Builder" - ], - "attributes": [ - "public" - ], - "methods": [ - "public void <init>()", - "public void <init>(com.yahoo.search.config.ClusterConfig)", - "public com.yahoo.search.config.ClusterConfig$Builder clusterId(int)", - "public com.yahoo.search.config.ClusterConfig$Builder cacheSize(int)", - "public com.yahoo.search.config.ClusterConfig$Builder cacheTimeout(double)", - "public com.yahoo.search.config.ClusterConfig$Builder failoverToRemote(boolean)", - "public com.yahoo.search.config.ClusterConfig$Builder clusterName(java.lang.String)", - "public com.yahoo.search.config.ClusterConfig$Builder maxQueryTimeout(double)", - "public com.yahoo.search.config.ClusterConfig$Builder maxQueryCacheTimeout(double)", - "public final boolean dispatchGetConfig(com.yahoo.config.ConfigInstance$Producer)", - "public final java.lang.String getDefMd5()", - "public final java.lang.String getDefName()", - "public final java.lang.String getDefNamespace()", - "public final boolean getApplyOnRestart()", - "public final void setApplyOnRestart(boolean)", - "public com.yahoo.search.config.ClusterConfig build()" - ], - "fields": [] - }, - "com.yahoo.search.config.ClusterConfig$Producer": { - "superClass": "java.lang.Object", - "interfaces": [ - "com.yahoo.config.ConfigInstance$Producer" - ], - "attributes": [ - "public", - "interface", - "abstract" - ], - "methods": [ - "public abstract void getConfig(com.yahoo.search.config.ClusterConfig$Builder)" - ], - "fields": [] - }, - "com.yahoo.search.config.ClusterConfig": { - "superClass": "com.yahoo.config.ConfigInstance", - "interfaces": [], - "attributes": [ - "public", - "final" - ], - "methods": [ - "public static java.lang.String getDefMd5()", - "public static java.lang.String getDefName()", - "public static java.lang.String getDefNamespace()", - "public static java.lang.String getDefVersion()", - "public void <init>(com.yahoo.search.config.ClusterConfig$Builder)", - "public int clusterId()", - "public int cacheSize()", - "public double cacheTimeout()", - "public boolean failoverToRemote()", - "public java.lang.String clusterName()", - "public double maxQueryTimeout()", - "public double maxQueryCacheTimeout()" - ], - "fields": [ - "public static final java.lang.String CONFIG_DEF_MD5", - "public static final java.lang.String CONFIG_DEF_NAME", - "public static final java.lang.String CONFIG_DEF_NAMESPACE", - "public static final java.lang.String CONFIG_DEF_VERSION", - "public static final java.lang.String[] CONFIG_DEF_SCHEMA" - ] - }, - "com.yahoo.search.config.IndexInfoConfig$Builder": { - "superClass": "java.lang.Object", - "interfaces": [ - "com.yahoo.config.ConfigInstance$Builder" - ], - "attributes": [ - "public" - ], - "methods": [ - "public void <init>()", - "public void <init>(com.yahoo.search.config.IndexInfoConfig)", - "public com.yahoo.search.config.IndexInfoConfig$Builder indexinfo(com.yahoo.search.config.IndexInfoConfig$Indexinfo$Builder)", - "public com.yahoo.search.config.IndexInfoConfig$Builder indexinfo(java.util.function.Consumer)", - "public com.yahoo.search.config.IndexInfoConfig$Builder indexinfo(java.util.List)", - "public final boolean dispatchGetConfig(com.yahoo.config.ConfigInstance$Producer)", - "public final java.lang.String getDefMd5()", - "public final java.lang.String getDefName()", - "public final java.lang.String getDefNamespace()", - "public final boolean getApplyOnRestart()", - "public final void setApplyOnRestart(boolean)", - "public com.yahoo.search.config.IndexInfoConfig build()" - ], - "fields": [ - "public java.util.List indexinfo" - ] - }, - "com.yahoo.search.config.IndexInfoConfig$Indexinfo$Alias$Builder": { - "superClass": "java.lang.Object", - "interfaces": [ - "com.yahoo.config.ConfigBuilder" - ], - "attributes": [ - "public" - ], - "methods": [ - "public void <init>()", - "public void <init>(com.yahoo.search.config.IndexInfoConfig$Indexinfo$Alias)", - "public com.yahoo.search.config.IndexInfoConfig$Indexinfo$Alias$Builder alias(java.lang.String)", - "public com.yahoo.search.config.IndexInfoConfig$Indexinfo$Alias$Builder indexname(java.lang.String)", - "public com.yahoo.search.config.IndexInfoConfig$Indexinfo$Alias build()" - ], - "fields": [] - }, - "com.yahoo.search.config.IndexInfoConfig$Indexinfo$Alias": { - "superClass": "com.yahoo.config.InnerNode", - "interfaces": [], - "attributes": [ - "public", - "final" - ], - "methods": [ - "public void <init>(com.yahoo.search.config.IndexInfoConfig$Indexinfo$Alias$Builder)", - "public java.lang.String alias()", - "public java.lang.String indexname()" - ], - "fields": [] - }, - "com.yahoo.search.config.IndexInfoConfig$Indexinfo$Builder": { - "superClass": "java.lang.Object", - "interfaces": [ - "com.yahoo.config.ConfigBuilder" - ], - "attributes": [ - "public" - ], - "methods": [ - "public void <init>()", - "public void <init>(com.yahoo.search.config.IndexInfoConfig$Indexinfo)", - "public com.yahoo.search.config.IndexInfoConfig$Indexinfo$Builder name(java.lang.String)", - "public com.yahoo.search.config.IndexInfoConfig$Indexinfo$Builder command(com.yahoo.search.config.IndexInfoConfig$Indexinfo$Command$Builder)", - "public com.yahoo.search.config.IndexInfoConfig$Indexinfo$Builder command(java.util.function.Consumer)", - "public com.yahoo.search.config.IndexInfoConfig$Indexinfo$Builder command(java.util.List)", - "public com.yahoo.search.config.IndexInfoConfig$Indexinfo$Builder alias(com.yahoo.search.config.IndexInfoConfig$Indexinfo$Alias$Builder)", - "public com.yahoo.search.config.IndexInfoConfig$Indexinfo$Builder alias(java.util.function.Consumer)", - "public com.yahoo.search.config.IndexInfoConfig$Indexinfo$Builder alias(java.util.List)", - "public com.yahoo.search.config.IndexInfoConfig$Indexinfo build()" - ], - "fields": [ - "public java.util.List command", - "public java.util.List alias" - ] - }, - "com.yahoo.search.config.IndexInfoConfig$Indexinfo$Command$Builder": { - "superClass": "java.lang.Object", - "interfaces": [ - "com.yahoo.config.ConfigBuilder" - ], - "attributes": [ - "public" - ], - "methods": [ - "public void <init>()", - "public void <init>(com.yahoo.search.config.IndexInfoConfig$Indexinfo$Command)", - "public com.yahoo.search.config.IndexInfoConfig$Indexinfo$Command$Builder indexname(java.lang.String)", - "public com.yahoo.search.config.IndexInfoConfig$Indexinfo$Command$Builder command(java.lang.String)", - "public com.yahoo.search.config.IndexInfoConfig$Indexinfo$Command build()" - ], - "fields": [] - }, - "com.yahoo.search.config.IndexInfoConfig$Indexinfo$Command": { - "superClass": "com.yahoo.config.InnerNode", - "interfaces": [], - "attributes": [ - "public", - "final" - ], - "methods": [ - "public void <init>(com.yahoo.search.config.IndexInfoConfig$Indexinfo$Command$Builder)", - "public java.lang.String indexname()", - "public java.lang.String command()" - ], - "fields": [] - }, - "com.yahoo.search.config.IndexInfoConfig$Indexinfo": { - "superClass": "com.yahoo.config.InnerNode", - "interfaces": [], - "attributes": [ - "public", - "final" - ], - "methods": [ - "public void <init>(com.yahoo.search.config.IndexInfoConfig$Indexinfo$Builder)", - "public java.lang.String name()", - "public java.util.List command()", - "public com.yahoo.search.config.IndexInfoConfig$Indexinfo$Command command(int)", - "public java.util.List alias()", - "public com.yahoo.search.config.IndexInfoConfig$Indexinfo$Alias alias(int)" - ], - "fields": [] - }, - "com.yahoo.search.config.IndexInfoConfig$Producer": { - "superClass": "java.lang.Object", - "interfaces": [ - "com.yahoo.config.ConfigInstance$Producer" - ], - "attributes": [ - "public", - "interface", - "abstract" - ], - "methods": [ - "public abstract void getConfig(com.yahoo.search.config.IndexInfoConfig$Builder)" - ], - "fields": [] - }, - "com.yahoo.search.config.IndexInfoConfig": { - "superClass": "com.yahoo.config.ConfigInstance", - "interfaces": [], - "attributes": [ - "public", - "final" - ], - "methods": [ - "public static java.lang.String getDefMd5()", - "public static java.lang.String getDefName()", - "public static java.lang.String getDefNamespace()", - "public static java.lang.String getDefVersion()", - "public void <init>(com.yahoo.search.config.IndexInfoConfig$Builder)", - "public java.util.List indexinfo()", - "public com.yahoo.search.config.IndexInfoConfig$Indexinfo indexinfo(int)" - ], - "fields": [ - "public static final java.lang.String CONFIG_DEF_MD5", - "public static final java.lang.String CONFIG_DEF_NAME", - "public static final java.lang.String CONFIG_DEF_NAMESPACE", - "public static final java.lang.String CONFIG_DEF_VERSION", - "public static final java.lang.String[] CONFIG_DEF_SCHEMA" - ] - }, - "com.yahoo.search.config.QrStartConfig$Builder": { - "superClass": "java.lang.Object", - "interfaces": [ - "com.yahoo.config.ConfigInstance$Builder" - ], - "attributes": [ - "public" - ], - "methods": [ - "public void <init>()", - "public void <init>(com.yahoo.search.config.QrStartConfig)", - "public com.yahoo.search.config.QrStartConfig$Builder jvm(com.yahoo.search.config.QrStartConfig$Jvm$Builder)", - "public com.yahoo.search.config.QrStartConfig$Builder jvm(java.util.function.Consumer)", - "public com.yahoo.search.config.QrStartConfig$Builder qrs(com.yahoo.search.config.QrStartConfig$Qrs$Builder)", - "public com.yahoo.search.config.QrStartConfig$Builder qrs(java.util.function.Consumer)", - "public com.yahoo.search.config.QrStartConfig$Builder ulimitv(java.lang.String)", - "public com.yahoo.search.config.QrStartConfig$Builder jdisc(com.yahoo.search.config.QrStartConfig$Jdisc$Builder)", - "public com.yahoo.search.config.QrStartConfig$Builder jdisc(java.util.function.Consumer)", - "public final boolean dispatchGetConfig(com.yahoo.config.ConfigInstance$Producer)", - "public final java.lang.String getDefMd5()", - "public final java.lang.String getDefName()", - "public final java.lang.String getDefNamespace()", - "public final boolean getApplyOnRestart()", - "public final void setApplyOnRestart(boolean)", - "public com.yahoo.search.config.QrStartConfig build()" - ], - "fields": [ - "public com.yahoo.search.config.QrStartConfig$Jvm$Builder jvm", - "public com.yahoo.search.config.QrStartConfig$Qrs$Builder qrs", - "public com.yahoo.search.config.QrStartConfig$Jdisc$Builder jdisc" - ] - }, - "com.yahoo.search.config.QrStartConfig$Jdisc$Builder": { - "superClass": "java.lang.Object", - "interfaces": [ - "com.yahoo.config.ConfigBuilder" - ], - "attributes": [ - "public" - ], - "methods": [ - "public void <init>()", - "public void <init>(com.yahoo.search.config.QrStartConfig$Jdisc)", - "public com.yahoo.search.config.QrStartConfig$Jdisc$Builder classpath_extra(java.lang.String)", - "public com.yahoo.search.config.QrStartConfig$Jdisc$Builder export_packages(java.lang.String)", - "public com.yahoo.search.config.QrStartConfig$Jdisc build()" - ], - "fields": [] - }, - "com.yahoo.search.config.QrStartConfig$Jdisc": { - "superClass": "com.yahoo.config.InnerNode", - "interfaces": [], - "attributes": [ - "public", - "final" - ], - "methods": [ - "public void <init>(com.yahoo.search.config.QrStartConfig$Jdisc$Builder)", - "public java.lang.String classpath_extra()", - "public java.lang.String export_packages()" - ], - "fields": [] - }, - "com.yahoo.search.config.QrStartConfig$Jvm$Builder": { - "superClass": "java.lang.Object", - "interfaces": [ - "com.yahoo.config.ConfigBuilder" - ], - "attributes": [ - "public" - ], - "methods": [ - "public void <init>()", - "public void <init>(com.yahoo.search.config.QrStartConfig$Jvm)", - "public com.yahoo.search.config.QrStartConfig$Jvm$Builder server(boolean)", - "public com.yahoo.search.config.QrStartConfig$Jvm$Builder verbosegc(boolean)", - "public com.yahoo.search.config.QrStartConfig$Jvm$Builder gcopts(java.lang.String)", - "public com.yahoo.search.config.QrStartConfig$Jvm$Builder heapsize(int)", - "public com.yahoo.search.config.QrStartConfig$Jvm$Builder minHeapsize(int)", - "public com.yahoo.search.config.QrStartConfig$Jvm$Builder stacksize(int)", - "public com.yahoo.search.config.QrStartConfig$Jvm$Builder compressedClassSpaceSize(int)", - "public com.yahoo.search.config.QrStartConfig$Jvm$Builder baseMaxDirectMemorySize(int)", - "public com.yahoo.search.config.QrStartConfig$Jvm$Builder directMemorySizeCache(int)", - "public com.yahoo.search.config.QrStartConfig$Jvm$Builder heapSizeAsPercentageOfPhysicalMemory(int)", - "public com.yahoo.search.config.QrStartConfig$Jvm$Builder availableProcessors(int)", - "public com.yahoo.search.config.QrStartConfig$Jvm build()" - ], - "fields": [] - }, - "com.yahoo.search.config.QrStartConfig$Jvm": { - "superClass": "com.yahoo.config.InnerNode", - "interfaces": [], - "attributes": [ - "public", - "final" - ], - "methods": [ - "public void <init>(com.yahoo.search.config.QrStartConfig$Jvm$Builder)", - "public boolean server()", - "public boolean verbosegc()", - "public java.lang.String gcopts()", - "public int heapsize()", - "public int minHeapsize()", - "public int stacksize()", - "public int compressedClassSpaceSize()", - "public int baseMaxDirectMemorySize()", - "public int directMemorySizeCache()", - "public int heapSizeAsPercentageOfPhysicalMemory()", - "public int availableProcessors()" - ], - "fields": [] - }, - "com.yahoo.search.config.QrStartConfig$Producer": { - "superClass": "java.lang.Object", - "interfaces": [ - "com.yahoo.config.ConfigInstance$Producer" - ], - "attributes": [ - "public", - "interface", - "abstract" - ], - "methods": [ - "public abstract void getConfig(com.yahoo.search.config.QrStartConfig$Builder)" - ], - "fields": [] - }, - "com.yahoo.search.config.QrStartConfig$Qrs$Builder": { - "superClass": "java.lang.Object", - "interfaces": [ - "com.yahoo.config.ConfigBuilder" - ], - "attributes": [ - "public" - ], - "methods": [ - "public void <init>()", - "public void <init>(com.yahoo.search.config.QrStartConfig$Qrs)", - "public com.yahoo.search.config.QrStartConfig$Qrs$Builder env(java.lang.String)", - "public com.yahoo.search.config.QrStartConfig$Qrs build()" - ], - "fields": [] - }, - "com.yahoo.search.config.QrStartConfig$Qrs": { - "superClass": "com.yahoo.config.InnerNode", - "interfaces": [], - "attributes": [ - "public", - "final" - ], - "methods": [ - "public void <init>(com.yahoo.search.config.QrStartConfig$Qrs$Builder)", - "public java.lang.String env()" - ], - "fields": [] - }, - "com.yahoo.search.config.QrStartConfig": { - "superClass": "com.yahoo.config.ConfigInstance", - "interfaces": [], - "attributes": [ - "public", - "final" - ], - "methods": [ - "public static java.lang.String getDefMd5()", - "public static java.lang.String getDefName()", - "public static java.lang.String getDefNamespace()", - "public static java.lang.String getDefVersion()", - "public void <init>(com.yahoo.search.config.QrStartConfig$Builder)", - "public com.yahoo.search.config.QrStartConfig$Jvm jvm()", - "public com.yahoo.search.config.QrStartConfig$Qrs qrs()", - "public java.lang.String ulimitv()", - "public com.yahoo.search.config.QrStartConfig$Jdisc jdisc()" - ], - "fields": [ - "public static final java.lang.String CONFIG_DEF_MD5", - "public static final java.lang.String CONFIG_DEF_NAME", - "public static final java.lang.String CONFIG_DEF_NAMESPACE", - "public static final java.lang.String CONFIG_DEF_VERSION", - "public static final java.lang.String[] CONFIG_DEF_SCHEMA" - ] - }, - "com.yahoo.search.config.RankProfile$Builder": { - "superClass": "java.lang.Object", - "interfaces": [], - "attributes": [ - "public" - ], - "methods": [ - "public void <init>(java.lang.String)", - "public com.yahoo.search.config.RankProfile$Builder setHasSummaryFeatures(boolean)", - "public com.yahoo.search.config.RankProfile$Builder setHasRankFeatures(boolean)", - "public com.yahoo.search.config.RankProfile$Builder addInput(java.lang.String, com.yahoo.tensor.TensorType)", - "public com.yahoo.search.config.RankProfile build()" - ], - "fields": [] - }, - "com.yahoo.search.config.RankProfile": { - "superClass": "java.lang.Object", - "interfaces": [], - "attributes": [ - "public" - ], - "methods": [ - "public java.lang.String name()", - "public boolean hasSummaryFeatures()", - "public boolean hasRankFeatures()", - "public java.util.Map inputs()", - "public boolean equals(java.lang.Object)", - "public int hashCode()", - "public java.lang.String toString()" - ], - "fields": [] - }, - "com.yahoo.search.config.RateLimitingConfig$Builder": { - "superClass": "java.lang.Object", - "interfaces": [ - "com.yahoo.config.ConfigInstance$Builder" - ], - "attributes": [ - "public" - ], - "methods": [ - "public void <init>()", - "public void <init>(com.yahoo.search.config.RateLimitingConfig)", - "public com.yahoo.search.config.RateLimitingConfig$Builder capacityIncrement(double)", - "public com.yahoo.search.config.RateLimitingConfig$Builder maxAvailableCapacity(double)", - "public com.yahoo.search.config.RateLimitingConfig$Builder recheckForCapacityProbability(double)", - "public final boolean dispatchGetConfig(com.yahoo.config.ConfigInstance$Producer)", - "public final java.lang.String getDefMd5()", - "public final java.lang.String getDefName()", - "public final java.lang.String getDefNamespace()", - "public final boolean getApplyOnRestart()", - "public final void setApplyOnRestart(boolean)", - "public com.yahoo.search.config.RateLimitingConfig build()" - ], - "fields": [] - }, - "com.yahoo.search.config.RateLimitingConfig$Producer": { - "superClass": "java.lang.Object", - "interfaces": [ - "com.yahoo.config.ConfigInstance$Producer" - ], - "attributes": [ - "public", - "interface", - "abstract" - ], - "methods": [ - "public abstract void getConfig(com.yahoo.search.config.RateLimitingConfig$Builder)" - ], - "fields": [] - }, - "com.yahoo.search.config.RateLimitingConfig": { - "superClass": "com.yahoo.config.ConfigInstance", - "interfaces": [], - "attributes": [ - "public", - "final" - ], - "methods": [ - "public static java.lang.String getDefMd5()", - "public static java.lang.String getDefName()", - "public static java.lang.String getDefNamespace()", - "public static java.lang.String getDefVersion()", - "public void <init>(com.yahoo.search.config.RateLimitingConfig$Builder)", - "public double capacityIncrement()", - "public double maxAvailableCapacity()", - "public double recheckForCapacityProbability()" - ], - "fields": [ - "public static final java.lang.String CONFIG_DEF_MD5", - "public static final java.lang.String CONFIG_DEF_NAME", - "public static final java.lang.String CONFIG_DEF_NAMESPACE", - "public static final java.lang.String CONFIG_DEF_VERSION", - "public static final java.lang.String[] CONFIG_DEF_SCHEMA" - ] - }, - "com.yahoo.search.config.Schema$Builder": { - "superClass": "java.lang.Object", - "interfaces": [], - "attributes": [ - "public" - ], - "methods": [ - "public void <init>(java.lang.String)", - "public com.yahoo.search.config.Schema$Builder add(com.yahoo.search.config.RankProfile)", - "public com.yahoo.search.config.Schema build()" - ], - "fields": [] - }, - "com.yahoo.search.config.Schema": { - "superClass": "java.lang.Object", - "interfaces": [], - "attributes": [ - "public" - ], - "methods": [ - "public java.lang.String name()", - "public java.util.Map rankProfiles()", - "public boolean equals(java.lang.Object)", - "public int hashCode()", - "public java.lang.String toString()" - ], - "fields": [] - }, - "com.yahoo.search.config.SchemaInfo$Session": { - "superClass": "java.lang.Object", - "interfaces": [], - "attributes": [ - "public" - ], - "methods": [ - "public com.yahoo.tensor.TensorType rankProfileInput(java.lang.String, java.lang.String)" - ], - "fields": [] - }, - "com.yahoo.search.config.SchemaInfo": { - "superClass": "java.lang.Object", - "interfaces": [], - "attributes": [ - "public" - ], - "methods": [ - "public void <init>(com.yahoo.search.config.IndexInfoConfig, com.yahoo.prelude.fastsearch.DocumentdbInfoConfig, com.yahoo.container.QrSearchersConfig)", - "public void <init>(java.util.List, java.util.Map)", - "public com.yahoo.search.config.SchemaInfo$Session newSession(com.yahoo.search.Query)", - "public static com.yahoo.search.config.SchemaInfo empty()", - "public boolean equals(java.lang.Object)", - "public int hashCode()" - ], - "fields": [] - }, "com.yahoo.search.federation.selection.FederationTarget": { "superClass": "java.lang.Object", "interfaces": [], @@ -7293,7 +6719,7 @@ "public" ], "methods": [ - "public void <init>(com.yahoo.search.config.SchemaInfo, com.yahoo.search.Query, java.util.Map)", + "public void <init>(com.yahoo.search.schema.SchemaInfo, com.yahoo.search.Query, java.util.Map)", "public void set(com.yahoo.processing.request.CompoundName, java.lang.Object, java.util.Map)", "public void requireSettable(com.yahoo.processing.request.CompoundName, java.lang.Object, java.util.Map)" ], @@ -8548,6 +7974,93 @@ ], "fields": [] }, + "com.yahoo.search.schema.RankProfile$Builder": { + "superClass": "java.lang.Object", + "interfaces": [], + "attributes": [ + "public" + ], + "methods": [ + "public void <init>(java.lang.String)", + "public com.yahoo.search.schema.RankProfile$Builder setHasSummaryFeatures(boolean)", + "public com.yahoo.search.schema.RankProfile$Builder setHasRankFeatures(boolean)", + "public com.yahoo.search.schema.RankProfile$Builder addInput(java.lang.String, com.yahoo.tensor.TensorType)", + "public com.yahoo.search.schema.RankProfile build()" + ], + "fields": [] + }, + "com.yahoo.search.schema.RankProfile": { + "superClass": "java.lang.Object", + "interfaces": [], + "attributes": [ + "public" + ], + "methods": [ + "public java.lang.String name()", + "public boolean hasSummaryFeatures()", + "public boolean hasRankFeatures()", + "public java.util.Map inputs()", + "public boolean equals(java.lang.Object)", + "public int hashCode()", + "public java.lang.String toString()" + ], + "fields": [] + }, + "com.yahoo.search.schema.Schema$Builder": { + "superClass": "java.lang.Object", + "interfaces": [], + "attributes": [ + "public" + ], + "methods": [ + "public void <init>(java.lang.String)", + "public com.yahoo.search.schema.Schema$Builder add(com.yahoo.search.schema.RankProfile)", + "public com.yahoo.search.schema.Schema build()" + ], + "fields": [] + }, + "com.yahoo.search.schema.Schema": { + "superClass": "java.lang.Object", + "interfaces": [], + "attributes": [ + "public" + ], + "methods": [ + "public java.lang.String name()", + "public java.util.Map rankProfiles()", + "public boolean equals(java.lang.Object)", + "public int hashCode()", + "public java.lang.String toString()" + ], + "fields": [] + }, + "com.yahoo.search.schema.SchemaInfo$Session": { + "superClass": "java.lang.Object", + "interfaces": [], + "attributes": [ + "public" + ], + "methods": [ + "public com.yahoo.tensor.TensorType rankProfileInput(java.lang.String, java.lang.String)" + ], + "fields": [] + }, + "com.yahoo.search.schema.SchemaInfo": { + "superClass": "java.lang.Object", + "interfaces": [], + "attributes": [ + "public" + ], + "methods": [ + "public void <init>(com.yahoo.search.config.IndexInfoConfig, com.yahoo.search.config.SchemaInfoConfig, com.yahoo.container.QrSearchersConfig)", + "public void <init>(java.util.List, java.util.Map)", + "public com.yahoo.search.schema.SchemaInfo$Session newSession(com.yahoo.search.Query)", + "public static com.yahoo.search.schema.SchemaInfo empty()", + "public boolean equals(java.lang.Object)", + "public int hashCode()" + ], + "fields": [] + }, "com.yahoo.search.searchchain.AsyncExecution": { "superClass": "java.lang.Object", "interfaces": [], @@ -8573,7 +8086,7 @@ "final" ], "methods": [ - "public void <init>(com.yahoo.search.searchchain.SearchChainRegistry, com.yahoo.prelude.IndexFacts, com.yahoo.search.config.SchemaInfo, com.yahoo.language.process.SpecialTokenRegistry, com.yahoo.search.rendering.RendererRegistry, com.yahoo.language.Linguistics, java.util.concurrent.Executor)", + "public void <init>(com.yahoo.search.searchchain.SearchChainRegistry, com.yahoo.prelude.IndexFacts, com.yahoo.search.schema.SchemaInfo, com.yahoo.language.process.SpecialTokenRegistry, com.yahoo.search.rendering.RendererRegistry, com.yahoo.language.Linguistics, java.util.concurrent.Executor)", "public void <init>(com.yahoo.search.searchchain.SearchChainRegistry, com.yahoo.prelude.IndexFacts, com.yahoo.language.process.SpecialTokenRegistry, com.yahoo.search.rendering.RendererRegistry, com.yahoo.language.Linguistics, java.util.concurrent.Executor)", "public void <init>(com.yahoo.search.searchchain.SearchChainRegistry, com.yahoo.prelude.IndexFacts, com.yahoo.language.process.SpecialTokenRegistry, com.yahoo.search.rendering.RendererRegistry, com.yahoo.language.Linguistics)", "public static com.yahoo.search.searchchain.Execution$Context createContextStub()", @@ -8582,7 +8095,7 @@ "public static com.yahoo.search.searchchain.Execution$Context createContextStub(com.yahoo.search.searchchain.SearchChainRegistry, com.yahoo.prelude.IndexFacts)", "public static com.yahoo.search.searchchain.Execution$Context createContextStub(com.yahoo.prelude.IndexFacts, com.yahoo.language.Linguistics)", "public static com.yahoo.search.searchchain.Execution$Context createContextStub(com.yahoo.search.searchchain.SearchChainRegistry, com.yahoo.prelude.IndexFacts, com.yahoo.language.Linguistics)", - "public static com.yahoo.search.searchchain.Execution$Context createContextStub(com.yahoo.search.searchchain.SearchChainRegistry, com.yahoo.prelude.IndexFacts, com.yahoo.search.config.SchemaInfo, com.yahoo.language.Linguistics)", + "public static com.yahoo.search.searchchain.Execution$Context createContextStub(com.yahoo.search.searchchain.SearchChainRegistry, com.yahoo.prelude.IndexFacts, com.yahoo.search.schema.SchemaInfo, com.yahoo.language.Linguistics)", "public void populateFrom(com.yahoo.search.searchchain.Execution$Context)", "public boolean equals(com.yahoo.search.searchchain.Execution$Context)", "public int hashCode()", @@ -8590,7 +8103,7 @@ "public com.yahoo.search.searchchain.Execution$Context shallowCopy()", "public com.yahoo.prelude.IndexFacts getIndexFacts()", "public void setIndexFacts(com.yahoo.prelude.IndexFacts)", - "public com.yahoo.search.config.SchemaInfo schemaInfo()", + "public com.yahoo.search.schema.SchemaInfo schemaInfo()", "public com.yahoo.search.searchchain.SearchChainRegistry searchChainRegistry()", "public com.yahoo.search.rendering.RendererRegistry rendererRegistry()", "public com.yahoo.language.process.SpecialTokenRegistry getTokenRegistry()", @@ -8642,14 +8155,14 @@ "public" ], "methods": [ - "public void <init>(com.yahoo.container.core.ChainsConfig, com.yahoo.search.config.IndexInfoConfig, com.yahoo.prelude.fastsearch.DocumentdbInfoConfig, com.yahoo.container.QrSearchersConfig, com.yahoo.component.provider.ComponentRegistry, com.yahoo.vespa.configdefinition.SpecialtokensConfig, com.yahoo.language.Linguistics, com.yahoo.component.provider.ComponentRegistry, java.util.concurrent.Executor)", + "public void <init>(com.yahoo.container.core.ChainsConfig, com.yahoo.search.config.IndexInfoConfig, com.yahoo.search.config.SchemaInfoConfig, com.yahoo.container.QrSearchersConfig, com.yahoo.component.provider.ComponentRegistry, com.yahoo.vespa.configdefinition.SpecialtokensConfig, com.yahoo.language.Linguistics, com.yahoo.component.provider.ComponentRegistry, java.util.concurrent.Executor)", "public void <init>(com.yahoo.container.core.ChainsConfig, com.yahoo.search.config.IndexInfoConfig, com.yahoo.container.QrSearchersConfig, com.yahoo.component.provider.ComponentRegistry, com.yahoo.vespa.configdefinition.SpecialtokensConfig, com.yahoo.language.Linguistics, com.yahoo.component.provider.ComponentRegistry, java.util.concurrent.Executor)", "public void <init>(com.yahoo.container.core.ChainsConfig, com.yahoo.search.config.IndexInfoConfig, com.yahoo.container.QrSearchersConfig, com.yahoo.component.provider.ComponentRegistry, com.yahoo.vespa.configdefinition.SpecialtokensConfig, com.yahoo.language.Linguistics, com.yahoo.component.provider.ComponentRegistry)", "public com.yahoo.search.searchchain.Execution newExecution(com.yahoo.component.chain.Chain)", "public com.yahoo.search.searchchain.Execution newExecution(java.lang.String)", "public com.yahoo.search.searchchain.SearchChainRegistry searchChainRegistry()", "public com.yahoo.search.rendering.RendererRegistry rendererRegistry()", - "public com.yahoo.search.config.SchemaInfo schemaInfo()", + "public com.yahoo.search.schema.SchemaInfo schemaInfo()", "public void deconstruct()", "public static com.yahoo.search.searchchain.ExecutionFactory empty()" ], diff --git a/container-search/src/main/java/com/yahoo/prelude/fastsearch/DocumentDatabase.java b/container-search/src/main/java/com/yahoo/prelude/fastsearch/DocumentDatabase.java index f35559ad2f4..67038e0e771 100644 --- a/container-search/src/main/java/com/yahoo/prelude/fastsearch/DocumentDatabase.java +++ b/container-search/src/main/java/com/yahoo/prelude/fastsearch/DocumentDatabase.java @@ -1,7 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.prelude.fastsearch; -import com.yahoo.search.config.RankProfile; +import com.yahoo.search.schema.RankProfile; import com.yahoo.tensor.TensorType; import java.util.ArrayList; diff --git a/container-search/src/main/java/com/yahoo/prelude/fastsearch/VespaBackEndSearcher.java b/container-search/src/main/java/com/yahoo/prelude/fastsearch/VespaBackEndSearcher.java index a6da823d990..3847e80d3c7 100644 --- a/container-search/src/main/java/com/yahoo/prelude/fastsearch/VespaBackEndSearcher.java +++ b/container-search/src/main/java/com/yahoo/prelude/fastsearch/VespaBackEndSearcher.java @@ -14,7 +14,7 @@ import com.yahoo.protect.Validator; import com.yahoo.search.Query; import com.yahoo.search.Result; import com.yahoo.search.cluster.PingableSearcher; -import com.yahoo.search.config.RankProfile; +import com.yahoo.search.schema.RankProfile; import com.yahoo.search.grouping.vespa.GroupingExecutor; import com.yahoo.search.result.ErrorMessage; import com.yahoo.search.result.Hit; diff --git a/container-search/src/main/java/com/yahoo/search/Query.java b/container-search/src/main/java/com/yahoo/search/Query.java index 354881c763b..5e46815a152 100644 --- a/container-search/src/main/java/com/yahoo/search/Query.java +++ b/container-search/src/main/java/com/yahoo/search/Query.java @@ -13,7 +13,7 @@ import com.yahoo.prelude.fastsearch.DocumentDatabase; import com.yahoo.prelude.query.Highlight; import com.yahoo.prelude.query.textualrepresentation.TextualQueryRepresentation; import com.yahoo.processing.request.CompoundName; -import com.yahoo.search.config.SchemaInfo; +import com.yahoo.search.schema.SchemaInfo; import com.yahoo.search.dispatch.Dispatcher; import com.yahoo.search.dispatch.rpc.ProtobufSerialization; import com.yahoo.search.federation.FederationSearcher; @@ -60,7 +60,6 @@ import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Objects; -import java.util.Optional; import java.util.Set; import java.util.concurrent.TimeUnit; import java.util.logging.Level; diff --git a/container-search/src/main/java/com/yahoo/search/config/package-info.java b/container-search/src/main/java/com/yahoo/search/config/package-info.java index dd9c7bfcf04..bdfc1ce9790 100644 --- a/container-search/src/main/java/com/yahoo/search/config/package-info.java +++ b/container-search/src/main/java/com/yahoo/search/config/package-info.java @@ -1,11 +1,6 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. @ExportPackage -@PublicApi package com.yahoo.search.config; -import com.yahoo.api.annotations.PublicApi; import com.yahoo.osgi.annotation.ExportPackage; -/** - * Information about the current configuration this is running as a part of. - */
\ No newline at end of file diff --git a/container-search/src/main/java/com/yahoo/search/query/Model.java b/container-search/src/main/java/com/yahoo/search/query/Model.java index 82cda9e8a1b..1b12f3f3bb8 100644 --- a/container-search/src/main/java/com/yahoo/search/query/Model.java +++ b/container-search/src/main/java/com/yahoo/search/query/Model.java @@ -10,7 +10,7 @@ import com.yahoo.prelude.query.TaggableItem; import com.yahoo.processing.IllegalInputException; import com.yahoo.processing.request.CompoundName; import com.yahoo.search.Query; -import com.yahoo.search.config.SchemaInfo; +import com.yahoo.search.schema.SchemaInfo; import com.yahoo.search.query.parser.Parsable; import com.yahoo.search.query.parser.Parser; import com.yahoo.search.query.parser.ParserEnvironment; diff --git a/container-search/src/main/java/com/yahoo/search/query/profile/types/TensorFieldType.java b/container-search/src/main/java/com/yahoo/search/query/profile/types/TensorFieldType.java index e0dea744075..ccb9b2d0676 100644 --- a/container-search/src/main/java/com/yahoo/search/query/profile/types/TensorFieldType.java +++ b/container-search/src/main/java/com/yahoo/search/query/profile/types/TensorFieldType.java @@ -2,7 +2,7 @@ package com.yahoo.search.query.profile.types; import com.yahoo.processing.request.Properties; -import com.yahoo.search.config.internal.TensorConverter; +import com.yahoo.search.schema.internal.TensorConverter; import com.yahoo.search.query.profile.QueryProfileRegistry; import com.yahoo.search.query.profile.SubstituteString; import com.yahoo.tensor.Tensor; diff --git a/container-search/src/main/java/com/yahoo/search/query/properties/RankProfileInputProperties.java b/container-search/src/main/java/com/yahoo/search/query/properties/RankProfileInputProperties.java index 7f4cee07e8c..6c65a5e898a 100644 --- a/container-search/src/main/java/com/yahoo/search/query/properties/RankProfileInputProperties.java +++ b/container-search/src/main/java/com/yahoo/search/query/properties/RankProfileInputProperties.java @@ -5,10 +5,9 @@ import com.yahoo.api.annotations.Beta; import com.yahoo.language.process.Embedder; import com.yahoo.processing.request.CompoundName; import com.yahoo.search.Query; -import com.yahoo.search.config.SchemaInfo; -import com.yahoo.search.config.internal.TensorConverter; +import com.yahoo.search.schema.SchemaInfo; +import com.yahoo.search.schema.internal.TensorConverter; import com.yahoo.search.query.Properties; -import com.yahoo.search.query.Ranking; import com.yahoo.search.query.ranking.RankFeatures; import com.yahoo.tensor.Tensor; import com.yahoo.tensor.TensorType; diff --git a/container-search/src/main/java/com/yahoo/search/config/RankProfile.java b/container-search/src/main/java/com/yahoo/search/schema/RankProfile.java index 944a23f2964..8267e5c937b 100644 --- a/container-search/src/main/java/com/yahoo/search/config/RankProfile.java +++ b/container-search/src/main/java/com/yahoo/search/schema/RankProfile.java @@ -1,5 +1,5 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.search.config; +package com.yahoo.search.schema; import com.yahoo.tensor.TensorType; diff --git a/container-search/src/main/java/com/yahoo/search/config/Schema.java b/container-search/src/main/java/com/yahoo/search/schema/Schema.java index 57712c731f4..b66e6ce957a 100644 --- a/container-search/src/main/java/com/yahoo/search/config/Schema.java +++ b/container-search/src/main/java/com/yahoo/search/schema/Schema.java @@ -1,5 +1,5 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.search.config; +package com.yahoo.search.schema; import com.yahoo.api.annotations.Beta; diff --git a/container-search/src/main/java/com/yahoo/search/config/SchemaInfo.java b/container-search/src/main/java/com/yahoo/search/schema/SchemaInfo.java index 746f1c340f2..2d1d391640f 100644 --- a/container-search/src/main/java/com/yahoo/search/config/SchemaInfo.java +++ b/container-search/src/main/java/com/yahoo/search/schema/SchemaInfo.java @@ -1,11 +1,11 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.search.config; +package com.yahoo.search.schema; import com.yahoo.api.annotations.Beta; -import com.yahoo.component.annotation.Inject; import com.yahoo.container.QrSearchersConfig; -import com.yahoo.prelude.fastsearch.DocumentdbInfoConfig; import com.yahoo.search.Query; +import com.yahoo.search.config.IndexInfoConfig; +import com.yahoo.search.config.SchemaInfoConfig; import com.yahoo.tensor.TensorType; import java.util.HashSet; @@ -44,11 +44,10 @@ public class SchemaInfo { /** The schemas contained in each content cluster indexed by cluster name */ private final Map<String, List<String>> clusters; - @Inject public SchemaInfo(IndexInfoConfig indexInfo, // will be used in the future - DocumentdbInfoConfig documentdbInfoConfig, + SchemaInfoConfig schemaInfoConfig, QrSearchersConfig qrSearchersConfig) { - this(SchemaInfoConfigurer.toSchemas(documentdbInfoConfig), SchemaInfoConfigurer.toClusters(qrSearchersConfig)); + this(SchemaInfoConfigurer.toSchemas(schemaInfoConfig), SchemaInfoConfigurer.toClusters(qrSearchersConfig)); } public SchemaInfo(List<Schema> schemas, Map<String, List<String>> clusters) { diff --git a/container-search/src/main/java/com/yahoo/search/config/SchemaInfoConfigurer.java b/container-search/src/main/java/com/yahoo/search/schema/SchemaInfoConfigurer.java index ae06babda66..84ed9ae8e3d 100644 --- a/container-search/src/main/java/com/yahoo/search/config/SchemaInfoConfigurer.java +++ b/container-search/src/main/java/com/yahoo/search/schema/SchemaInfoConfigurer.java @@ -1,12 +1,11 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.search.config; +package com.yahoo.search.schema; import com.yahoo.container.QrSearchersConfig; -import com.yahoo.prelude.fastsearch.DocumentdbInfoConfig; +import com.yahoo.search.config.SchemaInfoConfig; import com.yahoo.tensor.TensorType; import java.util.ArrayList; -import java.util.Collection; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -19,13 +18,13 @@ import java.util.stream.Collectors; */ class SchemaInfoConfigurer { - static List<Schema> toSchemas(DocumentdbInfoConfig documentdbInfoConfig) { - return documentdbInfoConfig.documentdb().stream().map(config -> toSchema(config)).collect(Collectors.toList()); + static List<Schema> toSchemas(SchemaInfoConfig schemaInfoConfig) { + return schemaInfoConfig.schema().stream().map(config -> toSchema(config)).collect(Collectors.toList()); } - static Schema toSchema(DocumentdbInfoConfig.Documentdb documentDbConfig) { - Schema.Builder builder = new Schema.Builder(documentDbConfig.name()); - for (var profileConfig : documentDbConfig.rankprofile()) { + static Schema toSchema(SchemaInfoConfig.Schema schemaInfoConfig) { + Schema.Builder builder = new Schema.Builder(schemaInfoConfig.name()); + for (var profileConfig : schemaInfoConfig.rankprofile()) { RankProfile.Builder profileBuilder = new RankProfile.Builder(profileConfig.name()); profileBuilder.setHasSummaryFeatures(profileConfig.hasSummaryFeatures()); profileBuilder.setHasRankFeatures(profileConfig.hasRankFeatures()); diff --git a/container-search/src/main/java/com/yahoo/search/config/internal/TensorConverter.java b/container-search/src/main/java/com/yahoo/search/schema/internal/TensorConverter.java index fbe2ffb8984..2370513dba2 100644 --- a/container-search/src/main/java/com/yahoo/search/config/internal/TensorConverter.java +++ b/container-search/src/main/java/com/yahoo/search/schema/internal/TensorConverter.java @@ -1,5 +1,5 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.search.config.internal; +package com.yahoo.search.schema.internal; import com.yahoo.language.Language; import com.yahoo.language.process.Embedder; diff --git a/container-search/src/main/java/com/yahoo/search/schema/package-info.java b/container-search/src/main/java/com/yahoo/search/schema/package-info.java new file mode 100644 index 00000000000..f9c86afc3e1 --- /dev/null +++ b/container-search/src/main/java/com/yahoo/search/schema/package-info.java @@ -0,0 +1,11 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +@ExportPackage +@PublicApi +package com.yahoo.search.schema; + +import com.yahoo.api.annotations.PublicApi; +import com.yahoo.osgi.annotation.ExportPackage; + +/** + * Information about the current configuration this is running as a part of. + */
\ No newline at end of file diff --git a/container-search/src/main/java/com/yahoo/search/searchchain/Execution.java b/container-search/src/main/java/com/yahoo/search/searchchain/Execution.java index baf9f35c72b..aba72cb3404 100644 --- a/container-search/src/main/java/com/yahoo/search/searchchain/Execution.java +++ b/container-search/src/main/java/com/yahoo/search/searchchain/Execution.java @@ -16,8 +16,7 @@ import com.yahoo.search.Query; import com.yahoo.search.Result; import com.yahoo.search.Searcher; import com.yahoo.search.cluster.PingableSearcher; -import com.yahoo.search.config.SchemaInfo; -import com.yahoo.search.rendering.Renderer; +import com.yahoo.search.schema.SchemaInfo; import com.yahoo.search.rendering.RendererRegistry; import com.yahoo.search.statistics.TimeTracker; import java.util.Objects; @@ -140,7 +139,7 @@ public class Execution extends com.yahoo.processing.execution.Execution { public Context(SearchChainRegistry searchChainRegistry, IndexFacts indexFacts, SpecialTokenRegistry tokenRegistry, RendererRegistry rendererRegistry, Linguistics linguistics, Executor executor) { - this(searchChainRegistry, indexFacts, SchemaInfo.empty(), tokenRegistry, rendererRegistry, linguistics, Runnable::run); + this(searchChainRegistry, indexFacts, SchemaInfo.empty(), tokenRegistry, rendererRegistry, linguistics, executor); } /** @deprecated pass an executor */ diff --git a/container-search/src/main/java/com/yahoo/search/searchchain/ExecutionFactory.java b/container-search/src/main/java/com/yahoo/search/searchchain/ExecutionFactory.java index 06814a4c436..54874dbee3e 100644 --- a/container-search/src/main/java/com/yahoo/search/searchchain/ExecutionFactory.java +++ b/container-search/src/main/java/com/yahoo/search/searchchain/ExecutionFactory.java @@ -11,16 +11,16 @@ import com.yahoo.component.provider.ComponentRegistry; import com.yahoo.concurrent.ThreadFactoryFactory; import com.yahoo.container.QrSearchersConfig; import com.yahoo.container.core.ChainsConfig; +import com.yahoo.search.config.SchemaInfoConfig; import com.yahoo.language.Linguistics; import com.yahoo.language.simple.SimpleLinguistics; import com.yahoo.prelude.IndexFacts; import com.yahoo.prelude.IndexModel; import com.yahoo.language.process.SpecialTokenRegistry; -import com.yahoo.prelude.fastsearch.DocumentdbInfoConfig; import com.yahoo.processing.rendering.Renderer; import com.yahoo.search.Searcher; import com.yahoo.search.config.IndexInfoConfig; -import com.yahoo.search.config.SchemaInfo; +import com.yahoo.search.schema.SchemaInfo; import com.yahoo.search.rendering.RendererRegistry; import com.yahoo.vespa.configdefinition.SpecialtokensConfig; @@ -51,7 +51,7 @@ public class ExecutionFactory extends AbstractComponent { @Inject public ExecutionFactory(ChainsConfig chainsConfig, IndexInfoConfig indexInfo, - DocumentdbInfoConfig documentdbInfo, + SchemaInfoConfig schemaInfo, QrSearchersConfig clusters, ComponentRegistry<Searcher> searchers, SpecialtokensConfig specialTokens, @@ -60,7 +60,7 @@ public class ExecutionFactory extends AbstractComponent { Executor executor) { this.searchChainRegistry = createSearchChainRegistry(searchers, chainsConfig); this.indexFacts = new IndexFacts(new IndexModel(indexInfo, clusters)).freeze(); - this.schemaInfo = new SchemaInfo(indexInfo, documentdbInfo, clusters); + this.schemaInfo = new SchemaInfo(indexInfo, schemaInfo, clusters); this.specialTokens = new SpecialTokenRegistry(specialTokens); this.linguistics = linguistics; this.renderingExecutor = createRenderingExecutor(); @@ -78,7 +78,7 @@ public class ExecutionFactory extends AbstractComponent { Linguistics linguistics, ComponentRegistry<Renderer> renderers, Executor executor) { - this(chainsConfig, indexInfo, new DocumentdbInfoConfig.Builder().build(), clusters, searchers, specialTokens, linguistics, renderers, executor); + this(chainsConfig, indexInfo, new SchemaInfoConfig.Builder().build(), clusters, searchers, specialTokens, linguistics, renderers, executor); } /** @deprecated pass the container threadpool */ @@ -90,7 +90,7 @@ public class ExecutionFactory extends AbstractComponent { SpecialtokensConfig specialTokens, Linguistics linguistics, ComponentRegistry<Renderer> renderers) { - this(chainsConfig, indexInfo, new DocumentdbInfoConfig.Builder().build(), clusters, searchers, specialTokens, linguistics, renderers, null); + this(chainsConfig, indexInfo, new SchemaInfoConfig.Builder().build(), clusters, searchers, specialTokens, linguistics, renderers, null); } private SearchChainRegistry createSearchChainRegistry(ComponentRegistry<Searcher> searchers, @@ -144,7 +144,7 @@ public class ExecutionFactory extends AbstractComponent { public static ExecutionFactory empty() { return new ExecutionFactory(new ChainsConfig.Builder().build(), new IndexInfoConfig.Builder().build(), - new DocumentdbInfoConfig.Builder().build(), + new SchemaInfoConfig.Builder().build(), new QrSearchersConfig.Builder().build(), new ComponentRegistry<>(), new SpecialtokensConfig.Builder().build(), diff --git a/container-search/src/main/resources/configdefinitions/container.search.schema-info.def b/container-search/src/main/resources/configdefinitions/container.search.schema-info.def new file mode 100644 index 00000000000..e5b14db9b4e --- /dev/null +++ b/container-search/src/main/resources/configdefinitions/container.search.schema-info.def @@ -0,0 +1,25 @@ +# Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +namespace=search.config + +## The name of this schema +schema[].name string + +## The id of the summary class +# schema[].summaryclass[].id int +## The name of the summary class +# schema[].summaryclass[].name string +## The name of a field in the summary class +# schema[].summaryclass[].fields[].name string +## The type of a field in the summary class +# schema[].summaryclass[].fields[].type string +## Whether this field is a dynamic snippet +# schema[].summaryclass[].fields[].dynamic bool default=false + +## Information about rank profiles +schema[].rankprofile[].name string +schema[].rankprofile[].hasSummaryFeatures bool default=true +schema[].rankprofile[].hasRankFeatures bool default=true +# The name of an input (query rank feature) accepted by this profile +schema[].rankprofile[].input[].name string +# The tensor type of an input (query rank feature) accepted by this profile +schema[].rankprofile[].input[].type string diff --git a/container-search/src/test/java/com/yahoo/search/handler/test/config/schema-info.cfg b/container-search/src/test/java/com/yahoo/search/handler/test/config/schema-info.cfg new file mode 100644 index 00000000000..e69de29bb2d --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/handler/test/config/schema-info.cfg diff --git a/container-search/src/test/java/com/yahoo/search/query/RankProfileInputTest.java b/container-search/src/test/java/com/yahoo/search/query/RankProfileInputTest.java index 1b10e4cd0ba..f34ad783277 100644 --- a/container-search/src/test/java/com/yahoo/search/query/RankProfileInputTest.java +++ b/container-search/src/test/java/com/yahoo/search/query/RankProfileInputTest.java @@ -5,9 +5,9 @@ import com.yahoo.container.jdisc.HttpRequest; import com.yahoo.language.Language; import com.yahoo.language.process.Embedder; import com.yahoo.search.Query; -import com.yahoo.search.config.RankProfile; -import com.yahoo.search.config.Schema; -import com.yahoo.search.config.SchemaInfo; +import com.yahoo.search.schema.RankProfile; +import com.yahoo.search.schema.Schema; +import com.yahoo.search.schema.SchemaInfo; import com.yahoo.search.query.profile.QueryProfile; import com.yahoo.search.query.profile.QueryProfileRegistry; import com.yahoo.search.query.profile.compiled.CompiledQueryProfile; diff --git a/container-search/src/test/java/com/yahoo/search/query/profile/config/test/typed/schema-info.cfg b/container-search/src/test/java/com/yahoo/search/query/profile/config/test/typed/schema-info.cfg new file mode 100644 index 00000000000..e69de29bb2d --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/query/profile/config/test/typed/schema-info.cfg diff --git a/container-search/src/test/java/com/yahoo/search/query/profile/config/test/untyped/schema-info.cfg b/container-search/src/test/java/com/yahoo/search/query/profile/config/test/untyped/schema-info.cfg new file mode 100644 index 00000000000..e69de29bb2d --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/query/profile/config/test/untyped/schema-info.cfg diff --git a/container-search/src/test/java/com/yahoo/search/config/SchemaInfoTest.java b/container-search/src/test/java/com/yahoo/search/schema/SchemaInfoTest.java index 728ebbf8f7f..d2cd2c4bb33 100644 --- a/container-search/src/test/java/com/yahoo/search/config/SchemaInfoTest.java +++ b/container-search/src/test/java/com/yahoo/search/schema/SchemaInfoTest.java @@ -1,8 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.search.config; +package com.yahoo.search.schema; import com.yahoo.tensor.TensorType; -import com.yahoo.yolean.Exceptions; import org.junit.Test; import static org.junit.Assert.assertEquals; diff --git a/container-search/src/test/java/com/yahoo/search/config/SchemaInfoTester.java b/container-search/src/test/java/com/yahoo/search/schema/SchemaInfoTester.java index d5b4522f3aa..ff996464c4a 100644 --- a/container-search/src/test/java/com/yahoo/search/config/SchemaInfoTester.java +++ b/container-search/src/test/java/com/yahoo/search/schema/SchemaInfoTester.java @@ -1,11 +1,14 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.search.config; +package com.yahoo.search.schema; import com.yahoo.container.QrSearchersConfig; -import com.yahoo.prelude.fastsearch.DocumentdbInfoConfig; import com.yahoo.search.Query; +import com.yahoo.search.config.IndexInfoConfig; +import com.yahoo.search.config.SchemaInfoConfig; +import com.yahoo.search.schema.RankProfile; +import com.yahoo.search.schema.Schema; +import com.yahoo.search.schema.SchemaInfo; import com.yahoo.tensor.TensorType; -import com.yahoo.yolean.Exceptions; import java.util.ArrayList; import java.util.HashMap; @@ -86,36 +89,36 @@ public class SchemaInfoTester { static SchemaInfo createSchemaInfoFromConfig() { var indexInfoConfig = new IndexInfoConfig.Builder(); - var rankProfileCommon = new DocumentdbInfoConfig.Documentdb.Rankprofile.Builder(); + var rankProfileCommon = new SchemaInfoConfig.Schema.Rankprofile.Builder(); rankProfileCommon.name("commonProfile"); - rankProfileCommon.input(new DocumentdbInfoConfig.Documentdb.Rankprofile.Input.Builder().name("query(myTensor1)").type("tensor(a{},b{})")); - rankProfileCommon.input(new DocumentdbInfoConfig.Documentdb.Rankprofile.Input.Builder().name("query(myTensor2)").type("tensor(x[2],y[2])")); - rankProfileCommon.input(new DocumentdbInfoConfig.Documentdb.Rankprofile.Input.Builder().name("query(myTensor3)").type("tensor(x[2],y[2])")); - rankProfileCommon.input(new DocumentdbInfoConfig.Documentdb.Rankprofile.Input.Builder().name("query(myTensor4)").type("tensor<float>(x[5])")); + rankProfileCommon.input(new SchemaInfoConfig.Schema.Rankprofile.Input.Builder().name("query(myTensor1)").type("tensor(a{},b{})")); + rankProfileCommon.input(new SchemaInfoConfig.Schema.Rankprofile.Input.Builder().name("query(myTensor2)").type("tensor(x[2],y[2])")); + rankProfileCommon.input(new SchemaInfoConfig.Schema.Rankprofile.Input.Builder().name("query(myTensor3)").type("tensor(x[2],y[2])")); + rankProfileCommon.input(new SchemaInfoConfig.Schema.Rankprofile.Input.Builder().name("query(myTensor4)").type("tensor<float>(x[5])")); - var documentDbInfoInfoConfig = new DocumentdbInfoConfig.Builder(); + var schemaInfoInfoConfig = new SchemaInfoConfig.Builder(); - var documentDbA = new DocumentdbInfoConfig.Documentdb.Builder(); - documentDbA.name("a"); - documentDbA.rankprofile(rankProfileCommon); - var rankProfileInconsistentA = new DocumentdbInfoConfig.Documentdb.Rankprofile.Builder(); + var schemaA = new SchemaInfoConfig.Schema.Builder(); + schemaA.name("a"); + schemaA.rankprofile(rankProfileCommon); + var rankProfileInconsistentA = new SchemaInfoConfig.Schema.Rankprofile.Builder(); rankProfileInconsistentA.name("inconsistent"); - rankProfileInconsistentA.input(new DocumentdbInfoConfig.Documentdb.Rankprofile.Input.Builder().name("query(myTensor1)").type("tensor(a{},b{})")); - documentDbA.rankprofile(rankProfileInconsistentA); - documentDbInfoInfoConfig.documentdb(documentDbA); - - var documentDbB = new DocumentdbInfoConfig.Documentdb.Builder(); - documentDbB.name("b"); - documentDbB.rankprofile(rankProfileCommon); - var rankProfileInconsistentB = new DocumentdbInfoConfig.Documentdb.Rankprofile.Builder(); + rankProfileInconsistentA.input(new SchemaInfoConfig.Schema.Rankprofile.Input.Builder().name("query(myTensor1)").type("tensor(a{},b{})")); + schemaA.rankprofile(rankProfileInconsistentA); + schemaInfoInfoConfig.schema(schemaA); + + var schemaB = new SchemaInfoConfig.Schema.Builder(); + schemaB.name("b"); + schemaB.rankprofile(rankProfileCommon); + var rankProfileInconsistentB = new SchemaInfoConfig.Schema.Rankprofile.Builder(); rankProfileInconsistentB.name("inconsistent"); - rankProfileInconsistentB.input(new DocumentdbInfoConfig.Documentdb.Rankprofile.Input.Builder().name("query(myTensor1)").type("tensor(x[10])")); - documentDbB.rankprofile(rankProfileInconsistentB); - var rankProfileBOnly = new DocumentdbInfoConfig.Documentdb.Rankprofile.Builder(); + rankProfileInconsistentB.input(new SchemaInfoConfig.Schema.Rankprofile.Input.Builder().name("query(myTensor1)").type("tensor(x[10])")); + schemaB.rankprofile(rankProfileInconsistentB); + var rankProfileBOnly = new SchemaInfoConfig.Schema.Rankprofile.Builder(); rankProfileBOnly.name("bOnly"); - rankProfileBOnly.input(new DocumentdbInfoConfig.Documentdb.Rankprofile.Input.Builder().name("query(myTensor1)").type("tensor(a{},b{})")); - documentDbB.rankprofile(rankProfileBOnly); - documentDbInfoInfoConfig.documentdb(documentDbB); + rankProfileBOnly.input(new SchemaInfoConfig.Schema.Rankprofile.Input.Builder().name("query(myTensor1)").type("tensor(a{},b{})")); + schemaB.rankprofile(rankProfileBOnly); + schemaInfoInfoConfig.schema(schemaB); var qrSearchersConfig = new QrSearchersConfig.Builder(); var clusterAB = new QrSearchersConfig.Searchcluster.Builder(); @@ -127,7 +130,7 @@ public class SchemaInfoTester { clusterA.searchdef("a"); qrSearchersConfig.searchcluster(clusterA); - return new SchemaInfo(indexInfoConfig.build(), documentDbInfoInfoConfig.build(), qrSearchersConfig.build()); + return new SchemaInfo(indexInfoConfig.build(), schemaInfoInfoConfig.build(), qrSearchersConfig.build()); } } diff --git a/container-search/src/test/java/com/yahoo/search/searchchain/config/test/DependencyConfigTestCase.java b/container-search/src/test/java/com/yahoo/search/searchchain/config/test/DependencyConfigTestCase.java index 7d691cae3ba..553a9233bf1 100644 --- a/container-search/src/test/java/com/yahoo/search/searchchain/config/test/DependencyConfigTestCase.java +++ b/container-search/src/test/java/com/yahoo/search/searchchain/config/test/DependencyConfigTestCase.java @@ -42,7 +42,7 @@ public class DependencyConfigTestCase { } @AfterClass - public static void removeComponentsConfig() throws IOException { + public static void removeComponentsConfig() { new File(root + "/components.cfg").delete(); tearDown(); } diff --git a/container-search/src/test/java/com/yahoo/search/searchchain/config/test/SearchChainConfigurerTestCase.java b/container-search/src/test/java/com/yahoo/search/searchchain/config/test/SearchChainConfigurerTestCase.java index 0f1f77d66b4..781370d9e25 100644 --- a/container-search/src/test/java/com/yahoo/search/searchchain/config/test/SearchChainConfigurerTestCase.java +++ b/container-search/src/test/java/com/yahoo/search/searchchain/config/test/SearchChainConfigurerTestCase.java @@ -155,6 +155,7 @@ public class SearchChainConfigurerTestCase { copyFile(testDir + "qr-search.cfg", cfgDir + "/qr-search.cfg"); copyFile(testDir + "qr-searchers.cfg", cfgDir + "/qr-searchers.cfg"); copyFile(testDir + "index-info.cfg", cfgDir + "/index-info.cfg"); + copyFile(testDir + "schema-info.cfg", cfgDir + "/schema-info.cfg"); copyFile(testDir + "specialtokens.cfg", cfgDir + "/specialtokens.cfg"); copyFile(testDir + "three-searchers.cfg", cfgDir + "/chains.cfg"); copyFile(testDir + "container-http.cfg", cfgDir + "/container-http.cfg"); @@ -208,6 +209,7 @@ public class SearchChainConfigurerTestCase { copyFile(testDir + "qr-search.cfg", cfgDir + "/qr-search.cfg"); copyFile(testDir + "qr-searchers.cfg", cfgDir + "/qr-searchers.cfg"); copyFile(testDir + "index-info.cfg", cfgDir + "/index-info.cfg"); + copyFile(testDir + "schema-info.cfg", cfgDir + "/schema-info.cfg"); copyFile(testDir + "specialtokens.cfg", cfgDir + "/specialtokens.cfg"); copyFile(testDir + "chainsConfigUpdate_1.cfg", cfgDir + "/chains.cfg"); copyFile(testDir + "container-http.cfg", cfgDir + "/container-http.cfg"); diff --git a/container-search/src/test/java/com/yahoo/search/searchchain/config/test/dependencyConfig/schema-info.cfg b/container-search/src/test/java/com/yahoo/search/searchchain/config/test/dependencyConfig/schema-info.cfg new file mode 100644 index 00000000000..e69de29bb2d --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/searchchain/config/test/dependencyConfig/schema-info.cfg diff --git a/container-search/src/test/java/com/yahoo/search/searchchain/config/test/schema-info.cfg b/container-search/src/test/java/com/yahoo/search/searchchain/config/test/schema-info.cfg new file mode 100644 index 00000000000..e69de29bb2d --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/searchchain/config/test/schema-info.cfg diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/application/v4/model/SubmitOptions.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/application/v4/model/SubmitOptions.java deleted file mode 100644 index c1144d5f18d..00000000000 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/application/v4/model/SubmitOptions.java +++ /dev/null @@ -1,55 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.controller.api.application.v4.model; - -import com.fasterxml.jackson.annotation.JsonIgnoreProperties; -import com.yahoo.vespa.hosted.controller.api.identifiers.GitBranch; -import com.yahoo.vespa.hosted.controller.api.identifiers.GitCommit; -import com.yahoo.vespa.hosted.controller.api.identifiers.GitRepository; - -import java.util.Objects; - -/** - * Additional options to be sent along the application package and the application test package - * when submitting an application to the controller - * - * @author freva - */ -@JsonIgnoreProperties(ignoreUnknown = true) -public class SubmitOptions { - - public GitRepository repository; - public GitBranch branch; - public GitCommit commit; - - public static SubmitOptions from(String repository, String branch, String commit) { - SubmitOptions options = new SubmitOptions(); - options.repository = new GitRepository(repository); - options.branch = new GitBranch(branch); - options.commit = new GitCommit(commit); - return options; - } - - @Override - public String toString() { - return "SubmitOptions{" + - "repository=" + repository + - ", branch=" + branch + - ", commit=" + commit + - '}'; - } - - @Override - public boolean equals(Object o) { - if (this == o) return true; - if (o == null || getClass() != o.getClass()) return false; - SubmitOptions that = (SubmitOptions) o; - return Objects.equals(repository, that.repository) && - Objects.equals(branch, that.branch) && - Objects.equals(commit, that.commit); - } - - @Override - public int hashCode() { - return Objects.hash(repository, branch, commit); - } -} diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/application/v4/model/SubmitResult.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/application/v4/model/SubmitResult.java deleted file mode 100644 index f575464e84b..00000000000 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/application/v4/model/SubmitResult.java +++ /dev/null @@ -1,37 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.controller.api.application.v4.model; - -import com.fasterxml.jackson.annotation.JsonIgnoreProperties; - -import java.util.Objects; - -/** - * Represents the response from application submit request - * - * @author freva - */ -@JsonIgnoreProperties(ignoreUnknown = true) -public class SubmitResult { - - public String version; - - @Override - public String toString() { - return "SubmitResult{" + - "version='" + version + '\'' + - '}'; - } - - @Override - public boolean equals(Object o) { - if (this == o) return true; - if (o == null || getClass() != o.getClass()) return false; - SubmitResult that = (SubmitResult) o; - return Objects.equals(version, that.version); - } - - @Override - public int hashCode() { - return Objects.hash(version); - } -} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/pkg/ApplicationPackage.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/pkg/ApplicationPackage.java index 0d39703b70d..344ed7ec729 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/pkg/ApplicationPackage.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/pkg/ApplicationPackage.java @@ -78,6 +78,7 @@ public class ApplicationPackage { private final ZipArchiveCache files; private final Optional<Version> compileVersion; private final Optional<Instant> buildTime; + private final Optional<Version> parentVersion; private final List<X509Certificate> trustedCertificates; /** @@ -110,6 +111,7 @@ public class ApplicationPackage { Optional<Inspector> buildMetaObject = files.get(buildMetaFile).map(SlimeUtils::jsonToSlime).map(Slime::get); this.compileVersion = buildMetaObject.flatMap(object -> parse(object, "compileVersion", field -> Version.fromString(field.asString()))); this.buildTime = buildMetaObject.flatMap(object -> parse(object, "buildTime", field -> Instant.ofEpochMilli(field.asLong()))); + this.parentVersion = buildMetaObject.flatMap(object -> parse(object, "parentVersion", field -> Version.fromString(field.asString()))); this.trustedCertificates = files.get(trustedCertificatesFile).map(bytes -> X509CertificateUtils.certificateListFromPem(new String(bytes, UTF_8))).orElse(List.of()); @@ -159,6 +161,9 @@ public class ApplicationPackage { /** Returns the time this package was built, if known. */ public Optional<Instant> buildTime() { return buildTime; } + /** Returns the parent version used to compile the package, if known. */ + public Optional<Version> parentVersion() { return parentVersion; } + /** Returns the list of certificates trusted by this application, or an empty list if no trust configured. */ public List<X509Certificate> trustedCertificates() { return trustedCertificates; @@ -166,7 +171,7 @@ public class ApplicationPackage { private static <Type> Optional<Type> parse(Inspector buildMetaObject, String fieldName, Function<Inspector, Type> mapper) { if ( ! buildMetaObject.field(fieldName).valid()) - throw new IllegalArgumentException("Missing value '" + fieldName + "' in '" + buildMetaFile + "'"); + return Optional.empty(); try { return Optional.of(mapper.apply(buildMetaObject.field(fieldName))); } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java index b0966f7db21..1d56e2db08b 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java @@ -461,9 +461,7 @@ public class JobController { } /** Accepts and stores a new application package and test jar pair under a generated application version key. */ - public ApplicationVersion submit(TenantAndApplicationId id, Optional<SourceRevision> revision, Optional<String> authorEmail, - Optional<String> sourceUrl, long projectId, ApplicationPackage applicationPackage, - byte[] testPackageBytes, Optional<String> description, int risk) { + public ApplicationVersion submit(TenantAndApplicationId id, Submission submission, long projectId) { ApplicationController applications = controller.applications(); AtomicReference<ApplicationVersion> version = new AtomicReference<>(); applications.lockApplicationOrThrow(id, application -> { @@ -471,47 +469,46 @@ public class JobController { Optional<ApplicationPackage> previousPackage = previousVersion.flatMap(previous -> applications.applicationStore().find(id.tenant(), id.application(), previous.buildNumber().getAsLong())) .map(ApplicationPackage::new); long previousBuild = previousVersion.map(latestVersion -> latestVersion.buildNumber().getAsLong()).orElse(0L); - String packageHash = applicationPackage.bundleHash() + ApplicationPackage.calculateHash(testPackageBytes); - RevisionId revisionId = RevisionId.forProduction(1 + previousBuild); - version.set(ApplicationVersion.forProduction(revisionId, - revision, - authorEmail, - applicationPackage.compileVersion(), - applicationPackage.buildTime(), - sourceUrl, - revision.map(SourceRevision::commit), - Optional.of(packageHash), - description, - risk)); - - byte[] diff = previousPackage.map(previous -> ApplicationPackageDiff.diff(previous, applicationPackage)) - .orElseGet(() -> ApplicationPackageDiff.diffAgainstEmpty(applicationPackage)); + version.set(submission.toApplicationVersion(1 + previousBuild)); + + byte[] diff = previousPackage.map(previous -> ApplicationPackageDiff.diff(previous, submission.applicationPackage())) + .orElseGet(() -> ApplicationPackageDiff.diffAgainstEmpty(submission.applicationPackage())); applications.applicationStore().put(id.tenant(), id.application(), version.get().id(), - applicationPackage.zippedContent(), - testPackageBytes, + submission.applicationPackage().zippedContent(), + submission.testPackage(), diff); applications.applicationStore().putMeta(id.tenant(), id.application(), controller.clock().instant(), - applicationPackage.metaDataZip()); + submission.applicationPackage().metaDataZip()); application = application.withProjectId(projectId == -1 ? OptionalLong.empty() : OptionalLong.of(projectId)); application = application.withRevisions(revisions -> revisions.with(version.get())); application = withPrunedPackages(application); - TestSummary testSummary = TestPackage.validateTests(applicationPackage.deploymentSpec(), testPackageBytes); + TestSummary testSummary = TestPackage.validateTests(submission.applicationPackage().deploymentSpec(), submission.testPackage()); if (testSummary.problems().isEmpty()) - controller.notificationsDb().removeNotification(NotificationSource.from(id), - Type.testPackage); + controller.notificationsDb().removeNotification(NotificationSource.from(id), Type.testPackage); else controller.notificationsDb().setNotification(NotificationSource.from(id), Type.testPackage, Notification.Level.warning, testSummary.problems()); - applications.storeWithUpdatedConfig(application, applicationPackage); + submission.applicationPackage().parentVersion().ifPresent(parent -> { + if (parent.getMajor() < controller.readSystemVersion().getMajor()) + controller.notificationsDb().setNotification(NotificationSource.from(id), + Type.submission, + Notification.Level.warning, + "Parent version used to compile the application is on a " + + "lower major version than the current Vespa Cloud version"); + else + controller.notificationsDb().removeNotification(NotificationSource.from(id), Type.submission); + }); + + applications.storeWithUpdatedConfig(application, submission.applicationPackage()); applications.deploymentTrigger().triggerNewRevision(id); }); return version.get(); diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/Submission.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/Submission.java new file mode 100644 index 00000000000..e366920690b --- /dev/null +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/Submission.java @@ -0,0 +1,57 @@ +package com.yahoo.vespa.hosted.controller.deployment; + +import com.yahoo.vespa.hosted.controller.api.integration.deployment.ApplicationVersion; +import com.yahoo.vespa.hosted.controller.api.integration.deployment.RevisionId; +import com.yahoo.vespa.hosted.controller.api.integration.deployment.SourceRevision; +import com.yahoo.vespa.hosted.controller.application.pkg.ApplicationPackage; + +import java.util.Optional; + +import static com.yahoo.vespa.hosted.controller.application.pkg.ApplicationPackage.calculateHash; + +/** + * @author jonmv + */ +public class Submission { + + private final ApplicationPackage applicationPackage; + private final byte[] testPackage; + private final Optional<String> sourceUrl; + private final Optional<SourceRevision> source; + private final Optional<String> authorEmail; + private final Optional<String> description; + private final int risk; + + public Submission(ApplicationPackage applicationPackage, byte[] testPackage, Optional<String> sourceUrl, + Optional<SourceRevision> source, Optional<String> authorEmail, Optional<String> description, int risk) { + this.applicationPackage = applicationPackage; + this.testPackage = testPackage; + this.sourceUrl = sourceUrl; + this.source = source; + this.authorEmail = authorEmail; + this.description = description; + this.risk = risk; + } + + public static Submission basic(ApplicationPackage applicationPackage, byte[] testPackage) { + return new Submission(applicationPackage, testPackage, Optional.empty(), Optional.empty(), Optional.empty(), Optional.empty(), 0); + } + + public ApplicationVersion toApplicationVersion(long number) { + return ApplicationVersion.forProduction(RevisionId.forProduction(number), + source, + authorEmail, + applicationPackage.compileVersion(), + applicationPackage.buildTime(), + sourceUrl, + source.map(SourceRevision::commit), + Optional.of(applicationPackage.bundleHash() + calculateHash(testPackage)), + description, + risk); + } + + public ApplicationPackage applicationPackage() { return applicationPackage; } + + public byte[] testPackage() { return testPackage; } + +} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/CloudTrialExpirer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/CloudTrialExpirer.java index 28476bcbe6c..8765884e23c 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/CloudTrialExpirer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/CloudTrialExpirer.java @@ -47,8 +47,7 @@ public class CloudTrialExpirer extends ControllerMaintainer { private void moveInactiveTenantsToNonePlan() { var predicate = tenantReadersNotLoggedIn(loginExpiry) - .and(this::tenantHasTrialPlan) - .and(this::tenantHasNoDeployments); + .and(this::tenantHasTrialPlan); forTenant("'none' plan", predicate, this::setPlanNone); } @@ -61,11 +60,12 @@ public class CloudTrialExpirer extends ControllerMaintainer { } private void forTenant(String name, Predicate<Tenant> p, Consumer<List<Tenant>> c) { - var predicate = ((Predicate<Tenant>) this::tenantIsCloudTenant) - .and(this::tenantIsNotExemptFromExpiry); + var predicate = p.and(this::tenantIsCloudTenant) + .and(this::tenantIsNotExemptFromExpiry) + .and(this::tenantHasNoDeployments); var tenants = controller().tenants().asList().stream() - .filter(predicate.and(p)) + .filter(predicate) .collect(Collectors.toList()); if (! tenants.isEmpty()) { @@ -119,7 +119,15 @@ public class CloudTrialExpirer extends ControllerMaintainer { private void tombstoneTenants(List<Tenant> tenants) { tenants.forEach(tenant -> { + deleteApplicationsWithNoDeployments(tenant); controller().tenants().delete(tenant.name(), Optional.empty(), false); }); } + + private void deleteApplicationsWithNoDeployments(Tenant tenant) { + controller().applications().asList(tenant.name()).forEach(application -> { + // this only removes applications with no active deployments + controller().applications().deleteApplication(application.id(), Optional.empty()); + }); + } } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/notification/Notification.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/notification/Notification.java index 38cb09355ca..8a363405c41 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/notification/Notification.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/notification/Notification.java @@ -69,6 +69,9 @@ public class Notification { /** Related to contents of application package, e.g., usage of deprecated features/syntax */ applicationPackage, + /** Related to contents of application package, e.g., old parent or compile version, or errors detectable on submission */ + submission, + /** Related to contents of application test package, e.g., mismatch between deployment spec and provided tests */ testPackage, diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/NotificationsSerializer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/NotificationsSerializer.java index 570dbdd870e..16ec240a116 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/NotificationsSerializer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/NotificationsSerializer.java @@ -92,6 +92,7 @@ public class NotificationsSerializer { private static String asString(Notification.Type type) { switch (type) { case applicationPackage: return "applicationPackage"; + case submission: return "submission"; case testPackage: return "testPackage"; case deployment: return "deployment"; case feedBlock: return "feedBlock"; @@ -103,6 +104,7 @@ public class NotificationsSerializer { private static Notification.Type typeFrom(Inspector field) { switch (field.asString()) { case "applicationPackage": return Notification.Type.applicationPackage; + case "submission": return Notification.Type.submission; case "testPackage": return Notification.Type.testPackage; case "deployment": return Notification.Type.deployment; case "feedBlock": return Notification.Type.feedBlock; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java index 6dc5e4565de..88b319d0051 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java @@ -94,6 +94,7 @@ import com.yahoo.vespa.hosted.controller.deployment.DeploymentTrigger; import com.yahoo.vespa.hosted.controller.deployment.DeploymentTrigger.ChangesToCancel; import com.yahoo.vespa.hosted.controller.deployment.JobStatus; import com.yahoo.vespa.hosted.controller.deployment.Run; +import com.yahoo.vespa.hosted.controller.deployment.Submission; import com.yahoo.vespa.hosted.controller.deployment.TestConfigSerializer; import com.yahoo.vespa.hosted.controller.maintenance.ResourceMeterMaintainer; import com.yahoo.vespa.hosted.controller.notification.Notification; @@ -744,6 +745,7 @@ public class ApplicationApiHandler extends AuditLoggingRequestHandler { private static String notificationTypeAsString(Notification.Type type) { switch (type) { + case submission: case applicationPackage: return "applicationPackage"; case testPackage: return "testPackage"; case deployment: return "deployment"; @@ -2713,31 +2715,26 @@ public class ApplicationApiHandler extends AuditLoggingRequestHandler { ApplicationPackage applicationPackage = new ApplicationPackage(dataParts.get(EnvironmentResource.APPLICATION_ZIP), true); + byte[] testPackage = dataParts.get(EnvironmentResource.APPLICATION_TEST_ZIP); + Submission submission = new Submission(applicationPackage, testPackage, sourceUrl, sourceRevision, authorEmail, description, risk); + controller.applications().verifyApplicationIdentityConfiguration(TenantName.from(tenant), Optional.empty(), Optional.empty(), applicationPackage, Optional.of(requireUserPrincipal(request))); - ensureApplicationExists(TenantAndApplicationId.from(tenant, application), request); - - return JobControllerApiHandlerHelper.submitResponse(controller.jobController(), - tenant, - application, - sourceRevision, - authorEmail, - sourceUrl, - description, - risk, - projectId, - applicationPackage, - dataParts.get(EnvironmentResource.APPLICATION_TEST_ZIP)); + TenantAndApplicationId id = TenantAndApplicationId.from(tenant, application); + ensureApplicationExists(id, request); + return JobControllerApiHandlerHelper.submitResponse(controller.jobController(), id, submission, projectId); } private HttpResponse removeAllProdDeployments(String tenant, String application) { - JobControllerApiHandlerHelper.submitResponse(controller.jobController(), tenant, application, - Optional.empty(), Optional.empty(), Optional.empty(), Optional.empty(), 0, 1, - ApplicationPackage.deploymentRemoval(), new byte[0]); + JobControllerApiHandlerHelper.submitResponse(controller.jobController(), + TenantAndApplicationId.from(tenant, application), + new Submission(ApplicationPackage.deploymentRemoval(), new byte[0], Optional.empty(), + Optional.empty(), Optional.empty(), Optional.empty(), 0), + 0); return new MessageResponse("All deployments removed"); } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/JobControllerApiHandlerHelper.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/JobControllerApiHandlerHelper.java index de03bbfb767..80425609aa6 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/JobControllerApiHandlerHelper.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/JobControllerApiHandlerHelper.java @@ -34,6 +34,7 @@ import com.yahoo.vespa.hosted.controller.deployment.Run; import com.yahoo.vespa.hosted.controller.deployment.RunLog; import com.yahoo.vespa.hosted.controller.deployment.RunStatus; import com.yahoo.vespa.hosted.controller.deployment.Step; +import com.yahoo.vespa.hosted.controller.deployment.Submission; import com.yahoo.vespa.hosted.controller.deployment.Versions; import com.yahoo.vespa.hosted.controller.versions.VersionStatus; import com.yahoo.vespa.hosted.controller.versions.VespaVersion; @@ -194,14 +195,8 @@ class JobControllerApiHandlerHelper { * * @return Response with the new application version */ - static HttpResponse submitResponse(JobController jobController, String tenant, String application, - Optional<SourceRevision> sourceRevision, Optional<String> authorEmail, - Optional<String> sourceUrl, Optional<String> description, int risk, long projectId, - ApplicationPackage applicationPackage, byte[] testPackage) { - ApplicationVersion version = jobController.submit(TenantAndApplicationId.from(tenant, application), sourceRevision, authorEmail, - sourceUrl, projectId, applicationPackage, testPackage, description, risk); - - return new MessageResponse("application " + version); + static HttpResponse submitResponse(JobController jobController, TenantAndApplicationId id, Submission submission, long projectId) { + return new MessageResponse("application " + jobController.submit(id, submission, projectId)); } /** Aborts any job of the given type. */ diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/billing/BillingApiHandlerV2.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/billing/BillingApiHandlerV2.java index 612b584c7c0..0f3e5b7f76b 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/billing/BillingApiHandlerV2.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/billing/BillingApiHandlerV2.java @@ -28,12 +28,9 @@ import com.yahoo.vespa.hosted.controller.tenant.Tenant; import javax.ws.rs.BadRequestException; import java.math.BigDecimal; import java.time.Clock; -import java.time.Instant; import java.time.LocalDate; import java.time.ZoneOffset; -import java.time.chrono.ChronoZonedDateTime; import java.time.format.DateTimeFormatter; -import java.time.temporal.ChronoUnit; import java.util.Comparator; import java.util.List; import java.util.Optional; @@ -181,8 +178,7 @@ public class BillingApiHandlerV2 extends RestApiRequestHandler<BillingApiHandler var tenantName = TenantName.from(requestContext.pathParameters().getStringOrThrow("tenant")); var tenant = tenants.require(tenantName, CloudTenant.class); var untilAt = untilParameter(requestContext); - var usage = billing.createUncommittedBill(tenant.name(), untilAt.atZone(ZoneOffset.UTC).toLocalDate()); - + var usage = billing.createUncommittedBill(tenant.name(), untilAt); var slime = new Slime(); usageToSlime(slime.setObject(), usage); return slime; @@ -192,7 +188,7 @@ public class BillingApiHandlerV2 extends RestApiRequestHandler<BillingApiHandler private Slime accountant(RestApi.RequestContext requestContext) { var untilAt = untilParameter(requestContext); - var usagePerTenant = billing.createUncommittedBills(untilAt.atZone(ZoneOffset.UTC).toLocalDate()); + var usagePerTenant = billing.createUncommittedBills(untilAt); var response = new Slime(); var tenantsResponse = response.setObject().setArray("tenants"); @@ -214,7 +210,7 @@ public class BillingApiHandlerV2 extends RestApiRequestHandler<BillingApiHandler var tenant = tenants.require(tenantName, CloudTenant.class); var untilAt = untilParameter(requestContext); - var usage = billing.createUncommittedBill(tenant.name(), untilAt.atZone(ZoneOffset.UTC).toLocalDate()); + var usage = billing.createUncommittedBill(tenant.name(), untilAt); var slime = new Slime(); toSlime(slime.setObject(), usage); @@ -320,21 +316,15 @@ public class BillingApiHandlerV2 extends RestApiRequestHandler<BillingApiHandler // ---------- END INVOICE RENDERING ---------- - private Instant untilParameter(RestApi.RequestContext ctx) { + private LocalDate untilParameter(RestApi.RequestContext ctx) { return ctx.queryParameters().getString("until") .map(LocalDate::parse) .map(date -> date.plusDays(1)) - .map(date -> date.atStartOfDay(ZoneOffset.UTC)) - .map(ChronoZonedDateTime::toInstant) - .orElseGet(this::startOfDayTomorrowUTC); - } - - private Instant startOfDayTodayUTC() { - return LocalDate.now(clock.withZone(ZoneOffset.UTC)).atStartOfDay(ZoneOffset.UTC).toInstant(); + .orElseGet(this::tomorrow); } - private Instant startOfDayTomorrowUTC() { - return startOfDayTodayUTC().plus(1, ChronoUnit.DAYS); + private LocalDate tomorrow() { + return LocalDate.now(clock).plusDays(1); } private static String getInspectorFieldOrThrow(Inspector inspector, String field) { diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java index 20b64419f28..0ecac036913 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java @@ -38,6 +38,7 @@ import com.yahoo.vespa.hosted.controller.application.pkg.ApplicationPackage; import com.yahoo.vespa.hosted.controller.deployment.ApplicationPackageBuilder; import com.yahoo.vespa.hosted.controller.deployment.DeploymentContext; import com.yahoo.vespa.hosted.controller.deployment.DeploymentTester; +import com.yahoo.vespa.hosted.controller.deployment.Submission; import com.yahoo.vespa.hosted.controller.integration.ZoneApiMock; import com.yahoo.vespa.hosted.controller.notification.Notification; import com.yahoo.vespa.hosted.controller.notification.Notification.Level; @@ -1118,8 +1119,7 @@ public class ControllerTest { ApplicationPackage applicationPackage = ApplicationPackageBuilder.fromDeploymentXml(deploymentXml); byte[] testPackage = ApplicationPackage.filesZip(Map.of("tests/staging-test/foo.json", new byte[0])); var app = tester.newDeploymentContext(); - tester.jobs().submit(app.application().id(), Optional.empty(), Optional.empty(), Optional.empty(), 1, - applicationPackage, testPackage, Optional.empty(), 0); + tester.jobs().submit(app.application().id(), Submission.basic(applicationPackage, testPackage), 1); assertEquals(List.of(new Notification(tester.clock().instant(), Type.testPackage, Level.warning, diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/ApplicationPackageBuilder.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/ApplicationPackageBuilder.java index a464e3d7e9b..27cf1554b4d 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/ApplicationPackageBuilder.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/ApplicationPackageBuilder.java @@ -317,7 +317,9 @@ public class ApplicationPackageBuilder { } private static byte[] buildMeta(Version compileVersion) { - return ("{\"compileVersion\":\"" + compileVersion.toFullString() + "\",\"buildTime\":1000}").getBytes(UTF_8); + return ("{\"compileVersion\":\"" + compileVersion.toFullString() + + "\",\"buildTime\":1000,\"parentVersion\":\"" + + compileVersion.toFullString() + "\"}").getBytes(UTF_8); } public ApplicationPackage build() { diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentContext.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentContext.java index ad6a987d42b..fd294f9cf9f 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentContext.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentContext.java @@ -311,7 +311,7 @@ public class DeploymentContext { .projectId() .orElse(1000); // These are really set through submission, so just pick one if it hasn't been set. var testerpackage = new byte[]{ (byte) (salt >> 56), (byte) (salt >> 48), (byte) (salt >> 40), (byte) (salt >> 32), (byte) (salt >> 24), (byte) (salt >> 16), (byte) (salt >> 8), (byte) salt }; - lastSubmission = jobs.submit(applicationId, sourceRevision, Optional.of("a@b"), Optional.empty(), projectId, applicationPackage, testerpackage, Optional.empty(), risk).id(); + lastSubmission = jobs.submit(applicationId, new Submission(applicationPackage, testerpackage, Optional.empty(), sourceRevision, Optional.of("a@b"), Optional.empty(), risk), projectId).id(); return this; } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/CloudTrialExpirerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/CloudTrialExpirerTest.java index f37a5a6893d..f768ab5e61b 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/CloudTrialExpirerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/CloudTrialExpirerTest.java @@ -18,6 +18,7 @@ import java.time.Duration; import java.util.List; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; /** * @author ogronnesby @@ -72,6 +73,16 @@ public class CloudTrialExpirerTest { assertPlan("with-deployments", "trial"); } + @Test + public void delete_tenants_with_applications_with_no_deployments() { + registerTenant("with-apps", "trial", Duration.ofDays(30)); + tester.createApplication("with-apps", "app1", "instance1"); + expirer.maintain(); + assertPlan("with-apps", "none"); + expirer.maintain(); + assertTrue(tester.controller().tenants().get("with-apps").isEmpty()); + } + private void registerTenant(String tenantName, String plan, Duration timeSinceLastLogin) { var name = TenantName.from(tenantName); tester.createTenant(tenantName, Tenant.Type.cloud); diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/JobRunnerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/JobRunnerTest.java index 19971a0ee5b..5fd1e8347ef 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/JobRunnerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/JobRunnerTest.java @@ -18,6 +18,7 @@ import com.yahoo.vespa.hosted.controller.deployment.RunStatus; import com.yahoo.vespa.hosted.controller.deployment.Step; import com.yahoo.vespa.hosted.controller.deployment.Step.Status; import com.yahoo.vespa.hosted.controller.deployment.StepRunner; +import com.yahoo.vespa.hosted.controller.deployment.Submission; import com.yahoo.vespa.hosted.controller.deployment.Versions; import com.yahoo.vespa.hosted.controller.integration.MetricsMock; import org.junit.Test; @@ -91,9 +92,9 @@ public class JobRunnerTest { TenantAndApplicationId appId = tester.createApplication("tenant", "real", "default").id(); ApplicationId id = appId.defaultInstance(); byte[] testPackageBytes = new byte[0]; - jobs.submit(appId, Optional.empty(), Optional.empty(), Optional.empty(), 2, applicationPackage, testPackageBytes, Optional.empty(), 0); + jobs.submit(appId, Submission.basic(applicationPackage, testPackageBytes), 2); - start(jobs, id, systemTest); + start(jobs, id, systemTest); try { start(jobs, id, systemTest); fail("Job is already running, so this should not be allowed!"); @@ -123,7 +124,7 @@ public class JobRunnerTest { TenantAndApplicationId appId = tester.createApplication("tenant", "real", "default").id(); ApplicationId id = appId.defaultInstance(); byte[] testPackageBytes = new byte[0]; - jobs.submit(appId, Optional.empty(), Optional.empty(), Optional.empty(), 2, applicationPackage, testPackageBytes, Optional.empty(), 0); + jobs.submit(appId, Submission.basic(applicationPackage, testPackageBytes), 2); Supplier<Run> run = () -> jobs.last(id, systemTest).get(); start(jobs, id, systemTest); @@ -231,7 +232,7 @@ public class JobRunnerTest { TenantAndApplicationId appId = tester.createApplication("tenant", "real", "default").id(); ApplicationId id = appId.defaultInstance(); byte[] testPackageBytes = new byte[0]; - jobs.submit(appId, Optional.empty(), Optional.empty(), Optional.empty(), 2, applicationPackage, testPackageBytes, Optional.empty(), 0); + jobs.submit(appId, Submission.basic(applicationPackage, testPackageBytes), 2); RunId runId = new RunId(id, systemTest, 1); start(jobs, id, systemTest); @@ -269,7 +270,7 @@ public class JobRunnerTest { ApplicationId instanceId = appId.defaultInstance(); JobId jobId = new JobId(instanceId, systemTest); byte[] testPackageBytes = new byte[0]; - jobs.submit(appId, Optional.empty(), Optional.empty(), Optional.empty(), 2, applicationPackage, testPackageBytes, Optional.empty(), 0); + jobs.submit(appId, Submission.basic(applicationPackage, testPackageBytes), 2); assertFalse(jobs.lastSuccess(jobId).isPresent()); for (int i = 0; i < jobs.historyLength(); i++) { @@ -365,7 +366,7 @@ public class JobRunnerTest { TenantAndApplicationId appId = tester.createApplication("tenant", "real", "default").id(); ApplicationId id = appId.defaultInstance(); byte[] testPackageBytes = new byte[0]; - jobs.submit(appId, Optional.empty(), Optional.empty(), Optional.empty(), 2, applicationPackage, testPackageBytes, Optional.empty(), 0); + jobs.submit(appId, Submission.basic(applicationPackage, testPackageBytes), 2); start(jobs, id, systemTest); tester.clock().advance(JobRunner.jobTimeout.plus(Duration.ofSeconds(1))); @@ -383,7 +384,7 @@ public class JobRunnerTest { TenantAndApplicationId appId = tester.createApplication("tenant", "real", "default").id(); ApplicationId id = appId.defaultInstance(); byte[] testPackageBytes = new byte[0]; - jobs.submit(appId, Optional.empty(), Optional.empty(), Optional.empty(), 2, applicationPackage, testPackageBytes, Optional.empty(), 0); + jobs.submit(appId, Submission.basic(applicationPackage, testPackageBytes), 2); for (Step step : JobProfile.of(systemTest).steps()) outcomes.put(step, running); diff --git a/searchcore/src/vespa/searchcore/proton/documentmetastore/raw_document_meta_data.h b/searchcore/src/vespa/searchcore/proton/documentmetastore/raw_document_meta_data.h index e5f7dcc7192..b5e512fcd9e 100644 --- a/searchcore/src/vespa/searchcore/proton/documentmetastore/raw_document_meta_data.h +++ b/searchcore/src/vespa/searchcore/proton/documentmetastore/raw_document_meta_data.h @@ -5,6 +5,8 @@ #include <vespa/document/base/globalid.h> #include <vespa/document/bucket/bucketid.h> #include <persistence/spi/types.h> +#include <algorithm> +#include <atomic> #include <cassert> namespace proton { @@ -18,35 +20,41 @@ struct RawDocumentMetaData using BucketId = document::BucketId; using Timestamp = storage::spi::Timestamp; GlobalId _gid; - uint8_t _bucketUsedBits; - uint8_t _docSizeLow; - uint16_t _docSizeHigh; - Timestamp _timestamp; + std::atomic<uint32_t> _bucket_used_bits_and_doc_size; + std::atomic<uint64_t> _timestamp; + + static uint32_t capped_doc_size(uint32_t doc_size) { return std::min(0xffffffu, doc_size); } RawDocumentMetaData() noexcept : _gid(), - _bucketUsedBits(BucketId::minNumBits), - _docSizeLow(0), - _docSizeHigh(0), - _timestamp() + _bucket_used_bits_and_doc_size(BucketId::minNumBits), + _timestamp(0) { } RawDocumentMetaData(const GlobalId &gid, const BucketId &bucketId, const Timestamp ×tamp, uint32_t docSize) noexcept : _gid(gid), - _bucketUsedBits(bucketId.getUsedBits()), - _docSizeLow(docSize), - _docSizeHigh(docSize >> 8), + _bucket_used_bits_and_doc_size(bucketId.getUsedBits() | (capped_doc_size(docSize) << 8)), _timestamp(timestamp) { assert(bucketId.valid()); BucketId verId(gid.convertToBucketId()); - verId.setUsedBits(_bucketUsedBits); + verId.setUsedBits(bucketId.getUsedBits()); assert(bucketId.getRawId() == verId.getRawId() || bucketId.getRawId() == verId.getId()); - if (docSize >= (1u << 24)) { - _docSizeLow = 0xff; - _docSizeHigh = 0xffff; - } + } + + RawDocumentMetaData(const RawDocumentMetaData& rhs) + : _gid(rhs._gid), + _bucket_used_bits_and_doc_size(rhs._bucket_used_bits_and_doc_size.load(std::memory_order_relaxed)), + _timestamp(rhs._timestamp.load(std::memory_order_relaxed)) + { + } + + RawDocumentMetaData& operator=(const RawDocumentMetaData& rhs) { + _gid = rhs._gid; + _bucket_used_bits_and_doc_size.store(rhs._bucket_used_bits_and_doc_size.load(std::memory_order_relaxed), std::memory_order_relaxed); + _timestamp.store(rhs._timestamp.load(std::memory_order_relaxed), std::memory_order_relaxed); + return *this; } bool operator<(const GlobalId &rhs) const noexcept { return _gid < rhs; } @@ -57,17 +65,17 @@ struct RawDocumentMetaData const GlobalId &getGid() const { return _gid; } GlobalId &getGid() { return _gid; } void setGid(const GlobalId &rhs) { _gid = rhs; } - uint8_t getBucketUsedBits() const { return _bucketUsedBits; } + uint8_t getBucketUsedBits() const { return _bucket_used_bits_and_doc_size.load(std::memory_order_relaxed) & 0xffu; } BucketId getBucketId() const { BucketId ret(_gid.convertToBucketId()); - ret.setUsedBits(_bucketUsedBits); + ret.setUsedBits(getBucketUsedBits()); return ret; } void setBucketUsedBits(uint8_t bucketUsedBits) { assert(BucketId::validUsedBits(bucketUsedBits)); - _bucketUsedBits = bucketUsedBits; + _bucket_used_bits_and_doc_size.store((_bucket_used_bits_and_doc_size.load(std::memory_order_relaxed) & ~0xffu) | bucketUsedBits, std::memory_order_relaxed); } void setBucketId(const BucketId &bucketId) { @@ -77,15 +85,16 @@ struct RawDocumentMetaData verId.setUsedBits(bucketUsedBits); assert(bucketId.getRawId() == verId.getRawId() || bucketId.getRawId() == verId.getId()); - _bucketUsedBits = bucketUsedBits; + setBucketUsedBits(bucketUsedBits); } - Timestamp getTimestamp() const { return _timestamp; } + Timestamp getTimestamp() const { return Timestamp(_timestamp.load(std::memory_order_relaxed)); } + + void setTimestamp(const Timestamp ×tamp) { _timestamp.store(timestamp.getValue(), std::memory_order_relaxed); } - void setTimestamp(const Timestamp ×tamp) { _timestamp = timestamp; } + uint32_t getDocSize() const { return _bucket_used_bits_and_doc_size.load(std::memory_order_relaxed) >> 8; } + void setDocSize(uint32_t docSize) { _bucket_used_bits_and_doc_size.store((_bucket_used_bits_and_doc_size.load(std::memory_order_relaxed) & 0xffu) | (capped_doc_size(docSize) << 8), std::memory_order_relaxed); } - uint32_t getDocSize() const { return _docSizeLow + (static_cast<uint32_t>(_docSizeHigh) << 8); } - void setDocSize(uint32_t docSize) { _docSizeLow = docSize; _docSizeHigh = docSize >> 8; } }; } // namespace proton diff --git a/searchcore/src/vespa/searchcore/proton/matching/matcher.cpp b/searchcore/src/vespa/searchcore/proton/matching/matcher.cpp index 756af216988..381915f53d4 100644 --- a/searchcore/src/vespa/searchcore/proton/matching/matcher.cpp +++ b/searchcore/src/vespa/searchcore/proton/matching/matcher.cpp @@ -128,8 +128,7 @@ Matcher::getStats() { std::lock_guard<std::mutex> guard(_statsLock); MatchingStats stats = std::move(_stats); - _stats = MatchingStats(); - _stats.softDoomFactor(stats.softDoomFactor()); + _stats = MatchingStats(stats.softDoomFactor()); return stats; } diff --git a/searchcore/src/vespa/searchcore/proton/matching/matching_stats.cpp b/searchcore/src/vespa/searchcore/proton/matching/matching_stats.cpp index 2c826094ddf..bbb9dba9a30 100644 --- a/searchcore/src/vespa/searchcore/proton/matching/matching_stats.cpp +++ b/searchcore/src/vespa/searchcore/proton/matching/matching_stats.cpp @@ -19,7 +19,7 @@ constexpr double MAX_CHANGE_FACTOR = 5; } // namespace proton::matching::<unnamed> -MatchingStats::MatchingStats() +MatchingStats::MatchingStats(double prev_soft_doom_factor) : _queries(0), _limited_queries(0), _docidSpaceCovered(0), @@ -28,7 +28,7 @@ MatchingStats::MatchingStats() _docsReRanked(0), _softDoomed(0), _doomOvertime(), - _softDoomFactor(INITIAL_SOFT_DOOM_FACTOR), + _softDoomFactor(prev_soft_doom_factor), _queryCollateralTime(), // TODO: Remove in Vespa 8 _querySetupTime(), _queryLatency(), @@ -89,16 +89,18 @@ MatchingStats::updatesoftDoomFactor(vespalib::duration hardLimit, vespalib::dura // It is merely a safety measure to avoid overflow on bad input as can happen with time senstive stuff // in any soft real time system. if ((hardLimit >= MIN_TIMEOUT) && (softLimit >= MIN_TIMEOUT)) { + double factor = softDoomFactor(); double diff = vespalib::to_s(softLimit - duration)/vespalib::to_s(hardLimit); if (duration < softLimit) { // Since softdoom factor can become very small, allow a minimum change of some size - diff = std::min(diff, _softDoomFactor*MAX_CHANGE_FACTOR); - _softDoomFactor += 0.01*diff; + diff = std::min(diff, factor*MAX_CHANGE_FACTOR); + factor += 0.01*diff; } else { - diff = std::max(diff, -_softDoomFactor*MAX_CHANGE_FACTOR); - _softDoomFactor += 0.02*diff; + diff = std::max(diff, -factor*MAX_CHANGE_FACTOR); + factor += 0.02*diff; } - _softDoomFactor = std::max(_softDoomFactor, 0.01); // Never go below 1% + factor = std::max(factor, 0.01); // Never go below 1% + softDoomFactor(factor); } return *this; } diff --git a/searchcore/src/vespa/searchcore/proton/matching/matching_stats.h b/searchcore/src/vespa/searchcore/proton/matching/matching_stats.h index 047c6fcaf13..eafa14870fa 100644 --- a/searchcore/src/vespa/searchcore/proton/matching/matching_stats.h +++ b/searchcore/src/vespa/searchcore/proton/matching/matching_stats.h @@ -5,6 +5,7 @@ #include <vector> #include <cstddef> #include <vespa/vespalib/util/time.h> +#include <vespa/vespalib/datastore/atomic_value_wrapper.h> namespace proton::matching { @@ -124,7 +125,8 @@ private: size_t _docsReRanked; size_t _softDoomed; Avg _doomOvertime; - double _softDoomFactor; + using SoftDoomFactor = vespalib::datastore::AtomicValueWrapper<double>; + SoftDoomFactor _softDoomFactor; Avg _queryCollateralTime; // TODO: Remove in Vespa 8 Avg _querySetupTime; Avg _queryLatency; @@ -139,7 +141,7 @@ public: MatchingStats & operator = (const MatchingStats &) = delete; MatchingStats(MatchingStats &&) = default; MatchingStats & operator = (MatchingStats &&) = default; - MatchingStats(); + MatchingStats(double prev_soft_doom_factor = INITIAL_SOFT_DOOM_FACTOR); ~MatchingStats(); MatchingStats &queries(size_t value) { _queries = value; return *this; } @@ -165,8 +167,8 @@ public: vespalib::duration doomOvertime() const { return vespalib::from_s(_doomOvertime.max()); } - MatchingStats &softDoomFactor(double value) { _softDoomFactor = value; return *this; } - double softDoomFactor() const { return _softDoomFactor; } + MatchingStats &softDoomFactor(double value) { _softDoomFactor.store_relaxed(value); return *this; } + double softDoomFactor() const { return _softDoomFactor.load_relaxed(); } MatchingStats &updatesoftDoomFactor(vespalib::duration hardLimit, vespalib::duration softLimit, vespalib::duration duration); // TODO: Remove in Vespa 8 diff --git a/searchlib/src/vespa/searchlib/attribute/attributevector.cpp b/searchlib/src/vespa/searchlib/attribute/attributevector.cpp index 1edd5d9fe76..73ff91f135c 100644 --- a/searchlib/src/vespa/searchlib/attribute/attributevector.cpp +++ b/searchlib/src/vespa/searchlib/attribute/attributevector.cpp @@ -528,7 +528,7 @@ AttributeVector::compactLidSpace(uint32_t wantedLidLimit) { } commit(); _committedDocIdLimit.store(wantedLidLimit, std::memory_order_release); - _compactLidSpaceGeneration = _genHandler.getCurrentGeneration(); + _compactLidSpaceGeneration.store(_genHandler.getCurrentGeneration(), std::memory_order_relaxed); incGeneration(); } @@ -536,7 +536,7 @@ AttributeVector::compactLidSpace(uint32_t wantedLidLimit) { bool AttributeVector::canShrinkLidSpace() const { return wantShrinkLidSpace() && - _compactLidSpaceGeneration < getFirstUsedGeneration(); + _compactLidSpaceGeneration.load(std::memory_order_relaxed) < getFirstUsedGeneration(); } diff --git a/searchlib/src/vespa/searchlib/attribute/attributevector.h b/searchlib/src/vespa/searchlib/attribute/attributevector.h index 5f336ab921f..b40f36ad6bd 100644 --- a/searchlib/src/vespa/searchlib/attribute/attributevector.h +++ b/searchlib/src/vespa/searchlib/attribute/attributevector.h @@ -501,7 +501,7 @@ private: std::atomic<uint32_t> _committedDocIdLimit; // docid limit for search uint32_t _uncommittedDocIdLimit; // based on queued changes uint64_t _createSerialNum; - uint64_t _compactLidSpaceGeneration; + std::atomic<uint64_t> _compactLidSpaceGeneration; bool _hasEnum; bool _loaded; bool _isUpdateableInMemoryOnly; diff --git a/searchlib/src/vespa/searchlib/common/packets.h b/searchlib/src/vespa/searchlib/common/packets.h index 07c942a997b..f13cbe24ce4 100644 --- a/searchlib/src/vespa/searchlib/common/packets.h +++ b/searchlib/src/vespa/searchlib/common/packets.h @@ -5,6 +5,7 @@ #include <vespa/vespalib/util/compressionconfig.h> #include <vespa/vespalib/util/memory.h> #include <vector> +#include <atomic> class FNET_DataBuffer; @@ -15,21 +16,21 @@ using vespalib::string; class FS4PersistentPacketStreamer { using CompressionConfig = vespalib::compression::CompressionConfig; - unsigned int _compressionLimit; - unsigned int _compressionLevel; - CompressionConfig::Type _compressionType; + std::atomic<unsigned int> _compressionLimit; + std::atomic<unsigned int> _compressionLevel; + std::atomic<CompressionConfig::Type> _compressionType; public: static FS4PersistentPacketStreamer Instance; FS4PersistentPacketStreamer(); - void SetCompressionLimit(unsigned int limit) { _compressionLimit = limit; } - void SetCompressionLevel(unsigned int level) { _compressionLevel = level; } - void SetCompressionType(CompressionConfig::Type compressionType) { _compressionType = compressionType; } - CompressionConfig::Type getCompressionType() const { return _compressionType; } - uint32_t getCompressionLimit() const { return _compressionLimit; } - uint32_t getCompressionLevel() const { return _compressionLevel; } + void SetCompressionLimit(unsigned int limit) { _compressionLimit.store(limit, std::memory_order_relaxed); } + void SetCompressionLevel(unsigned int level) { _compressionLevel.store(level, std::memory_order_relaxed); } + void SetCompressionType(CompressionConfig::Type compressionType) { _compressionType.store(compressionType, std::memory_order_relaxed); } + CompressionConfig::Type getCompressionType() const { return _compressionType.load(std::memory_order_relaxed); } + uint32_t getCompressionLimit() const { return _compressionLimit.load(std::memory_order_relaxed); } + uint32_t getCompressionLevel() const { return _compressionLevel.load(std::memory_order_relaxed); } }; //========================================================================== diff --git a/searchlib/src/vespa/searchlib/memoryindex/memory_index.h b/searchlib/src/vespa/searchlib/memoryindex/memory_index.h index dc1b5d8060d..4296ba5bd1a 100644 --- a/searchlib/src/vespa/searchlib/memoryindex/memory_index.h +++ b/searchlib/src/vespa/searchlib/memoryindex/memory_index.h @@ -8,6 +8,7 @@ #include <vespa/searchlib/queryeval/searchable.h> #include <vespa/vespalib/stllike/hash_set.h> #include <vespa/vespalib/util/memoryusage.h> +#include <atomic> namespace search::index { class IFieldLengthInspector; @@ -52,7 +53,7 @@ private: std::unique_ptr<DocumentInverterCollection> _inverters; bool _frozen; uint32_t _maxDocId; - uint32_t _numDocs; + std::atomic<uint32_t> _numDocs; mutable std::mutex _lock; std::vector<bool> _hiddenFields; index::Schema::SP _prunedSchema; @@ -70,11 +71,13 @@ private: } } void incNumDocs() { - ++_numDocs; + auto num_docs = _numDocs.load(std::memory_order_relaxed); + _numDocs.store(num_docs + 1, std::memory_order_relaxed); } void decNumDocs() { - if (_numDocs > 0) { - --_numDocs; + auto num_docs = _numDocs.load(std::memory_order_relaxed); + if (num_docs > 0) { + _numDocs.store(num_docs - 1, std::memory_order_relaxed); } } @@ -154,7 +157,7 @@ public: } virtual uint32_t getNumDocs() const { - return _numDocs; + return _numDocs.load(std::memory_order_relaxed); } virtual uint64_t getNumWords() const; diff --git a/vespajlib/src/main/java/com/yahoo/nativec/GLibcVersion.java b/vespajlib/src/main/java/com/yahoo/nativec/GLibcVersion.java index 67ae30c84f5..2dfa4f6d11b 100644 --- a/vespajlib/src/main/java/com/yahoo/nativec/GLibcVersion.java +++ b/vespajlib/src/main/java/com/yahoo/nativec/GLibcVersion.java @@ -1,5 +1,10 @@ package com.yahoo.nativec; +/** + * Gives access to the C library version. + * + * @author baldersheim + */ public class GLibcVersion { private final static Throwable initException = NativeC.loadLibrary(GLibcVersion.class); public static Throwable init() { diff --git a/vespajlib/src/main/java/com/yahoo/nativec/MallInfo.java b/vespajlib/src/main/java/com/yahoo/nativec/MallInfo.java index a4f5486ccf1..eda6c7d1af7 100644 --- a/vespajlib/src/main/java/com/yahoo/nativec/MallInfo.java +++ b/vespajlib/src/main/java/com/yahoo/nativec/MallInfo.java @@ -2,7 +2,12 @@ package com.yahoo.nativec; import com.sun.jna.Structure; -public class MallInfo { +/** + * Gives access to the information provided by the C library mallinfo() function. + * + * @author baldersheim + */ +public class MallInfo extends NativeHeap { private final static Throwable initException = NativeC.loadLibrary(MallInfo.class); public static Throwable init() { return initException; @@ -23,8 +28,27 @@ public class MallInfo { public int keepcost; /* Top-most, releasable space (bytes) */ } private static native MallInfoStruct.ByValue mallinfo(); + + private final MallInfoStruct mallinfo; public MallInfo() { mallinfo = mallinfo(); } - private final MallInfoStruct mallinfo; + + @Override + public long usedSize() { + long v = mallinfo.uordblks; + return v << 20; // Due to too few bits in ancient mallinfo vespamalloc reports in 1M units + } + + @Override + public long totalSize() { + long v = mallinfo.arena; + return v << 20; // Due to too few bits in ancient mallinfo vespamalloc reports in 1M units + } + + @Override + public long availableSize() { + long v = mallinfo.fordblks; + return v << 20; // Due to too few bits in ancient mallinfo vespamalloc reports in 1M units + } } diff --git a/vespajlib/src/main/java/com/yahoo/nativec/MallInfo2.java b/vespajlib/src/main/java/com/yahoo/nativec/MallInfo2.java index 1ae3bc590e2..ea735046843 100644 --- a/vespajlib/src/main/java/com/yahoo/nativec/MallInfo2.java +++ b/vespajlib/src/main/java/com/yahoo/nativec/MallInfo2.java @@ -2,7 +2,12 @@ package com.yahoo.nativec; import com.sun.jna.Structure; -public class MallInfo2 { +/** + * Gives access to the information provided by the C library mallinfo2() function. + * + * @author baldersheim + */ +public class MallInfo2 extends NativeHeap { private final static Throwable initException = NativeC.loadLibrary(MallInfo2.class); public static Throwable init() { return initException; @@ -23,8 +28,24 @@ public class MallInfo2 { public long keepcost; /* Top-most, releasable space (bytes) */ } private static native MallInfo2Struct.ByValue mallinfo2(); + private final MallInfo2Struct mallinfo; + public MallInfo2() { mallinfo = mallinfo2(); } - private final MallInfo2Struct mallinfo; + + @Override + public long usedSize() { + return mallinfo.uordblks; + } + + @Override + public long totalSize() { + return mallinfo.arena; + } + + @Override + public long availableSize() { + return mallinfo.fordblks; + } } diff --git a/vespajlib/src/main/java/com/yahoo/nativec/NativeHeap.java b/vespajlib/src/main/java/com/yahoo/nativec/NativeHeap.java new file mode 100644 index 00000000000..ddff2e33230 --- /dev/null +++ b/vespajlib/src/main/java/com/yahoo/nativec/NativeHeap.java @@ -0,0 +1,24 @@ +package com.yahoo.nativec; + +import com.sun.jna.Platform; + +/** + * Represents the native C heap if accessible + * + * @author baldersheim + */ +public class NativeHeap { + public long usedSize() { return 0; } + public long totalSize() { return 0; } + public long availableSize() { return 0; } + public static NativeHeap sample() { + if (Platform.isLinux()) { + GLibcVersion version = new GLibcVersion(); + if ((version.major() >= 3) || ((version.major() == 2) && (version.minor() >= 33))) { + return new MallInfo2(); + } + return new MallInfo(); + } + return new NativeHeap(); + } +} diff --git a/vespajlib/src/main/java/com/yahoo/nativec/PosixFAdvise.java b/vespajlib/src/main/java/com/yahoo/nativec/PosixFAdvise.java index 3e2c26d2ef2..0fdcbca5f14 100644 --- a/vespajlib/src/main/java/com/yahoo/nativec/PosixFAdvise.java +++ b/vespajlib/src/main/java/com/yahoo/nativec/PosixFAdvise.java @@ -2,6 +2,11 @@ package com.yahoo.nativec; import com.sun.jna.LastErrorException; +/** + * Gives access to the C library posix_fadvise() function. + * + * @author baldersheim + */ public class PosixFAdvise { public static final int POSIX_FADV_DONTNEED = 4; // See /usr/include/linux/fadvise.h private final static Throwable initException = NativeC.loadLibrary(PosixFAdvise.class); diff --git a/vespalib/src/apps/vespa-tsan-digest/tsan_digest.cpp b/vespalib/src/apps/vespa-tsan-digest/tsan_digest.cpp index bebb32ac1ec..6df9cd84c54 100644 --- a/vespalib/src/apps/vespa-tsan-digest/tsan_digest.cpp +++ b/vespalib/src/apps/vespa-tsan-digest/tsan_digest.cpp @@ -7,6 +7,7 @@ #include <cassert> #include <vector> #include <map> +#include <set> #include <memory> #include <algorithm> #include <unistd.h> @@ -18,7 +19,7 @@ constexpr auto npos = vespalib::string::npos; //----------------------------------------------------------------------------- -size_t trace_limit = 7; +size_t trace_limit = 9; //----------------------------------------------------------------------------- @@ -44,6 +45,33 @@ uint64_t get_hash(const std::vector<vespalib::string> &list) { //----------------------------------------------------------------------------- +class FrameHist { +private: + std::map<vespalib::string,size_t> _hist; +public: + void add(const vespalib::string &value, size_t weight) { + _hist[value] += weight; + } + void dump(FILE *dst) { + std::vector<std::pair<vespalib::string,size_t>> entries; + for (const auto &entry: _hist) { + entries.push_back(entry); + } + std::sort(entries.begin(), entries.end(), [](const auto &a, const auto &b){ + if (a.second != b.second) { + return (a.second > b.second); + } + return (a.first < b.first); + }); + fprintf(dst, " top rated frames:\n"); + for (size_t i = 0; i < entries.size() && i < trace_limit; ++i) { + fprintf(dst, "%s -- score: %zu\n", entries[i].first.c_str(), entries[i].second); + } + } +}; + +//----------------------------------------------------------------------------- + class StackTrace { private: vespalib::string _heading; @@ -52,11 +80,17 @@ private: public: StackTrace(const vespalib::string &heading) noexcept : _heading(heading), _frames(), _hash() {} + ~StackTrace() {} void add_frame(const vespalib::string &frame) { _frames.push_back(frame); } void done() { _hash = get_hash(_frames); } uint64_t hash() const { return _hash; } + void update(FrameHist &hist, size_t weight) const { + for (const auto &frame: _frames) { + hist.add(frame, weight); + } + } void dump(FILE *dst) const { fprintf(dst, "%s\n", _heading.c_str()); for (const auto &frame: _frames) { @@ -124,8 +158,9 @@ void dump_delimiter(FILE *dst) { struct Report { using UP = std::unique_ptr<Report>; - virtual vespalib::string make_key() const = 0; - virtual void add(Report::UP report) = 0; + using SP = std::shared_ptr<Report>; + virtual std::vector<vespalib::string> make_keys() const = 0; + virtual void merge(const Report &report) = 0; virtual size_t count() const = 0; virtual void dump(FILE *dst) const = 0; virtual ~Report() {} @@ -134,16 +169,15 @@ struct Report { class RawReport : public Report { private: std::vector<vespalib::string> _lines; + size_t _count; public: RawReport(const std::vector<vespalib::string> &lines) - : _lines(lines) {} - vespalib::string make_key() const override { - return fmt("raw:%zu", get_hash(_lines)); + : _lines(lines), _count(1) {} + std::vector<vespalib::string> make_keys() const override { + return {fmt("raw:%zu", get_hash(_lines))}; } - void add(Report::UP) override { - fprintf(stderr, "WARNING: hash collision for raw report\n"); - } - size_t count() const override { return 1; } + void merge(const Report &) override { ++_count; } + size_t count() const override { return _count; } void dump(FILE *dst) const override { for (const auto &line: _lines) { fprintf(dst, "%s\n", line.c_str()); @@ -153,49 +187,97 @@ public: class RaceReport : public Report { private: - StackTrace _trace1; - StackTrace _trace2; - size_t _total; - size_t _inverted; + struct Node { + StackTrace trace; + size_t count; + }; + std::vector<Node> _nodes; + size_t _count; + + void add(const Node &node) { + for (Node &dst: _nodes) { + if (dst.trace.hash() == node.trace.hash()) { + dst.count += node.count; + return; + } + } + _nodes.push_back(node); + } public: - RaceReport(const StackTrace &trace1, const StackTrace &trace2) - : _trace1(trace1), _trace2(trace2), _total(1), _inverted(0) {} + RaceReport(const StackTrace &a, const StackTrace &b) + : _nodes({{a, 1}, {b, 1}}), _count(1) {} - vespalib::string make_key() const override { - if (_trace2.hash() < _trace1.hash()) { - return fmt("race:%zu,%zu", _trace2.hash(), _trace1.hash()); + std::vector<vespalib::string> make_keys() const override { + std::vector<vespalib::string> result; + for (const auto &node: _nodes) { + result.push_back(fmt("race:%zu", node.trace.hash())); } - return fmt("race:%zu,%zu", _trace1.hash(), _trace2.hash()); + return result; } - void add(Report::UP report) override { + void merge(const Report &report) override { // should have correct type due to key prefix - const RaceReport &rhs = dynamic_cast<RaceReport&>(*report); - ++_total; - if (_trace1.hash() != rhs._trace1.hash()) { - ++_inverted; + const auto &rhs = dynamic_cast<const RaceReport &>(report); + _count += rhs._count; + for (const auto &node: rhs._nodes) { + add(node); } } - size_t count() const override { return _total; } + size_t count() const override { return _count; } void dump(FILE *dst) const override { - fprintf(dst, "WARNING: ThreadSanitizer: data race\n"); - _trace1.dump(dst); - _trace2.dump(dst); - fprintf(dst, "INFO: total: %zu (inverted: %zu)\n", _total, _inverted); + std::vector<const Node *> list; + for (const auto &node: _nodes) { + list.push_back(&node); + } + std::sort(list.begin(), list.end(), + [](const auto *a, const auto *b) { + return (a->count > b->count); + }); + fprintf(dst, "WARNING: data race cluster with %zu conflicts between %zu traces\n", _count, list.size()); + FrameHist frame_hist; + for (const auto *node: list) { + node->trace.update(frame_hist, node->count); + node->trace.dump(dst); + } + frame_hist.dump(dst); } }; //----------------------------------------------------------------------------- +using ReportMap = std::map<vespalib::string,Report::SP>; +using MapPos = ReportMap::const_iterator; + size_t total_reports = 0; -std::map<vespalib::string,Report::UP> reports; +ReportMap report_map; +FrameHist race_frame_hist; void handle_report(std::unique_ptr<Report> report) { ++total_reports; - auto [pos, first] = reports.try_emplace(report->make_key(), std::move(report)); - if (!first) { - assert(report && "should still be valid"); - pos->second->add(std::move(report)); + auto keys = report->make_keys(); + std::vector<Report::SP> found; + for (const auto &key: keys) { + auto pos = report_map.find(key); + if (pos != report_map.end()) { + found.push_back(pos->second); + } + } + if (found.empty()) { + Report::SP my_report = std::move(report); + for (const auto &key: keys) { + report_map[key] = my_report; + } + } else { + for (size_t i = 1; i < found.size(); ++i) { + if (found[0].get() != found[i].get()) { + found[0]->merge(*found[i]); + } + } + found[0]->merge(*report); + keys = found[0]->make_keys(); + for (const auto &key: keys) { + report_map[key] = found[0]; + } } } @@ -204,6 +286,8 @@ void make_report(const std::vector<vespalib::string> &lines) { if (type == ReportType::RACE) { auto traces = extract_traces(lines, 2); if (traces.size() == 2) { + traces[0].update(race_frame_hist, 1); + traces[1].update(race_frame_hist, 1); return handle_report(std::make_unique<RaceReport>(traces[0], traces[1])); } } @@ -249,13 +333,15 @@ void read_input() { } void write_output() { - std::vector<Report*> list; - list.reserve(reports.size()); - for (const auto &[key, value]: reports) { - list.push_back(value.get()); + std::set<const Report *> seen; + std::vector<const Report *> list; + for (const auto &[key, value]: report_map) { + if (seen.insert(value.get()).second) { + list.push_back(value.get()); + } } std::sort(list.begin(), list.end(), - [](const auto &a, const auto &b) { + [](const auto *a, const auto *b) { return (a->count() > b->count()); }); for (const auto *report: list) { @@ -263,7 +349,8 @@ void write_output() { report->dump(stdout); dump_delimiter(stdout); } - fprintf(stderr, "%zu reports in, %zu reports out\n", total_reports, reports.size()); + fprintf(stderr, "%zu reports in, %zu reports out\n", total_reports, list.size()); + race_frame_hist.dump(stderr); } int main(int, char **) { diff --git a/vespalib/src/tests/datastore/buffer_type/buffer_type_test.cpp b/vespalib/src/tests/datastore/buffer_type/buffer_type_test.cpp index a31dfffc25b..6988e41add1 100644 --- a/vespalib/src/tests/datastore/buffer_type/buffer_type_test.cpp +++ b/vespalib/src/tests/datastore/buffer_type/buffer_type_test.cpp @@ -12,9 +12,9 @@ constexpr uint32_t NUM_ARRAYS_FOR_NEW_BUFFER(0); struct Setup { uint32_t _minArrays; - ElemCount _usedElems; + std::atomic<ElemCount> _usedElems; ElemCount _neededElems; - ElemCount _deadElems; + std::atomic<ElemCount> _deadElems; uint32_t _bufferId; float _allocGrowFactor; bool _resizing; @@ -27,6 +27,7 @@ struct Setup { _allocGrowFactor(0.5), _resizing(false) {} + Setup(const Setup& rhs); Setup &minArrays(uint32_t value) { _minArrays = value; return *this; } Setup &used(size_t value) { _usedElems = value; return *this; } Setup &needed(size_t value) { _neededElems = value; return *this; } @@ -35,6 +36,17 @@ struct Setup { Setup &resizing(bool value) { _resizing = value; return *this; } }; +Setup::Setup(const Setup& rhs) + : _minArrays(rhs._minArrays), + _usedElems(rhs._usedElems.load(std::memory_order_relaxed)), + _neededElems(rhs._neededElems), + _deadElems(rhs._deadElems.load(std::memory_order_relaxed)), + _bufferId(rhs._bufferId), + _allocGrowFactor(rhs._allocGrowFactor), + _resizing(rhs._resizing) +{ +} + struct Fixture { std::vector<Setup> setups; IntBufferType bufferType; diff --git a/vespalib/src/tests/net/tls/openssl_impl/openssl_impl_test.cpp b/vespalib/src/tests/net/tls/openssl_impl/openssl_impl_test.cpp index 45e9c92343e..e20cd30c597 100644 --- a/vespalib/src/tests/net/tls/openssl_impl/openssl_impl_test.cpp +++ b/vespalib/src/tests/net/tls/openssl_impl/openssl_impl_test.cpp @@ -535,40 +535,40 @@ struct CertFixture : Fixture { CertFixture::~CertFixture() = default; struct PrintingCertificateCallback : CertificateVerificationCallback { - bool verify(const PeerCredentials& peer_creds) const override { + AuthorizationResult verify(const PeerCredentials& peer_creds) const override { if (!peer_creds.common_name.empty()) { fprintf(stderr, "Got a CN: %s\n", peer_creds.common_name.c_str()); } for (auto& dns : peer_creds.dns_sans) { fprintf(stderr, "Got a DNS SAN entry: %s\n", dns.c_str()); } - return true; + return AuthorizationResult::make_authorized_for_all_roles(); } }; // Single-use mock verifier struct MockCertificateCallback : CertificateVerificationCallback { mutable PeerCredentials creds; // only used in single thread testing context - bool verify(const PeerCredentials& peer_creds) const override { + AuthorizationResult verify(const PeerCredentials& peer_creds) const override { creds = peer_creds; - return true; + return AuthorizationResult::make_authorized_for_all_roles(); } }; struct AlwaysFailVerifyCallback : CertificateVerificationCallback { - bool verify([[maybe_unused]] const PeerCredentials& peer_creds) const override { + AuthorizationResult verify([[maybe_unused]] const PeerCredentials& peer_creds) const override { fprintf(stderr, "Rejecting certificate, none shall pass!\n"); - return false; + return AuthorizationResult::make_not_authorized(); } }; struct ExceptionThrowingCallback : CertificateVerificationCallback { - bool verify([[maybe_unused]] const PeerCredentials& peer_creds) const override { + AuthorizationResult verify([[maybe_unused]] const PeerCredentials& peer_creds) const override { throw std::runtime_error("oh no what is going on"); } }; -TEST_F("Certificate verification callback returning false breaks handshake", CertFixture) { +TEST_F("Certificate verification callback returning unauthorized breaks handshake", CertFixture) { auto ck = f.create_ca_issued_peer_cert({"hello.world.example.com"}, {}); f.reset_client_with_cert_opts(ck, std::make_shared<PrintingCertificateCallback>()); @@ -602,8 +602,40 @@ TEST_F("Certificate verification callback observes CN, DNS SANs and URI SANs", C ASSERT_EQUAL(2u, creds.dns_sans.size()); EXPECT_EQUAL("crash.wile.example.com", creds.dns_sans[0]); EXPECT_EQUAL("burn.wile.example.com", creds.dns_sans[1]); - ASSERT_EQUAL(1u, server_cb->creds.uri_sans.size()); - EXPECT_EQUAL("foo://bar.baz/zoid", server_cb->creds.uri_sans[0]); + ASSERT_EQUAL(1u, creds.uri_sans.size()); + EXPECT_EQUAL("foo://bar.baz/zoid", creds.uri_sans[0]); +} + +TEST_F("Peer credentials are propagated to CryptoCodec", CertFixture) { + auto cli_cert = f.create_ca_issued_peer_cert( + {{"rockets.wile.example.com"}}, + {{"DNS:crash.wile.example.com"}, {"DNS:burn.wile.example.com"}, + {"URI:foo://bar.baz/zoid"}}); + auto serv_cert = f.create_ca_issued_peer_cert( + {{"birdseed.roadrunner.example.com"}}, + {{"DNS:fake.tunnel.example.com"}}); + f.reset_client_with_cert_opts(cli_cert, std::make_shared<PrintingCertificateCallback>()); + auto server_cb = std::make_shared<MockCertificateCallback>(); + f.reset_server_with_cert_opts(serv_cert, server_cb); + ASSERT_TRUE(f.handshake()); + + auto& client_creds = f.server->peer_credentials(); + auto& server_creds = f.client->peer_credentials(); + + fprintf(stderr, "Client credentials (observed by server): %s\n", to_string(client_creds).c_str()); + fprintf(stderr, "Server credentials (observed by client): %s\n", to_string(server_creds).c_str()); + + EXPECT_EQUAL("rockets.wile.example.com", client_creds.common_name); + ASSERT_EQUAL(2u, client_creds.dns_sans.size()); + EXPECT_EQUAL("crash.wile.example.com", client_creds.dns_sans[0]); + EXPECT_EQUAL("burn.wile.example.com", client_creds.dns_sans[1]); + ASSERT_EQUAL(1u, client_creds.uri_sans.size()); + EXPECT_EQUAL("foo://bar.baz/zoid", client_creds.uri_sans[0]); + + EXPECT_EQUAL("birdseed.roadrunner.example.com", server_creds.common_name); + ASSERT_EQUAL(1u, server_creds.dns_sans.size()); + EXPECT_EQUAL("fake.tunnel.example.com", server_creds.dns_sans[0]); + ASSERT_EQUAL(0u, server_creds.uri_sans.size()); } TEST_F("Last occurring CN is given to verification callback if multiple CNs are present", CertFixture) { diff --git a/vespalib/src/tests/net/tls/policy_checking_certificate_verifier/policy_checking_certificate_verifier_test.cpp b/vespalib/src/tests/net/tls/policy_checking_certificate_verifier/policy_checking_certificate_verifier_test.cpp index 812d06868fd..8c9e50f17b4 100644 --- a/vespalib/src/tests/net/tls/policy_checking_certificate_verifier/policy_checking_certificate_verifier_test.cpp +++ b/vespalib/src/tests/net/tls/policy_checking_certificate_verifier/policy_checking_certificate_verifier_test.cpp @@ -124,7 +124,12 @@ PeerCredentials creds_with_cn(vespalib::stringref cn) { bool verify(AuthorizedPeers authorized_peers, const PeerCredentials& peer_creds) { auto verifier = create_verify_callback_from(std::move(authorized_peers)); - return verifier->verify(peer_creds); + return verifier->verify(peer_creds).success(); +} + +AssumedRoles verify_roles(AuthorizedPeers authorized_peers, const PeerCredentials& peer_creds) { + auto verifier = create_verify_callback_from(std::move(authorized_peers)); + return verifier->verify(peer_creds).steal_assumed_roles(); } TEST("Default-constructed AuthorizedPeers does not allow all authenticated peers") { @@ -137,6 +142,16 @@ TEST("Specially constructed set of policies allows all authenticated peers") { EXPECT_TRUE(verify(allow_all, creds_with_dns_sans({{"anything.goes"}}))); } +TEST("specially constructed set of policies returns wildcard role set") { + auto allow_all = AuthorizedPeers::allow_all_authenticated(); + EXPECT_EQUAL(verify_roles(allow_all, creds_with_dns_sans({{"anything.goes"}})), AssumedRoles::make_wildcard_role()); +} + +TEST("policy without explicit role set implicitly returns wildcard role set") { + auto authorized = authorized_peers({policy_with({required_san_dns("yolo.swag")})}); + EXPECT_EQUAL(verify_roles(authorized, creds_with_dns_sans({{"yolo.swag"}})), AssumedRoles::make_wildcard_role()); +} + TEST("Non-empty policies do not allow all authenticated peers") { auto allow_not_all = authorized_peers({policy_with({required_san_dns("hello.world")})}); EXPECT_FALSE(allow_not_all.allows_all_authenticated()); @@ -231,10 +246,11 @@ struct MultiPolicyMatchFixture { }; MultiPolicyMatchFixture::MultiPolicyMatchFixture() - : authorized(authorized_peers({policy_with({required_san_dns("hello.world")}), - policy_with({required_san_dns("foo.bar")}), - policy_with({required_san_dns("zoid.berg")}), - policy_with({required_san_uri("zoid://be.rg/")})})) + : authorized(authorized_peers({policy_with({required_san_dns("hello.world")}, assumed_roles({"r1"})), + policy_with({required_san_dns("foo.bar")}, assumed_roles({"r2"})), + policy_with({required_san_dns("zoid.berg")}, assumed_roles({"r2", "r3"})), + policy_with({required_san_dns("secret.sauce")}, AssumedRoles::make_wildcard_role()), + policy_with({required_san_uri("zoid://be.rg/")}, assumed_roles({"r4"}))})) {} MultiPolicyMatchFixture::~MultiPolicyMatchFixture() = default; @@ -246,14 +262,34 @@ TEST_F("peer verifies if it matches at least 1 policy of multiple", MultiPolicyM EXPECT_TRUE(verify(f.authorized, creds_with_uri_sans({{"zoid://be.rg/"}}))); } +TEST_F("role set is returned for single matched policy", MultiPolicyMatchFixture) { + EXPECT_EQUAL(verify_roles(f.authorized, creds_with_dns_sans({{"hello.world"}})), assumed_roles({"r1"})); + EXPECT_EQUAL(verify_roles(f.authorized, creds_with_dns_sans({{"foo.bar"}})), assumed_roles({"r2"})); + EXPECT_EQUAL(verify_roles(f.authorized, creds_with_dns_sans({{"zoid.berg"}})), assumed_roles({"r2", "r3"})); + EXPECT_EQUAL(verify_roles(f.authorized, creds_with_dns_sans({{"secret.sauce"}})), AssumedRoles::make_wildcard_role()); + EXPECT_EQUAL(verify_roles(f.authorized, creds_with_uri_sans({{"zoid://be.rg/"}})), assumed_roles({"r4"})); +} + TEST_F("peer verifies if it matches multiple policies", MultiPolicyMatchFixture) { EXPECT_TRUE(verify(f.authorized, creds_with_dns_sans({{"hello.world"}, {"zoid.berg"}}))); } +TEST_F("union role set is returned if multiple policies match", MultiPolicyMatchFixture) { + EXPECT_EQUAL(verify_roles(f.authorized, creds_with_dns_sans({{"hello.world"}, {"foo.bar"}, {"zoid.berg"}})), + assumed_roles({"r1", "r2", "r3"})); + // Wildcard role is tracked as a distinct role string + EXPECT_EQUAL(verify_roles(f.authorized, creds_with_dns_sans({{"hello.world"}, {"foo.bar"}, {"secret.sauce"}})), + assumed_roles({"r1", "r2", "*"})); +} + TEST_F("peer must match at least 1 of multiple policies", MultiPolicyMatchFixture) { EXPECT_FALSE(verify(f.authorized, creds_with_dns_sans({{"does.not.exist"}}))); } +TEST_F("empty role set is returned if no policies match", MultiPolicyMatchFixture) { + EXPECT_EQUAL(verify_roles(f.authorized, creds_with_dns_sans({{"does.not.exist"}})), AssumedRoles::make_empty()); +} + TEST("CN requirement without glob pattern is matched as exact string") { auto authorized = authorized_peers({policy_with({required_cn("hello.world")})}); EXPECT_TRUE(verify(authorized, creds_with_cn("hello.world"))); @@ -272,6 +308,64 @@ TEST("CN requirement can include glob wildcards") { EXPECT_FALSE(verify(authorized, creds_with_cn("world"))); } +TEST("AssumedRoles by default contains no roles") { + AssumedRoles roles; + EXPECT_TRUE(roles.empty()); + EXPECT_FALSE(roles.can_assume_role("foo")); + auto empty = AssumedRoles::make_empty(); + EXPECT_EQUAL(roles, empty); +} + +TEST("AssumedRoles can be constructed with an explicit set of roles") { + auto roles = AssumedRoles::make_for_roles({"foo", "bar"}); + EXPECT_TRUE(roles.can_assume_role("foo")); + EXPECT_TRUE(roles.can_assume_role("bar")); + EXPECT_FALSE(roles.can_assume_role("baz")); +} + +TEST("AssumedRoles wildcard role can assume any role") { + auto roles = AssumedRoles::make_wildcard_role(); + EXPECT_TRUE(roles.can_assume_role("foo")); + EXPECT_TRUE(roles.can_assume_role("bar")); +} + +TEST("AssumedRolesBuilder builds union set of added roles") { + AssumedRolesBuilder builder; + builder.add_union(AssumedRoles::make_for_roles({"hello", "world"})); + builder.add_union(AssumedRoles::make_for_roles({"hello", "moon"})); + builder.add_union(AssumedRoles::make_for_roles({"goodbye", "moon"})); + auto roles = builder.build_with_move(); + EXPECT_EQUAL(roles, AssumedRoles::make_for_roles({"hello", "goodbye", "moon", "world"})); +} + +TEST("AuthorizationResult is not authorized by default") { + AuthorizationResult result; + EXPECT_FALSE(result.success()); + EXPECT_TRUE(result.assumed_roles().empty()); +} + +TEST("AuthorizationResult can be explicitly created as not authorzed") { + auto result = AuthorizationResult::make_not_authorized(); + EXPECT_FALSE(result.success()); + EXPECT_TRUE(result.assumed_roles().empty()); +} + +TEST("AuthorizationResult can be pre-authorized for all roles") { + auto result = AuthorizationResult::make_authorized_for_all_roles(); + EXPECT_TRUE(result.success()); + EXPECT_FALSE(result.assumed_roles().empty()); + EXPECT_TRUE(result.assumed_roles().can_assume_role("foo")); +} + +TEST("AuthorizationResult can be pre-authorized for an explicit set of roles") { + auto result = AuthorizationResult::make_authorized_for_roles(AssumedRoles::make_for_roles({"elden", "ring"})); + EXPECT_TRUE(result.success()); + EXPECT_FALSE(result.assumed_roles().empty()); + EXPECT_TRUE(result.assumed_roles().can_assume_role("elden")); + EXPECT_TRUE(result.assumed_roles().can_assume_role("ring")); + EXPECT_FALSE(result.assumed_roles().can_assume_role("O you don't have the right")); +} + // TODO test CN _and_ SAN TEST_MAIN() { TEST_RUN_ALL(); } diff --git a/vespalib/src/vespa/vespalib/datastore/buffer_type.cpp b/vespalib/src/vespa/vespalib/datastore/buffer_type.cpp index ca908d48210..dec24d6fe46 100644 --- a/vespalib/src/vespa/vespalib/datastore/buffer_type.cpp +++ b/vespalib/src/vespa/vespalib/datastore/buffer_type.cpp @@ -17,10 +17,12 @@ constexpr float DEFAULT_ALLOC_GROW_FACTOR = 0.2; void BufferTypeBase::CleanContext::extraBytesCleaned(size_t value) { - assert(_extraUsedBytes >= value); - assert(_extraHoldBytes >= value); - _extraUsedBytes -= value; - _extraHoldBytes -= value; + size_t extra_used_bytes = _extraUsedBytes.load(std::memory_order_relaxed); + size_t extra_hold_bytes = _extraHoldBytes.load(std::memory_order_relaxed); + assert(extra_used_bytes >= value); + assert(extra_hold_bytes >= value); + _extraUsedBytes.store(extra_used_bytes - value, std::memory_order_relaxed); + _extraHoldBytes.store(extra_hold_bytes - value, std::memory_order_relaxed); } BufferTypeBase::BufferTypeBase(uint32_t arraySize, @@ -62,7 +64,7 @@ BufferTypeBase::getReservedElements(uint32_t bufferId) const } void -BufferTypeBase::onActive(uint32_t bufferId, ElemCount* usedElems, ElemCount* deadElems, void* buffer) +BufferTypeBase::onActive(uint32_t bufferId, std::atomic<ElemCount>* usedElems, std::atomic<ElemCount>* deadElems, void* buffer) { _aggr_counts.add_buffer(usedElems, deadElems); assert(std::find(_active_buffers.begin(), _active_buffers.end(), bufferId) == _active_buffers.end()); @@ -76,7 +78,7 @@ BufferTypeBase::onActive(uint32_t bufferId, ElemCount* usedElems, ElemCount* dea } void -BufferTypeBase::onHold(uint32_t buffer_id, const ElemCount* usedElems, const ElemCount* deadElems) +BufferTypeBase::onHold(uint32_t buffer_id, const std::atomic<ElemCount>* usedElems, const std::atomic<ElemCount>* deadElems) { ++_holdBuffers; auto itr = std::find(_active_buffers.begin(), _active_buffers.end(), buffer_id); @@ -95,7 +97,7 @@ BufferTypeBase::onFree(ElemCount usedElems) } void -BufferTypeBase::resume_primary_buffer(uint32_t buffer_id, ElemCount* used_elems, ElemCount* dead_elems) +BufferTypeBase::resume_primary_buffer(uint32_t buffer_id, std::atomic<ElemCount>* used_elems, std::atomic<ElemCount>* dead_elems) { auto itr = std::find(_active_buffers.begin(), _active_buffers.end(), buffer_id); assert(itr != _active_buffers.end()); @@ -174,7 +176,7 @@ BufferTypeBase::AggregatedBufferCounts::AggregatedBufferCounts() } void -BufferTypeBase::AggregatedBufferCounts::add_buffer(const ElemCount* used_elems, const ElemCount* dead_elems) +BufferTypeBase::AggregatedBufferCounts::add_buffer(const std::atomic<ElemCount>* used_elems, const std::atomic<ElemCount>* dead_elems) { for (const auto& elem : _counts) { assert(elem.used_ptr != used_elems); @@ -184,7 +186,7 @@ BufferTypeBase::AggregatedBufferCounts::add_buffer(const ElemCount* used_elems, } void -BufferTypeBase::AggregatedBufferCounts::remove_buffer(const ElemCount* used_elems, const ElemCount* dead_elems) +BufferTypeBase::AggregatedBufferCounts::remove_buffer(const std::atomic<ElemCount>* used_elems, const std::atomic<ElemCount>* dead_elems) { auto itr = std::find_if(_counts.begin(), _counts.end(), [=](const auto& elem){ return elem.used_ptr == used_elems; }); @@ -199,8 +201,8 @@ BufferTypeBase::AggregatedBufferCounts::last_buffer() const BufferCounts result; assert(!_counts.empty()); const auto& last = _counts.back(); - result.used_elems += *last.used_ptr; - result.dead_elems += *last.dead_ptr; + result.used_elems += last.used_ptr->load(std::memory_order_relaxed); + result.dead_elems += last.dead_ptr->load(std::memory_order_relaxed); return result; } @@ -209,8 +211,8 @@ BufferTypeBase::AggregatedBufferCounts::all_buffers() const { BufferCounts result; for (const auto& elem : _counts) { - result.used_elems += *elem.used_ptr; - result.dead_elems += *elem.dead_ptr; + result.used_elems += elem.used_ptr->load(std::memory_order_relaxed); + result.dead_elems += elem.dead_ptr->load(std::memory_order_relaxed); } return result; } diff --git a/vespalib/src/vespa/vespalib/datastore/buffer_type.h b/vespalib/src/vespa/vespalib/datastore/buffer_type.h index 3394af18b04..53436df432f 100644 --- a/vespalib/src/vespa/vespalib/datastore/buffer_type.h +++ b/vespalib/src/vespa/vespalib/datastore/buffer_type.h @@ -25,10 +25,10 @@ public: using ElemCount = vespalib::datastore::ElemCount; class CleanContext { private: - size_t &_extraUsedBytes; - size_t &_extraHoldBytes; + std::atomic<size_t> &_extraUsedBytes; + std::atomic<size_t> &_extraHoldBytes; public: - CleanContext(size_t &extraUsedBytes, size_t &extraHoldBytes) + CleanContext(std::atomic<size_t>& extraUsedBytes, std::atomic<size_t>& extraHoldBytes) : _extraUsedBytes(extraUsedBytes), _extraHoldBytes(extraHoldBytes) {} @@ -60,10 +60,10 @@ public: virtual size_t elementSize() const = 0; virtual void cleanHold(void *buffer, size_t offset, ElemCount numElems, CleanContext cleanCtx) = 0; size_t getArraySize() const { return _arraySize; } - virtual void onActive(uint32_t bufferId, ElemCount* usedElems, ElemCount* deadElems, void* buffer); - void onHold(uint32_t buffer_id, const ElemCount* usedElems, const ElemCount* deadElems); + virtual void onActive(uint32_t bufferId, std::atomic<ElemCount>* usedElems, std::atomic<ElemCount>* deadElems, void* buffer); + void onHold(uint32_t buffer_id, const std::atomic<ElemCount>* usedElems, const std::atomic<ElemCount>* deadElems); virtual void onFree(ElemCount usedElems); - void resume_primary_buffer(uint32_t buffer_id, ElemCount* used_elems, ElemCount* dead_elems); + void resume_primary_buffer(uint32_t buffer_id, std::atomic<ElemCount>* used_elems, std::atomic<ElemCount>* dead_elems); virtual const alloc::MemoryAllocator* get_memory_allocator() const; /** @@ -95,10 +95,10 @@ protected: class AggregatedBufferCounts { private: struct Element { - const ElemCount* used_ptr; - const ElemCount* dead_ptr; + const std::atomic<ElemCount>* used_ptr; + const std::atomic<ElemCount>* dead_ptr; Element() noexcept : used_ptr(nullptr), dead_ptr(nullptr) {} - Element(const ElemCount* used_ptr_in, const ElemCount* dead_ptr_in) noexcept + Element(const std::atomic<ElemCount>* used_ptr_in, const std::atomic<ElemCount>* dead_ptr_in) noexcept : used_ptr(used_ptr_in), dead_ptr(dead_ptr_in) {} }; @@ -106,8 +106,8 @@ protected: public: AggregatedBufferCounts(); - void add_buffer(const ElemCount* used_elems, const ElemCount* dead_elems); - void remove_buffer(const ElemCount* used_elems, const ElemCount* dead_elems); + void add_buffer(const std::atomic<ElemCount>* used_elems, const std::atomic<ElemCount>* dead_elems); + void remove_buffer(const std::atomic<ElemCount>* used_elems, const std::atomic<ElemCount>* dead_elems); BufferCounts last_buffer() const; BufferCounts all_buffers() const; bool empty() const { return _counts.empty(); } diff --git a/vespalib/src/vespa/vespalib/datastore/bufferstate.cpp b/vespalib/src/vespa/vespalib/datastore/bufferstate.cpp index e0104c8bb71..cde891b477d 100644 --- a/vespalib/src/vespa/vespalib/datastore/bufferstate.cpp +++ b/vespalib/src/vespa/vespalib/datastore/bufferstate.cpp @@ -30,7 +30,7 @@ BufferState::BufferState() _buffer(Alloc::alloc(0, MemoryAllocator::HUGEPAGE_SIZE)), _arraySize(0), _typeId(0), - _state(FREE), + _state(State::FREE), _disableElemHoldList(false), _compacting(false) { @@ -38,7 +38,7 @@ BufferState::BufferState() BufferState::~BufferState() { - assert(_state == FREE); + assert(getState() == State::FREE); assert(_freeListList == nullptr); assert(_nextHasFree == nullptr); assert(_prevHasFree == nullptr); @@ -48,8 +48,9 @@ BufferState::~BufferState() void BufferState::decHoldElems(size_t value) { - assert(_holdElems >= value); - _holdElems -= value; + ElemCount hold_elems = getHoldElems(); + assert(hold_elems >= value); + _holdElems.store(hold_elems - value, std::memory_order_relaxed); } namespace { @@ -105,14 +106,14 @@ BufferState::onActive(uint32_t bufferId, uint32_t typeId, { assert(buffer.load(std::memory_order_relaxed) == nullptr); assert(_buffer.get() == nullptr); - assert(_state == FREE); + assert(getState() == State::FREE); assert(_typeHandler == nullptr); - assert(_allocElems == 0); - assert(_usedElems == 0); - assert(_deadElems == 0u); - assert(_holdElems == 0); - assert(_extraUsedBytes == 0); - assert(_extraHoldBytes == 0); + assert(capacity() == 0); + assert(size() == 0); + assert(getDeadElems() == 0u); + assert(getHoldElems() == 0); + assert(getExtraUsedBytes() == 0); + assert(getExtraHoldBytes() == 0); assert(isFreeListEmpty()); assert(_nextHasFree == nullptr); assert(_prevHasFree == nullptr); @@ -127,12 +128,12 @@ BufferState::onActive(uint32_t bufferId, uint32_t typeId, _buffer.create(alloc.bytes).swap(_buffer); assert(_buffer.get() != nullptr || alloc.elements == 0u); buffer.store(_buffer.get(), std::memory_order_release); - _allocElems = alloc.elements; - _state = ACTIVE; - _typeHandler = typeHandler; + _allocElems.store(alloc.elements, std::memory_order_relaxed); + _typeHandler.store(typeHandler, std::memory_order_release); assert(typeId <= std::numeric_limits<uint16_t>::max()); _typeId = typeId; - _arraySize = _typeHandler->getArraySize(); + _arraySize = typeHandler->getArraySize(); + _state.store(State::ACTIVE, std::memory_order_release); typeHandler->onActive(bufferId, &_usedElems, &_deadElems, buffer.load(std::memory_order::relaxed)); } @@ -140,15 +141,15 @@ BufferState::onActive(uint32_t bufferId, uint32_t typeId, void BufferState::onHold(uint32_t buffer_id) { - assert(_state == ACTIVE); - assert(_typeHandler != nullptr); - _state = HOLD; + assert(getState() == State::ACTIVE); + assert(getTypeHandler() != nullptr); + _state.store(State::HOLD, std::memory_order_release); _compacting = false; - assert(_deadElems <= _usedElems); - assert(_holdElems <= (_usedElems - _deadElems)); - _deadElems = 0; - _holdElems = _usedElems; // Put everyting on hold - _typeHandler->onHold(buffer_id, &_usedElems, &_deadElems); + assert(getDeadElems() <= size()); + assert(getHoldElems() <= (size() - getDeadElems())); + _deadElems.store(0, std::memory_order_relaxed); + _holdElems.store(size(), std::memory_order_relaxed); // Put everyting on hold + getTypeHandler()->onHold(buffer_id, &_usedElems, &_deadElems); if ( ! isFreeListEmpty()) { removeFromFreeListList(); FreeList().swap(_freeList); @@ -164,21 +165,21 @@ void BufferState::onFree(std::atomic<void*>& buffer) { assert(buffer.load(std::memory_order_relaxed) == _buffer.get()); - assert(_state == HOLD); + assert(getState() == State::HOLD); assert(_typeHandler != nullptr); - assert(_deadElems <= _usedElems); - assert(_holdElems == _usedElems - _deadElems); - _typeHandler->destroyElements(buffer, _usedElems); + assert(getDeadElems() <= size()); + assert(getHoldElems() == size() - getDeadElems()); + getTypeHandler()->destroyElements(buffer, size()); Alloc::alloc().swap(_buffer); - _typeHandler->onFree(_usedElems); + getTypeHandler()->onFree(size()); buffer.store(nullptr, std::memory_order_release); - _usedElems = 0; - _allocElems = 0; - _deadElems = 0u; - _holdElems = 0u; - _extraUsedBytes = 0; - _extraHoldBytes = 0; - _state = FREE; + _usedElems.store(0, std::memory_order_relaxed); + _allocElems.store(0, std::memory_order_relaxed); + _deadElems.store(0, std::memory_order_relaxed); + _holdElems.store(0, std::memory_order_relaxed); + _extraUsedBytes.store(0, std::memory_order_relaxed); + _extraHoldBytes.store(0, std::memory_order_relaxed); + _state.store(State::FREE, std::memory_order_release); _typeHandler = nullptr; _arraySize = 0; assert(isFreeListEmpty()); @@ -193,18 +194,18 @@ BufferState::onFree(std::atomic<void*>& buffer) void BufferState::dropBuffer(uint32_t buffer_id, std::atomic<void*>& buffer) { - if (_state == FREE) { + if (getState() == State::FREE) { assert(buffer.load(std::memory_order_relaxed) == nullptr); return; } - assert(buffer.load(std::memory_order_relaxed) != nullptr || _allocElems == 0); - if (_state == ACTIVE) { + assert(buffer.load(std::memory_order_relaxed) != nullptr || capacity() == 0); + if (getState() == State::ACTIVE) { onHold(buffer_id); } - if (_state == HOLD) { + if (getState() == State::HOLD) { onFree(buffer); } - assert(_state == FREE); + assert(getState() == State::FREE); assert(buffer.load(std::memory_order_relaxed) == nullptr); } @@ -212,7 +213,7 @@ BufferState::dropBuffer(uint32_t buffer_id, std::atomic<void*>& buffer) void BufferState::setFreeListList(FreeListList *freeListList) { - if (_state == FREE && freeListList != nullptr) { + if (getState() == State::FREE && freeListList != nullptr) { return; } if (freeListList == _freeListList) { @@ -285,25 +286,25 @@ BufferState::fallbackResize(uint32_t bufferId, std::atomic<void*>& buffer, Alloc &holdBuffer) { - assert(_state == ACTIVE); + assert(getState() == State::ACTIVE); assert(_typeHandler != nullptr); assert(holdBuffer.get() == nullptr); AllocResult alloc = calcAllocation(bufferId, *_typeHandler, elementsNeeded, true); - assert(alloc.elements >= _usedElems + elementsNeeded); - assert(alloc.elements > _allocElems); + assert(alloc.elements >= size() + elementsNeeded); + assert(alloc.elements > capacity()); Alloc newBuffer = _buffer.create(alloc.bytes); - _typeHandler->fallbackCopy(newBuffer.get(), buffer.load(std::memory_order_relaxed), _usedElems); + getTypeHandler()->fallbackCopy(newBuffer.get(), buffer.load(std::memory_order_relaxed), size()); holdBuffer.swap(_buffer); std::atomic_thread_fence(std::memory_order_release); _buffer = std::move(newBuffer); buffer.store(_buffer.get(), std::memory_order_release); - _allocElems = alloc.elements; + _allocElems.store(alloc.elements, std::memory_order_relaxed); } void BufferState::resume_primary_buffer(uint32_t buffer_id) { - _typeHandler->resume_primary_buffer(buffer_id, &_usedElems, &_deadElems); + getTypeHandler()->resume_primary_buffer(buffer_id, &_usedElems, &_deadElems); } } diff --git a/vespalib/src/vespa/vespalib/datastore/bufferstate.h b/vespalib/src/vespa/vespalib/datastore/bufferstate.h index 7862f58cfe1..639f4f9dfc6 100644 --- a/vespalib/src/vespa/vespalib/datastore/bufferstate.h +++ b/vespalib/src/vespa/vespalib/datastore/bufferstate.h @@ -41,23 +41,23 @@ public: using FreeList = vespalib::Array<EntryRef>; - enum State : uint8_t { + enum class State : uint8_t { FREE, ACTIVE, HOLD }; private: - ElemCount _usedElems; - ElemCount _allocElems; - ElemCount _deadElems; - ElemCount _holdElems; + std::atomic<ElemCount> _usedElems; + std::atomic<ElemCount> _allocElems; + std::atomic<ElemCount> _deadElems; + std::atomic<ElemCount> _holdElems; // Number of bytes that are heap allocated by elements that are stored in this buffer. // For simple types this is 0. - size_t _extraUsedBytes; + std::atomic<size_t> _extraUsedBytes; // Number of bytes that are heap allocated by elements that are stored in this buffer and is now on hold. // For simple types this is 0. - size_t _extraHoldBytes; + std::atomic<size_t> _extraHoldBytes; FreeList _freeList; FreeListList *_freeListList; // non-nullptr if free lists are enabled @@ -65,11 +65,11 @@ private: BufferState *_nextHasFree; BufferState *_prevHasFree; - BufferTypeBase *_typeHandler; + std::atomic<BufferTypeBase*> _typeHandler; Alloc _buffer; uint32_t _arraySize; uint16_t _typeId; - State _state : 8; + std::atomic<State> _state; bool _disableElemHoldList : 1; bool _compacting : 1; @@ -140,51 +140,51 @@ public: if (isFreeListEmpty()) { removeFromFreeListList(); } - _deadElems -= _arraySize; + _deadElems.store(getDeadElems() - _arraySize, std::memory_order_relaxed); return ret; } - size_t size() const { return _usedElems; } - size_t capacity() const { return _allocElems; } - size_t remaining() const { return _allocElems - _usedElems; } + size_t size() const { return _usedElems.load(std::memory_order_relaxed); } + size_t capacity() const { return _allocElems.load(std::memory_order_relaxed); } + size_t remaining() const { return capacity() - size(); } void pushed_back(size_t numElems) { pushed_back(numElems, 0); } void pushed_back(size_t numElems, size_t extraBytes) { - _usedElems += numElems; - _extraUsedBytes += extraBytes; + _usedElems.store(size() + numElems, std::memory_order_relaxed); + _extraUsedBytes.store(getExtraUsedBytes() + extraBytes, std::memory_order_relaxed); } void cleanHold(void *buffer, size_t offset, ElemCount numElems) { - _typeHandler->cleanHold(buffer, offset, numElems, BufferTypeBase::CleanContext(_extraUsedBytes, _extraHoldBytes)); + getTypeHandler()->cleanHold(buffer, offset, numElems, BufferTypeBase::CleanContext(_extraUsedBytes, _extraHoldBytes)); } void dropBuffer(uint32_t buffer_id, std::atomic<void*>& buffer); uint32_t getTypeId() const { return _typeId; } uint32_t getArraySize() const { return _arraySize; } - size_t getDeadElems() const { return _deadElems; } - size_t getHoldElems() const { return _holdElems; } - size_t getExtraUsedBytes() const { return _extraUsedBytes; } - size_t getExtraHoldBytes() const { return _extraHoldBytes; } + size_t getDeadElems() const { return _deadElems.load(std::memory_order_relaxed); } + size_t getHoldElems() const { return _holdElems.load(std::memory_order_relaxed); } + size_t getExtraUsedBytes() const { return _extraUsedBytes.load(std::memory_order_relaxed); } + size_t getExtraHoldBytes() const { return _extraHoldBytes.load(std::memory_order_relaxed); } bool getCompacting() const { return _compacting; } void setCompacting() { _compacting = true; } - uint32_t get_used_arrays() const noexcept { return _usedElems / _arraySize; } + uint32_t get_used_arrays() const noexcept { return size() / _arraySize; } void fallbackResize(uint32_t bufferId, size_t elementsNeeded, std::atomic<void*>& buffer, Alloc &holdBuffer); bool isActive(uint32_t typeId) const { - return ((_state == ACTIVE) && (_typeId == typeId)); + return (isActive() && (_typeId == typeId)); } - bool isActive() const { return (_state == ACTIVE); } - bool isOnHold() const { return (_state == HOLD); } - bool isFree() const { return (_state == FREE); } - State getState() const { return _state; } - const BufferTypeBase *getTypeHandler() const { return _typeHandler; } - BufferTypeBase *getTypeHandler() { return _typeHandler; } - - void incDeadElems(size_t value) { _deadElems += value; } - void incHoldElems(size_t value) { _holdElems += value; } + bool isActive() const { return (getState() == State::ACTIVE); } + bool isOnHold() const { return (getState() == State::HOLD); } + bool isFree() const { return (getState() == State::FREE); } + State getState() const { return _state.load(std::memory_order_relaxed); } + const BufferTypeBase *getTypeHandler() const { return _typeHandler.load(std::memory_order_relaxed); } + BufferTypeBase *getTypeHandler() { return _typeHandler.load(std::memory_order_relaxed); } + + void incDeadElems(size_t value) { _deadElems.store(getDeadElems() + value, std::memory_order_relaxed); } + void incHoldElems(size_t value) { _holdElems.store(getHoldElems() + value, std::memory_order_relaxed); } void decHoldElems(size_t value); - void incExtraUsedBytes(size_t value) { _extraUsedBytes += value; } + void incExtraUsedBytes(size_t value) { _extraUsedBytes.store(getExtraUsedBytes() + value, std::memory_order_relaxed); } void incExtraHoldBytes(size_t value) { - _extraHoldBytes += value; + _extraHoldBytes.store(getExtraHoldBytes() + value, std::memory_order_relaxed); } bool hasDisabledElemHoldList() const { return _disableElemHoldList; } diff --git a/vespalib/src/vespa/vespalib/datastore/datastorebase.cpp b/vespalib/src/vespa/vespalib/datastore/datastorebase.cpp index 43e47296029..cf5406bf343 100644 --- a/vespalib/src/vespa/vespalib/datastore/datastorebase.cpp +++ b/vespalib/src/vespa/vespalib/datastore/datastorebase.cpp @@ -365,13 +365,13 @@ DataStoreBase::getMemStats() const for (const BufferState & bState: _states) { auto typeHandler = bState.getTypeHandler(); BufferState::State state = bState.getState(); - if ((state == BufferState::FREE) || (typeHandler == nullptr)) { + if ((state == BufferState::State::FREE) || (typeHandler == nullptr)) { ++stats._freeBuffers; - } else if (state == BufferState::ACTIVE) { + } else if (state == BufferState::State::ACTIVE) { size_t elementSize = typeHandler->elementSize(); ++stats._activeBuffers; add_buffer_state_to_mem_stats(bState, elementSize, stats); - } else if (state == BufferState::HOLD) { + } else if (state == BufferState::State::HOLD) { size_t elementSize = typeHandler->elementSize(); ++stats._holdBuffers; add_buffer_state_to_mem_stats(bState, elementSize, stats); diff --git a/vespalib/src/vespa/vespalib/net/tls/CMakeLists.txt b/vespalib/src/vespa/vespalib/net/tls/CMakeLists.txt index f1e64241533..424c2bd672f 100644 --- a/vespalib/src/vespa/vespalib/net/tls/CMakeLists.txt +++ b/vespalib/src/vespa/vespalib/net/tls/CMakeLists.txt @@ -1,7 +1,9 @@ # Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. vespa_add_library(vespalib_vespalib_net_tls OBJECT SOURCES + assumed_roles.cpp authorization_mode.cpp + authorization_result.cpp auto_reloading_tls_crypto_engine.cpp crypto_codec.cpp crypto_codec_adapter.cpp diff --git a/vespalib/src/vespa/vespalib/net/tls/assumed_roles.cpp b/vespalib/src/vespa/vespalib/net/tls/assumed_roles.cpp new file mode 100644 index 00000000000..672458d0024 --- /dev/null +++ b/vespalib/src/vespa/vespalib/net/tls/assumed_roles.cpp @@ -0,0 +1,95 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include "assumed_roles.h" +#include <vespa/vespalib/stllike/asciistream.h> +#include <algorithm> +#include <ostream> + +namespace vespalib::net::tls { + +const string AssumedRoles::WildcardRole("*"); + +AssumedRoles::AssumedRoles() = default; + +AssumedRoles::AssumedRoles(RoleSet assumed_roles) + : _assumed_roles(std::move(assumed_roles)) +{} + +AssumedRoles::AssumedRoles(const AssumedRoles&) = default; +AssumedRoles& AssumedRoles::operator=(const AssumedRoles&) = default; +AssumedRoles::AssumedRoles(AssumedRoles&&) noexcept = default; +AssumedRoles& AssumedRoles::operator=(AssumedRoles&&) noexcept = default; +AssumedRoles::~AssumedRoles() = default; + +bool AssumedRoles::can_assume_role(const string& role) const noexcept { + return (_assumed_roles.contains(role) || _assumed_roles.contains(WildcardRole)); +} + +std::vector<string> AssumedRoles::ordered_roles() const { + std::vector<string> roles; + for (const auto& r : _assumed_roles) { + roles.emplace_back(r); + } + std::sort(roles.begin(), roles.end()); + return roles; +} + +bool AssumedRoles::operator==(const AssumedRoles& rhs) const noexcept { + return (_assumed_roles == rhs._assumed_roles); +} + +void AssumedRoles::print(asciistream& os) const { + os << "AssumedRoles(roles: ["; + auto roles = ordered_roles(); + for (size_t i = 0; i < roles.size(); ++i) { + if (i > 0) { + os << ", "; + } + os << roles[i]; + } + os << "])"; +} + +asciistream& operator<<(asciistream& os, const AssumedRoles& res) { + res.print(os); + return os; +} + +std::ostream& operator<<(std::ostream& os, const AssumedRoles& res) { + os << to_string(res); + return os; +} + +string to_string(const AssumedRoles& res) { + asciistream os; + os << res; + return os.str(); +} + +AssumedRoles AssumedRoles::make_for_roles(RoleSet assumed_roles) { + return AssumedRoles(std::move(assumed_roles)); +} + +AssumedRoles AssumedRoles::make_wildcard_role() { + return AssumedRoles(RoleSet({WildcardRole})); +} + +AssumedRoles AssumedRoles::make_empty() { + return {}; +} + +AssumedRolesBuilder::AssumedRolesBuilder() = default; +AssumedRolesBuilder::~AssumedRolesBuilder() = default; + +void AssumedRolesBuilder::add_union(const AssumedRoles& roles) { + // TODO fix hash_set iterator range insert() + for (const auto& role : roles.unordered_roles()) { + _wip_roles.insert(role); + } +} + +AssumedRoles AssumedRolesBuilder::build_with_move() { + return AssumedRoles::make_for_roles(std::move(_wip_roles)); +} + +} + diff --git a/vespalib/src/vespa/vespalib/net/tls/assumed_roles.h b/vespalib/src/vespa/vespalib/net/tls/assumed_roles.h new file mode 100644 index 00000000000..00d800916fd --- /dev/null +++ b/vespalib/src/vespa/vespalib/net/tls/assumed_roles.h @@ -0,0 +1,80 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#pragma once + +#include <vespa/vespalib/stllike/hash_set.h> +#include <vespa/vespalib/stllike/string.h> +#include <vector> +#include <iosfwd> + +namespace vespalib { class asciistream; } + +namespace vespalib::net::tls { + +/** + * Encapsulates a set of roles that requests over a particular authenticated + * connection can assume, based on the authorization rules it matched during mTLS + * handshaking. + * + * If at least one role is a wildcard ('*') role, the connection can assume _any_ + * possible role. This is the default when no role constraints are specified in + * the TLS configuration file (legacy behavior). However, a default-constructed + * AssumedRoles instance does not allow any roles to be assumed. + */ +class AssumedRoles { +public: + using RoleSet = hash_set<string>; +private: + RoleSet _assumed_roles; + + static const string WildcardRole; + + explicit AssumedRoles(RoleSet assumed_roles); +public: + AssumedRoles(); + AssumedRoles(const AssumedRoles&); + AssumedRoles& operator=(const AssumedRoles&); + AssumedRoles(AssumedRoles&&) noexcept; + AssumedRoles& operator=(AssumedRoles&&) noexcept; + ~AssumedRoles(); + + [[nodiscard]] bool empty() const noexcept { + return _assumed_roles.empty(); + } + + /** + * Returns true iff `role` is present in the role set OR the role set contains + * the special wildcard role. + */ + [[nodiscard]] bool can_assume_role(const string& role) const noexcept; + + [[nodiscard]] const RoleSet& unordered_roles() const noexcept { + return _assumed_roles; + } + + [[nodiscard]] std::vector<string> ordered_roles() const; + + bool operator==(const AssumedRoles& rhs) const noexcept; + + void print(asciistream& os) const; + + static AssumedRoles make_for_roles(RoleSet assumed_roles); + static AssumedRoles make_wildcard_role(); // Allows assuming _all_ possible roles + static AssumedRoles make_empty(); // Matches _no_ possible roles +}; + +asciistream& operator<<(asciistream&, const AssumedRoles&); +std::ostream& operator<<(std::ostream&, const AssumedRoles&); +string to_string(const AssumedRoles&); + +class AssumedRolesBuilder { + AssumedRoles::RoleSet _wip_roles; +public: + AssumedRolesBuilder(); + ~AssumedRolesBuilder(); + + void add_union(const AssumedRoles& roles); + [[nodiscard]] bool empty() const noexcept { return _wip_roles.empty(); } + [[nodiscard]] AssumedRoles build_with_move(); +}; + +} diff --git a/vespalib/src/vespa/vespalib/net/tls/authorization_result.cpp b/vespalib/src/vespa/vespalib/net/tls/authorization_result.cpp new file mode 100644 index 00000000000..069e971833c --- /dev/null +++ b/vespalib/src/vespa/vespalib/net/tls/authorization_result.cpp @@ -0,0 +1,62 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include "authorization_result.h" +#include <vespa/vespalib/stllike/asciistream.h> +#include <ostream> + +namespace vespalib::net::tls { + +AuthorizationResult::AuthorizationResult() = default; + +AuthorizationResult::AuthorizationResult(AssumedRoles assumed_roles) + : _assumed_roles(std::move(assumed_roles)) +{} + +AuthorizationResult::AuthorizationResult(const AuthorizationResult&) = default; +AuthorizationResult& AuthorizationResult::operator=(const AuthorizationResult&) = default; +AuthorizationResult::AuthorizationResult(AuthorizationResult&&) noexcept = default; +AuthorizationResult& AuthorizationResult::operator=(AuthorizationResult&&) noexcept = default; +AuthorizationResult::~AuthorizationResult() = default; + +void AuthorizationResult::print(asciistream& os) const { + os << "AuthorizationResult("; + if (!success()) { + os << "NOT AUTHORIZED"; + } else { + os << _assumed_roles; + } + os << ')'; +} + +AuthorizationResult +AuthorizationResult::make_authorized_for_roles(AssumedRoles assumed_roles) { + return AuthorizationResult(std::move(assumed_roles)); +} + +AuthorizationResult +AuthorizationResult::make_authorized_for_all_roles() { + return AuthorizationResult(AssumedRoles::make_wildcard_role()); +} + +AuthorizationResult +AuthorizationResult::make_not_authorized() { + return {}; +} + +asciistream& operator<<(asciistream& os, const AuthorizationResult& res) { + res.print(os); + return os; +} + +std::ostream& operator<<(std::ostream& os, const AuthorizationResult& res) { + os << to_string(res); + return os; +} + +string to_string(const AuthorizationResult& res) { + asciistream os; + os << res; + return os.str(); +} + +} diff --git a/vespalib/src/vespa/vespalib/net/tls/authorization_result.h b/vespalib/src/vespa/vespalib/net/tls/authorization_result.h new file mode 100644 index 00000000000..b92bbbca9dd --- /dev/null +++ b/vespalib/src/vespa/vespalib/net/tls/authorization_result.h @@ -0,0 +1,55 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#pragma once + +#include "assumed_roles.h" +#include <vespa/vespalib/stllike/string.h> +#include <iosfwd> + +namespace vespalib { class asciistream; } + +namespace vespalib::net::tls { + +/** + * The result of evaluating configured mTLS authorization rules against the + * credentials presented by a successfully authenticated peer certificate. + * + * This result contains the union set of all roles specified by the matching + * authorization rules. If no rules matched, the set will be empty. The role + * set will also be empty for a default-constructed instance. + */ +class AuthorizationResult { + AssumedRoles _assumed_roles; + + explicit AuthorizationResult(AssumedRoles assumed_roles); +public: + AuthorizationResult(); + AuthorizationResult(const AuthorizationResult&); + AuthorizationResult& operator=(const AuthorizationResult&); + AuthorizationResult(AuthorizationResult&&) noexcept; + AuthorizationResult& operator=(AuthorizationResult&&) noexcept; + ~AuthorizationResult(); + + // Returns true iff at least one assumed role has been granted. + [[nodiscard]] bool success() const noexcept { + return !_assumed_roles.empty(); + } + + [[nodiscard]] const AssumedRoles& assumed_roles() const noexcept { + return _assumed_roles; + } + [[nodiscard]] AssumedRoles steal_assumed_roles() noexcept { + return std::move(_assumed_roles); + } + + void print(asciistream& os) const; + + static AuthorizationResult make_authorized_for_roles(AssumedRoles assumed_roles); + static AuthorizationResult make_authorized_for_all_roles(); + static AuthorizationResult make_not_authorized(); +}; + +asciistream& operator<<(asciistream&, const AuthorizationResult&); +std::ostream& operator<<(std::ostream&, const AuthorizationResult&); +string to_string(const AuthorizationResult&); + +} diff --git a/vespalib/src/vespa/vespalib/net/tls/certificate_verification_callback.h b/vespalib/src/vespa/vespalib/net/tls/certificate_verification_callback.h index dec00486dcd..0c18ba1a789 100644 --- a/vespalib/src/vespa/vespalib/net/tls/certificate_verification_callback.h +++ b/vespalib/src/vespa/vespalib/net/tls/certificate_verification_callback.h @@ -1,6 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #pragma once +#include "authorization_result.h" #include "peer_credentials.h" namespace vespalib::net::tls { @@ -13,15 +14,15 @@ struct CertificateVerificationCallback { virtual ~CertificateVerificationCallback() = default; // Return true iff the peer credentials pass verification, false otherwise. // Must be thread safe. - virtual bool verify(const PeerCredentials& peer_creds) const = 0; + [[nodiscard]] virtual AuthorizationResult verify(const PeerCredentials& peer_creds) const = 0; }; // Simplest possible certificate verification callback which accepts the certificate // iff all its pre-verification by OpenSSL has passed. This means its chain is valid // and it is signed by a trusted CA. struct AcceptAllPreVerifiedCertificates : CertificateVerificationCallback { - bool verify([[maybe_unused]] const PeerCredentials& peer_creds) const override { - return true; // yolo + AuthorizationResult verify([[maybe_unused]] const PeerCredentials& peer_creds) const override { + return AuthorizationResult::make_authorized_for_all_roles(); // yolo } }; diff --git a/vespalib/src/vespa/vespalib/net/tls/crypto_codec.h b/vespalib/src/vespa/vespalib/net/tls/crypto_codec.h index 7448bf49799..86ccaf3eb64 100644 --- a/vespalib/src/vespa/vespalib/net/tls/crypto_codec.h +++ b/vespalib/src/vespa/vespalib/net/tls/crypto_codec.h @@ -53,6 +53,8 @@ struct DecodeResult { }; struct TlsContext; +class PeerCredentials; +class AssumedRoles; // TODO move to different namespace, not dependent on TLS? @@ -175,6 +177,18 @@ public: */ virtual EncodeResult half_close(char* ciphertext, size_t ciphertext_size) noexcept = 0; + /** + * Credentials of the _remote peer_ as observed during certificate exchange. E.g. + * if this is a client codec, peer_credentials() returns the _server_ credentials + * and vice versa. + */ + [[nodiscard]] virtual const PeerCredentials& peer_credentials() const noexcept = 0; + + /** + * Union set of all assumed roles in the peer policy rules that fully matched the peer's credentials. + */ + [[nodiscard]] virtual const AssumedRoles& assumed_roles() const noexcept = 0; + /* * Creates an implementation defined CryptoCodec that provides at least TLSv1.2 * compliant handshaking and full duplex data transfer. diff --git a/vespalib/src/vespa/vespalib/net/tls/impl/openssl_crypto_codec_impl.h b/vespalib/src/vespa/vespalib/net/tls/impl/openssl_crypto_codec_impl.h index 34ca31a8f6c..5be2146b349 100644 --- a/vespalib/src/vespa/vespalib/net/tls/impl/openssl_crypto_codec_impl.h +++ b/vespalib/src/vespa/vespalib/net/tls/impl/openssl_crypto_codec_impl.h @@ -4,8 +4,10 @@ #include <vespa/vespalib/crypto/openssl_typedefs.h> #include <vespa/vespalib/net/socket_address.h> #include <vespa/vespalib/net/socket_spec.h> -#include <vespa/vespalib/net/tls/transport_security_options.h> +#include <vespa/vespalib/net/tls/assumed_roles.h> #include <vespa/vespalib/net/tls/crypto_codec.h> +#include <vespa/vespalib/net/tls/peer_credentials.h> +#include <vespa/vespalib/net/tls/transport_security_options.h> #include <memory> #include <optional> @@ -55,6 +57,8 @@ class OpenSslCryptoCodecImpl : public CryptoCodec { Mode _mode; std::optional<DeferredHandshakeParams> _deferred_handshake_params; std::optional<HandshakeResult> _deferred_handshake_result; + PeerCredentials _peer_credentials; + AssumedRoles _assumed_roles; public: ~OpenSslCryptoCodecImpl() override; @@ -95,6 +99,14 @@ public: char* plaintext, size_t plaintext_size) noexcept override; EncodeResult half_close(char* ciphertext, size_t ciphertext_size) noexcept override; + [[nodiscard]] const PeerCredentials& peer_credentials() const noexcept override { + return _peer_credentials; + } + + [[nodiscard]] const AssumedRoles& assumed_roles() const noexcept override { + return _assumed_roles; + } + const SocketAddress& peer_address() const noexcept { return _peer_address; } /* * If a client has sent a SNI extension field as part of the handshake, @@ -102,6 +114,15 @@ public: * call this for codecs in server mode. */ std::optional<vespalib::string> client_provided_sni_extension() const; + + // Only used by code bridging OpenSSL certificate verification callbacks and + // evaluation of custom authorization rules. + void set_peer_credentials(PeerCredentials peer_credentials) { + _peer_credentials = std::move(peer_credentials); + } + void set_assumed_roles(AssumedRoles assumed_roles) { + _assumed_roles = std::move(assumed_roles); + } private: OpenSslCryptoCodecImpl(std::shared_ptr<OpenSslTlsContextImpl> ctx, const SocketSpec& peer_spec, diff --git a/vespalib/src/vespa/vespalib/net/tls/impl/openssl_tls_context_impl.cpp b/vespalib/src/vespa/vespalib/net/tls/impl/openssl_tls_context_impl.cpp index 40e4e1adbcf..3810140854b 100644 --- a/vespalib/src/vespa/vespalib/net/tls/impl/openssl_tls_context_impl.cpp +++ b/vespalib/src/vespa/vespalib/net/tls/impl/openssl_tls_context_impl.cpp @@ -451,14 +451,14 @@ int OpenSslTlsContextImpl::verify_cb_wrapper(int preverified_ok, ::X509_STORE_CT auto* self = static_cast<OpenSslTlsContextImpl*>(SSL_CTX_get_app_data(ssl_ctx)); LOG_ASSERT(self != nullptr); - if (self->verify_trusted_certificate(store_ctx, codec_impl->peer_address())) { + if (self->verify_trusted_certificate(store_ctx, *codec_impl)) { return 1; } ConnectionStatistics::get(SSL_in_accept_init(ssl) != 0).inc_invalid_peer_credentials(); return 0; } -bool OpenSslTlsContextImpl::verify_trusted_certificate(::X509_STORE_CTX* store_ctx, const SocketAddress& peer_address) { +bool OpenSslTlsContextImpl::verify_trusted_certificate(::X509_STORE_CTX* store_ctx, OpenSslCryptoCodecImpl& codec_impl) { const auto authz_mode = authorization_mode(); // TODO consider if we want to fill in peer credentials even if authorization is disabled if (authz_mode == AuthorizationMode::Disable) { @@ -477,18 +477,22 @@ bool OpenSslTlsContextImpl::verify_trusted_certificate(::X509_STORE_CTX* store_c return false; } try { - const bool verified_by_cb = _cert_verify_callback->verify(creds); - if (!verified_by_cb) { + auto authz_result = _cert_verify_callback->verify(creds); + if (!authz_result.success()) { // Buffer warnings on peer IP address to avoid log flooding. - LOGBT(warning, peer_address.ip_address(), + LOGBT(warning, codec_impl.peer_address().ip_address(), "Certificate verification of peer '%s' failed with %s", - peer_address.spec().c_str(), to_string(creds).c_str()); + codec_impl.peer_address().spec().c_str(), to_string(creds).c_str()); return (authz_mode != AuthorizationMode::Enforce); } + // Store away credentials and role set for later use by requests that arrive over this connection. + // TODO encapsulate as const shared_ptr to immutable object to better facilitate sharing? + codec_impl.set_peer_credentials(std::move(creds)); + codec_impl.set_assumed_roles(authz_result.steal_assumed_roles()); } catch (std::exception& e) { - LOGBT(error, peer_address.ip_address(), + LOGBT(error, codec_impl.peer_address().ip_address(), "Got exception during certificate verification callback for peer '%s': %s", - peer_address.spec().c_str(), e.what()); + codec_impl.peer_address().spec().c_str(), e.what()); return false; } // we don't expect any non-std::exception derived exceptions, so let them terminate the process. return true; diff --git a/vespalib/src/vespa/vespalib/net/tls/impl/openssl_tls_context_impl.h b/vespalib/src/vespa/vespalib/net/tls/impl/openssl_tls_context_impl.h index b84a599dead..d9e161a7b0f 100644 --- a/vespalib/src/vespa/vespalib/net/tls/impl/openssl_tls_context_impl.h +++ b/vespalib/src/vespa/vespalib/net/tls/impl/openssl_tls_context_impl.h @@ -12,6 +12,8 @@ namespace vespalib::net::tls::impl { +class OpenSslCryptoCodecImpl; + class OpenSslTlsContextImpl : public TlsContext { crypto::SslCtxPtr _ctx; AuthorizationMode _authorization_mode; @@ -47,7 +49,7 @@ private: void set_ssl_ctx_self_reference(); void set_accepted_cipher_suites(const std::vector<vespalib::string>& ciphers); - bool verify_trusted_certificate(::X509_STORE_CTX* store_ctx, const SocketAddress& peer_address); + bool verify_trusted_certificate(::X509_STORE_CTX* store_ctx, OpenSslCryptoCodecImpl& codec_impl); static int verify_cb_wrapper(int preverified_ok, ::X509_STORE_CTX* store_ctx); }; diff --git a/vespalib/src/vespa/vespalib/net/tls/peer_credentials.cpp b/vespalib/src/vespa/vespalib/net/tls/peer_credentials.cpp index e00d4804fbe..9a001e24fea 100644 --- a/vespalib/src/vespa/vespalib/net/tls/peer_credentials.cpp +++ b/vespalib/src/vespa/vespalib/net/tls/peer_credentials.cpp @@ -2,12 +2,15 @@ #include "peer_credentials.h" #include <vespa/vespalib/stllike/asciistream.h> -#include <iostream> -#include <sstream> +#include <ostream> namespace vespalib::net::tls { PeerCredentials::PeerCredentials() = default; +PeerCredentials::PeerCredentials(const PeerCredentials&) = default; +PeerCredentials& PeerCredentials::operator=(const PeerCredentials&) = default; +PeerCredentials::PeerCredentials(PeerCredentials&&) noexcept = default; +PeerCredentials& PeerCredentials::operator=(PeerCredentials&&) noexcept = default; PeerCredentials::~PeerCredentials() = default; std::ostream& operator<<(std::ostream& os, const PeerCredentials& creds) { @@ -15,17 +18,40 @@ std::ostream& operator<<(std::ostream& os, const PeerCredentials& creds) { return os; } -vespalib::string to_string(const PeerCredentials& creds) { - vespalib::asciistream os; - os << "PeerCredentials(CN '" << creds.common_name - << "', DNS SANs ["; - for (size_t i = 0; i < creds.dns_sans.size(); ++i) { +namespace { +void emit_comma_separated_string_list(asciistream& os, stringref title, + const std::vector<string>& strings, bool prefix_comma) +{ + if (prefix_comma) { + os << ", "; + } + os << title << " ["; + for (size_t i = 0; i < strings.size(); ++i) { if (i != 0) { os << ", "; } - os << '\'' << creds.dns_sans[i] << '\''; + os << '\'' << strings[i] << '\''; + } + os << ']'; +} +} + +vespalib::string to_string(const PeerCredentials& creds) { + asciistream os; + os << "PeerCredentials("; + bool emit_comma = false; + if (!creds.common_name.empty()) { + os << "CN '" << creds.common_name << "'"; + emit_comma = true; + } + if (!creds.dns_sans.empty()) { + emit_comma_separated_string_list(os, "DNS SANs", creds.dns_sans, emit_comma); + emit_comma = true; + } + if (!creds.uri_sans.empty()) { + emit_comma_separated_string_list(os, "URI SANs", creds.uri_sans, emit_comma); } - os << "])"; + os << ')'; return os.str(); } diff --git a/vespalib/src/vespa/vespalib/net/tls/peer_credentials.h b/vespalib/src/vespa/vespalib/net/tls/peer_credentials.h index 636d643a62f..b81772d2bce 100644 --- a/vespalib/src/vespa/vespalib/net/tls/peer_credentials.h +++ b/vespalib/src/vespa/vespalib/net/tls/peer_credentials.h @@ -18,6 +18,10 @@ struct PeerCredentials { std::vector<vespalib::string> uri_sans; PeerCredentials(); + PeerCredentials(const PeerCredentials&); + PeerCredentials& operator=(const PeerCredentials&); + PeerCredentials(PeerCredentials&&) noexcept; + PeerCredentials& operator=(PeerCredentials&&) noexcept; ~PeerCredentials(); }; diff --git a/vespalib/src/vespa/vespalib/net/tls/peer_policies.cpp b/vespalib/src/vespa/vespalib/net/tls/peer_policies.cpp index a476e23e6cb..a4e651f3f19 100644 --- a/vespalib/src/vespa/vespalib/net/tls/peer_policies.cpp +++ b/vespalib/src/vespa/vespalib/net/tls/peer_policies.cpp @@ -119,6 +119,21 @@ RequiredPeerCredential::RequiredPeerCredential(Field field, vespalib::string mus RequiredPeerCredential::~RequiredPeerCredential() = default; +PeerPolicy::PeerPolicy() = default; + +PeerPolicy::PeerPolicy(std::vector<RequiredPeerCredential> required_peer_credentials) + : _required_peer_credentials(std::move(required_peer_credentials)), + _assumed_roles(AssumedRoles::make_wildcard_role()) +{} + +PeerPolicy::PeerPolicy(std::vector<RequiredPeerCredential> required_peer_credentials, + AssumedRoles assumed_roles) + : _required_peer_credentials(std::move(required_peer_credentials)), + _assumed_roles(std::move(assumed_roles)) +{} + +PeerPolicy::~PeerPolicy() = default; + namespace { template <typename Collection> void print_joined(std::ostream& os, const Collection& coll, const char* sep) { diff --git a/vespalib/src/vespa/vespalib/net/tls/peer_policies.h b/vespalib/src/vespa/vespalib/net/tls/peer_policies.h index 4166efc4312..6eab8c2c9b2 100644 --- a/vespalib/src/vespa/vespalib/net/tls/peer_policies.h +++ b/vespalib/src/vespa/vespalib/net/tls/peer_policies.h @@ -1,6 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #pragma once +#include "assumed_roles.h" #include <vespa/vespalib/stllike/string.h> #include <memory> #include <vector> @@ -49,18 +50,27 @@ public: class PeerPolicy { // _All_ credentials must match for the policy itself to match. std::vector<RequiredPeerCredential> _required_peer_credentials; + AssumedRoles _assumed_roles; public: - PeerPolicy() = default; - explicit PeerPolicy(std::vector<RequiredPeerCredential> required_peer_credentials_) - : _required_peer_credentials(std::move(required_peer_credentials_)) - {} + PeerPolicy(); + // This policy is created with a wildcard role set, i.e. full access. + explicit PeerPolicy(std::vector<RequiredPeerCredential> required_peer_credentials); + + PeerPolicy(std::vector<RequiredPeerCredential> required_peer_credentials, + AssumedRoles assumed_roles); - bool operator==(const PeerPolicy& rhs) const { - return (_required_peer_credentials == rhs._required_peer_credentials); + ~PeerPolicy(); + + bool operator==(const PeerPolicy& rhs) const noexcept { + return ((_required_peer_credentials == rhs._required_peer_credentials) && + (_assumed_roles == rhs._assumed_roles)); } - const std::vector<RequiredPeerCredential>& required_peer_credentials() const noexcept { + [[nodiscard]] const std::vector<RequiredPeerCredential>& required_peer_credentials() const noexcept { return _required_peer_credentials; } + [[nodiscard]] const AssumedRoles& assumed_roles() const noexcept { + return _assumed_roles; + } }; class AuthorizedPeers { diff --git a/vespalib/src/vespa/vespalib/net/tls/policy_checking_certificate_verifier.cpp b/vespalib/src/vespa/vespalib/net/tls/policy_checking_certificate_verifier.cpp index e2c45ad7358..65e14434ff1 100644 --- a/vespalib/src/vespa/vespalib/net/tls/policy_checking_certificate_verifier.cpp +++ b/vespalib/src/vespa/vespalib/net/tls/policy_checking_certificate_verifier.cpp @@ -61,7 +61,7 @@ public: ~PolicyConfiguredCertificateVerifier() override; - bool verify(const PeerCredentials& peer_creds) const override; + AuthorizationResult verify(const PeerCredentials& peer_creds) const override; }; PolicyConfiguredCertificateVerifier::PolicyConfiguredCertificateVerifier(AuthorizedPeers authorized_peers) noexcept @@ -69,16 +69,21 @@ PolicyConfiguredCertificateVerifier::PolicyConfiguredCertificateVerifier(Authori PolicyConfiguredCertificateVerifier::~PolicyConfiguredCertificateVerifier() = default; -bool PolicyConfiguredCertificateVerifier::verify(const PeerCredentials& peer_creds) const { +AuthorizationResult PolicyConfiguredCertificateVerifier::verify(const PeerCredentials& peer_creds) const { if (_authorized_peers.allows_all_authenticated()) { - return true; + return AuthorizationResult::make_authorized_for_all_roles(); } + AssumedRolesBuilder roles_builder; for (const auto& policy : _authorized_peers.peer_policies()) { if (matches_all_policy_requirements(peer_creds, policy)) { - return true; + roles_builder.add_union(policy.assumed_roles()); } } - return false; + if (!roles_builder.empty()) { + return AuthorizationResult::make_authorized_for_roles(roles_builder.build_with_move()); + } else { + return AuthorizationResult::make_not_authorized(); + } } std::shared_ptr<CertificateVerificationCallback> create_verify_callback_from(AuthorizedPeers authorized_peers) { diff --git a/vespalib/src/vespa/vespalib/test/peer_policy_utils.cpp b/vespalib/src/vespa/vespalib/test/peer_policy_utils.cpp index 724efa63e47..82d7b9ea07b 100644 --- a/vespalib/src/vespa/vespalib/test/peer_policy_utils.cpp +++ b/vespalib/src/vespa/vespalib/test/peer_policy_utils.cpp @@ -16,10 +16,23 @@ RequiredPeerCredential required_san_uri(vespalib::stringref pattern) { return {RequiredPeerCredential::Field::SAN_URI, pattern}; } +AssumedRoles assumed_roles(const std::vector<string>& roles) { + // TODO fix hash_set iterator range ctor to make this a one-liner + AssumedRoles::RoleSet role_set; + for (const auto& role : roles) { + role_set.insert(role); + } + return AssumedRoles::make_for_roles(std::move(role_set)); +} + PeerPolicy policy_with(std::vector<RequiredPeerCredential> creds) { return PeerPolicy(std::move(creds)); } +PeerPolicy policy_with(std::vector<RequiredPeerCredential> creds, AssumedRoles roles) { + return {std::move(creds), std::move(roles)}; +} + AuthorizedPeers authorized_peers(std::vector<PeerPolicy> peer_policies) { return AuthorizedPeers(std::move(peer_policies)); } diff --git a/vespalib/src/vespa/vespalib/test/peer_policy_utils.h b/vespalib/src/vespa/vespalib/test/peer_policy_utils.h index fe382f01b50..72e9fde20de 100644 --- a/vespalib/src/vespa/vespalib/test/peer_policy_utils.h +++ b/vespalib/src/vespa/vespalib/test/peer_policy_utils.h @@ -8,7 +8,9 @@ namespace vespalib::net::tls { RequiredPeerCredential required_cn(vespalib::stringref pattern); RequiredPeerCredential required_san_dns(vespalib::stringref pattern); RequiredPeerCredential required_san_uri(vespalib::stringref pattern); +AssumedRoles assumed_roles(const std::vector<string>& roles); PeerPolicy policy_with(std::vector<RequiredPeerCredential> creds); +PeerPolicy policy_with(std::vector<RequiredPeerCredential> creds, AssumedRoles roles); AuthorizedPeers authorized_peers(std::vector<PeerPolicy> peer_policies); } diff --git a/vespalib/src/vespa/vespalib/util/generationhandler.cpp b/vespalib/src/vespa/vespalib/util/generationhandler.cpp index a4b3dd6f5e6..7797978d187 100644 --- a/vespalib/src/vespa/vespalib/util/generationhandler.cpp +++ b/vespalib/src/vespa/vespalib/util/generationhandler.cpp @@ -125,7 +125,7 @@ GenerationHandler::updateFirstUsedGeneration() toFree->_next = _free; _free = toFree; } - _firstUsedGeneration = _first->_generation; + _firstUsedGeneration.store(_first->_generation, std::memory_order_relaxed); } GenerationHandler::GenerationHandler() @@ -215,7 +215,7 @@ GenerationHandler::getGenerationRefCount(generation_t gen) const { if (static_cast<sgeneration_t>(gen - _generation) > 0) return 0u; - if (static_cast<sgeneration_t>(_firstUsedGeneration - gen) > 0) + if (static_cast<sgeneration_t>(getFirstUsedGeneration() - gen) > 0) return 0u; for (GenerationHold *hold = _first; hold != nullptr; hold = hold->_next) { if (hold->_generation.load(std::memory_order_relaxed) == gen) diff --git a/vespalib/src/vespa/vespalib/util/generationhandler.h b/vespalib/src/vespa/vespalib/util/generationhandler.h index 0c4b49a2d5b..2aeb4c2f886 100644 --- a/vespalib/src/vespa/vespalib/util/generationhandler.h +++ b/vespalib/src/vespa/vespalib/util/generationhandler.h @@ -73,7 +73,7 @@ public: private: generation_t _generation; - generation_t _firstUsedGeneration; + std::atomic<generation_t> _firstUsedGeneration; std::atomic<GenerationHold *> _last; // Points to "current generation" entry GenerationHold *_first; // Points to "firstUsedGeneration" entry GenerationHold *_free; // List of free entries @@ -109,7 +109,7 @@ public: * if writer hasn't updated first used generation after last reader left. */ generation_t getFirstUsedGeneration() const { - return _firstUsedGeneration; + return _firstUsedGeneration.load(std::memory_order_relaxed); } /** diff --git a/zkfacade/src/main/java/com/yahoo/vespa/curator/Lock.java b/zkfacade/src/main/java/com/yahoo/vespa/curator/Lock.java index 3e9c586f43c..a5d2397dadb 100644 --- a/zkfacade/src/main/java/com/yahoo/vespa/curator/Lock.java +++ b/zkfacade/src/main/java/com/yahoo/vespa/curator/Lock.java @@ -13,7 +13,6 @@ import java.time.Instant; import java.util.HashMap; import java.util.Map; import java.util.concurrent.TimeUnit; -import java.util.function.Consumer; /** * A cluster-wide re-entrant mutex which is released on (the last symmetric) close. @@ -66,11 +65,16 @@ public class Lock implements Mutex { " to acquire lock '" + lockPath + "'"); } - invoke(+1L, threadLockStats::lockAcquired); + invoke(+1L, (lockPath, debug) -> threadLockStats.lockAcquired(debug), lockPath); + } + + @FunctionalInterface + private interface BiConsumer2 { + void accept(String lockPath, String debug); } // TODO(hakon): Remove once debugging is unnecessary - private void invoke(long reentryCountDiff, Consumer<String> consumer) { + private void invoke(long reentryCountDiff, BiConsumer2 consumer, String lockPath) { long threadId = Thread.currentThread().getId(); final long sequenceNumber; final Map<Long, Long> reentriesByThreadIdCopy; @@ -92,20 +96,22 @@ public class Lock implements Mutex { "@" + created + " Curator 0x" + Integer.toHexString(System.identityHashCode(curator)) + " lock " + lockPath + " #" + sequenceNumber + ", reentries by thread ID = " + reentriesByThreadIdCopy; - consumer.accept(debug); + consumer.accept(lockPath, debug); } @Override public void close() { ThreadLockStats threadLockStats = LockStats.getForCurrentThread(); // Update metrics now before release() to avoid double-counting time in locked state. - invoke(-1L, threadLockStats::preRelease); + // The lockPath must be sent down as close() may be invoked in an order not necessarily + // equal to the reverse order of acquires. + invoke(-1L, threadLockStats::preRelease, lockPath); try { mutex.release(); - threadLockStats.postRelease(); + threadLockStats.postRelease(lockPath); } catch (Exception e) { - threadLockStats.releaseFailed(); + threadLockStats.releaseFailed(lockPath); throw new RuntimeException("Exception releasing lock '" + lockPath + "'", e); } } diff --git a/zkfacade/src/main/java/com/yahoo/vespa/curator/stats/ThreadLockStats.java b/zkfacade/src/main/java/com/yahoo/vespa/curator/stats/ThreadLockStats.java index beda7eaa142..24305539be5 100644 --- a/zkfacade/src/main/java/com/yahoo/vespa/curator/stats/ThreadLockStats.java +++ b/zkfacade/src/main/java/com/yahoo/vespa/curator/stats/ThreadLockStats.java @@ -5,8 +5,8 @@ import com.yahoo.vespa.curator.Lock; import java.time.Duration; import java.util.HashSet; +import java.util.Iterator; import java.util.List; -import java.util.Map; import java.util.Optional; import java.util.Set; import java.util.concurrent.ConcurrentLinkedDeque; @@ -113,8 +113,8 @@ public class ThreadLockStats { } /** Mutable method (see class doc) */ - public void preRelease(String debug) { - withLastLockAttempt(lockAttempt -> { + public void preRelease(String path, String debug) { + withLastLockAttemptFor(path, lockAttempt -> { // Note on the order of these two statement: Same concerns apply here as in lockAcquired(). if (!lockAttempt.isReentry()) { @@ -126,13 +126,13 @@ public class ThreadLockStats { } /** Mutable method (see class doc) */ - public void postRelease() { - removeLastLockAttempt(LockAttempt::postRelease); + public void postRelease(String lockPath) { + removeLastLockAttemptFor(lockPath, LockAttempt::postRelease); } /** Mutable method (see class doc) */ - public void releaseFailed() { - removeLastLockAttempt(LockAttempt::releaseFailed); + public void releaseFailed(String lockPath) { + removeLastLockAttemptFor(lockPath, LockAttempt::releaseFailed); } /** Mutable method (see class doc) */ @@ -240,4 +240,32 @@ public class ThreadLockStats { LockStats.getGlobal().maybeSample(lockAttempt); } + + private void withLastLockAttemptFor(String lockPath, Consumer<LockAttempt> consumer) { + Iterator<LockAttempt> lockAttemptIterator = lockAttemptsStack.descendingIterator(); + while (lockAttemptIterator.hasNext()) { + LockAttempt lockAttempt = lockAttemptIterator.next(); + if (lockAttempt.getLockPath().equals(lockPath)) { + consumer.accept(lockAttempt); + return; + } + } + + logger.warning("Unable to find any lock attempts for " + lockPath); + } + + private void removeLastLockAttemptFor(String lockPath, Consumer<LockAttempt> consumer) { + Iterator<LockAttempt> lockAttemptIterator = lockAttemptsStack.descendingIterator(); + while (lockAttemptIterator.hasNext()) { + LockAttempt lockAttempt = lockAttemptIterator.next(); + if (lockAttempt.getLockPath().equals(lockPath)) { + lockAttemptIterator.remove(); + consumer.accept(lockAttempt); + LockStats.getGlobal().maybeSample(lockAttempt); + return; + } + } + + logger.warning("Unable to remove last lock attempt as no locks were found for " + lockPath); + } } diff --git a/zkfacade/src/test/java/com/yahoo/vespa/curator/stats/LockTest.java b/zkfacade/src/test/java/com/yahoo/vespa/curator/stats/LockTest.java index e5a1ea7c683..524fcd3abd1 100644 --- a/zkfacade/src/test/java/com/yahoo/vespa/curator/stats/LockTest.java +++ b/zkfacade/src/test/java/com/yahoo/vespa/curator/stats/LockTest.java @@ -200,8 +200,46 @@ public class LockTest { assertEquals(lock2Path, lockAttempts.get(1).getLockPath()); assertEquals(LockAttempt.LockState.ACQUIRED, lockAttempts.get(1).getLockState()); + lock2.close(); lock.close(); + } + + @Test + public void locksReleaseOutOfOrder() throws Exception { + when(mutex.acquire(anyLong(), any())).thenReturn(true); + + lock.acquire(acquireTimeout); + assertLockMetricsIs(new LockMetrics().setAcquireCount(1) + .setCumulativeAcquireCount(1) + .setAcquireSucceededCount(1) + .setCumulativeAcquireSucceededCount(1)); + lock2.acquire(acquireTimeout); + assertLock2MetricsIs(new LockMetrics().setAcquireCount(1) + .setCumulativeAcquireCount(1) + .setAcquireSucceededCount(1) + .setCumulativeAcquireSucceededCount(1)); lock.close(); + assertLockMetricsIs(new LockMetrics().setAcquireCount(0) + .setCumulativeAcquireCount(1) + .setAcquireSucceededCount(0) + .setCumulativeAcquireSucceededCount(1) + .setReleaseCount(1) + .setCumulativeReleaseCount(1)); + + List<ThreadLockStats> threadLockStats = LockStats.getGlobal().getThreadLockStats(); + assertEquals(1, threadLockStats.size()); + List<LockAttempt> lockAttempts = threadLockStats.get(0).getOngoingLockAttempts(); + assertEquals(1, lockAttempts.size()); + assertEquals(lock2Path, lockAttempts.get(0).getLockPath()); + assertEquals(LockAttempt.LockState.ACQUIRED, lockAttempts.get(0).getLockState()); + + lock2.close(); + assertLock2MetricsIs(new LockMetrics().setAcquireCount(0) + .setCumulativeAcquireCount(1) + .setAcquireSucceededCount(0) + .setCumulativeAcquireSucceededCount(1) + .setReleaseCount(1) + .setCumulativeReleaseCount(1)); } @Test |