diff options
259 files changed, 3617 insertions, 2336 deletions
diff --git a/athenz-identity-provider-service/pom.xml b/athenz-identity-provider-service/pom.xml index 212c66a3431..f15ebe87b03 100644 --- a/athenz-identity-provider-service/pom.xml +++ b/athenz-identity-provider-service/pom.xml @@ -133,12 +133,18 @@ <version>${project.version}</version> <scope>test</scope> </dependency> - <dependency> - <groupId>com.yahoo.vespa</groupId> - <artifactId>testutil</artifactId> - <version>${project.version}</version> - <scope>test</scope> - </dependency> + <dependency> + <groupId>com.yahoo.vespa</groupId> + <artifactId>testutil</artifactId> + <version>${project.version}</version> + <scope>test</scope> + </dependency> + <dependency> + <groupId>com.yahoo.vespa</groupId> + <artifactId>flags</artifactId> + <version>${project.version}</version> + <scope>test</scope> + </dependency> </dependencies> <build> diff --git a/component/abi-spec.json b/component/abi-spec.json index a6b6a558db6..cd42b8ff95a 100644 --- a/component/abi-spec.json +++ b/component/abi-spec.json @@ -73,6 +73,7 @@ "public static com.yahoo.component.ComponentId fromString(java.lang.String)", "public java.lang.String toFileName()", "public static com.yahoo.component.ComponentId fromFileName(java.lang.String)", + "public static void resetGlobalCountersForTests()", "public bridge synthetic int compareTo(java.lang.Object)" ], "fields": [] diff --git a/component/src/main/java/com/yahoo/component/ComponentId.java b/component/src/main/java/com/yahoo/component/ComponentId.java index 05c710e3fc1..4613be09543 100644 --- a/component/src/main/java/com/yahoo/component/ComponentId.java +++ b/component/src/main/java/com/yahoo/component/ComponentId.java @@ -32,15 +32,15 @@ public final class ComponentId implements Comparable<ComponentId> { private int count = 0; public int getAndIncrement() { return count++; } } - private static ThreadLocal<Counter> gid = new ThreadLocal<Counter>() { + private static ThreadLocal<Counter> threadLocalUniqueId = new ThreadLocal<Counter>() { @Override protected Counter initialValue() { return new Counter(); } }; - private static AtomicInteger uniqueTid = new AtomicInteger(0); - private static ThreadLocal<String> tid = new ThreadLocal<String>() { + private static AtomicInteger threadIdCounter = new AtomicInteger(0); + private static ThreadLocal<String> threadId = new ThreadLocal<String>() { @Override protected String initialValue() { - return new String("_"+uniqueTid.getAndIncrement()+"_"); + return new String("_" + threadIdCounter.getAndIncrement() + "_"); } }; @@ -58,7 +58,7 @@ public final class ComponentId implements Comparable<ComponentId> { } private String createAnonymousName(String name) { - return new StringBuilder(name).append(tid.get()).append(gid.get().getAndIncrement()).toString(); + return new StringBuilder(name).append(threadId.get()).append(threadLocalUniqueId.get().getAndIncrement()).toString(); } public ComponentId(String name, Version version, ComponentId namespace) { @@ -148,10 +148,7 @@ public final class ComponentId implements Comparable<ComponentId> { return spec.compareTo(other.spec); } - /** - * Creates a componentId that is unique for this run-time instance - */ - // TODO: Check if we really need this. -JB + /** Creates a componentId that is unique for this run-time instance */ public static ComponentId createAnonymousComponentId(String baseName) { return new ComponentId(baseName, null, null, true); } @@ -196,27 +193,27 @@ public final class ComponentId implements Comparable<ComponentId> { * Creates an id from a file <b>first</b> name string encoded in the standard translation (see {@link #toFileName}). * <b>Note</b> that any file last name, like e.g ".xml" must be stripped off before handoff to this method. */ - public static ComponentId fromFileName(final String fileName) { + public static ComponentId fromFileName(String fileName) { // Initial assumptions - String id=fileName; - Version version =null; - ComponentId namespace=null; + String id = fileName; + Version version = null; + ComponentId namespace = null; // Split out namespace, if any - int at=id.indexOf("@"); - if (at>0) { - String newId=id.substring(0,at); - namespace=ComponentId.fromString(id.substring(at+1)); - id=newId; + int at = id.indexOf("@"); + if (at > 0) { + String newId = id.substring(0, at); + namespace = ComponentId.fromString(id.substring(at + 1)); + id = newId; } // Split out version, if any - int dash=id.lastIndexOf("-"); - if (dash>0) { - String newId=id.substring(0,dash); + int dash = id.lastIndexOf("-"); + if (dash > 0) { + String newId = id.substring(0, dash); try { - version=new Version(id.substring(dash+1)); - id=newId; + version = new Version(id.substring(dash + 1)); + id = newId; } catch (IllegalArgumentException e) { // don't interpret the text following the dash as a version @@ -229,4 +226,10 @@ public final class ComponentId implements Comparable<ComponentId> { return new ComponentId(id,version,namespace); } + /** WARNING: For testing only: Resets counters creating anonymous component ids for this thread. */ + public static void resetGlobalCountersForTests() { + threadId.set("_0_"); + threadLocalUniqueId.set(new Counter()); + } + } diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/derived/DerivedConfiguration.java b/config-model/src/main/java/com/yahoo/searchdefinition/derived/DerivedConfiguration.java index 245913d7822..fc8710fa1a1 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/derived/DerivedConfiguration.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/derived/DerivedConfiguration.java @@ -15,9 +15,11 @@ import com.yahoo.search.query.profile.QueryProfileRegistry; import com.yahoo.searchdefinition.RankProfileRegistry; import com.yahoo.searchdefinition.Search; import com.yahoo.searchdefinition.derived.validation.Validation; +import com.yahoo.vespa.model.container.search.QueryProfiles; import java.io.IOException; import java.io.Writer; +import java.util.logging.Level; /** * A set of all derived configuration of a search definition. Use this as a facade to individual configurations when @@ -39,12 +41,13 @@ public class DerivedConfiguration { private VsmSummary streamingSummary; private IndexSchema indexSchema; private ImportedFields importedFields; + private QueryProfileRegistry queryProfiles; /** * Creates a complete derived configuration from a search definition. * Only used in tests. * - * @param search The search to derive a configuration from. Derived objects will be snapshots, but this argument is + * @param search the search to derive a configuration from. Derived objects will be snapshots, but this argument is * live. Which means that this object will be inconsistent when the given search definition is later * modified. * @param rankProfileRegistry a {@link com.yahoo.searchdefinition.RankProfileRegistry} @@ -56,11 +59,11 @@ public class DerivedConfiguration { /** * Creates a complete derived configuration snapshot from a search definition. * - * @param search The search to derive a configuration from. Derived objects will be snapshots, but this + * @param search the search to derive a configuration from. Derived objects will be snapshots, but this * argument is live. Which means that this object will be inconsistent when the given * search definition is later modified. * @param deployLogger a {@link DeployLogger} for logging when doing operations on this - * @param deployProperties Properties set on deploy. + * @param deployProperties properties set on deploy * @param rankProfileRegistry a {@link com.yahoo.searchdefinition.RankProfileRegistry} * @param queryProfiles the query profiles of this application */ @@ -72,6 +75,7 @@ public class DerivedConfiguration { ImportedMlModels importedModels) { Validator.ensureNotNull("Search definition", search); this.search = search; + this.queryProfiles = queryProfiles; if ( ! search.isDocumentsOnly()) { streamingFields = new VsmFields(search); streamingSummary = new VsmSummary(search); @@ -120,6 +124,10 @@ public class DerivedConfiguration { exportCfg(new DocumenttypesConfig(documentTypesCfg), toDirectory + "/" + "documenttypes.cfg"); } + public static void exportQueryProfiles(QueryProfileRegistry queryProfileRegistry, String toDirectory) throws IOException { + exportCfg(new QueryProfiles(queryProfileRegistry, (level, message) -> {}).getConfig(), toDirectory + "/" + "query-profiles.cfg"); + } + private static void exportCfg(ConfigInstance instance, String fileName) throws IOException { Writer writer = null; try { @@ -186,4 +194,7 @@ public class DerivedConfiguration { public ImportedFields getImportedFields() { return importedFields; } + + public QueryProfileRegistry getQueryProfiles() { return queryProfiles; } + } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainer.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainer.java index fd924eb2a0f..fccacc3210d 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainer.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainer.java @@ -2,6 +2,8 @@ package com.yahoo.vespa.model.admin.metricsproxy; +import ai.vespa.metricsproxy.http.metrics.MetricsV2Handler; +import ai.vespa.metricsproxy.http.metrics.NodeInfoConfig; import ai.vespa.metricsproxy.metric.dimensions.NodeDimensions; import ai.vespa.metricsproxy.metric.dimensions.NodeDimensionsConfig; import ai.vespa.metricsproxy.metric.dimensions.PublicDimensions; @@ -20,6 +22,7 @@ import java.util.Map; import static com.yahoo.config.model.api.container.ContainerServiceType.METRICS_PROXY_CONTAINER; import static com.yahoo.vespa.model.admin.metricsproxy.MetricsProxyContainerCluster.METRICS_PROXY_BUNDLE_NAME; +import static com.yahoo.vespa.model.admin.metricsproxy.MetricsProxyContainerCluster.createMetricsHandler; /** * Container running a metrics proxy. @@ -28,9 +31,11 @@ import static com.yahoo.vespa.model.admin.metricsproxy.MetricsProxyContainerClus */ public class MetricsProxyContainer extends Container implements NodeDimensionsConfig.Producer, + NodeInfoConfig.Producer, RpcConnectorConfig.Producer, VespaServicesConfig.Producer { + public static final int BASEPORT = 19092; final boolean isHostedVespa; @@ -46,6 +51,7 @@ public class MetricsProxyContainer extends Container implements addMetricsProxyComponent(NodeDimensions.class); addMetricsProxyComponent(RpcConnector.class); addMetricsProxyComponent(VespaServices.class); + addHandler(createMetricsHandler(MetricsV2Handler.class, MetricsV2Handler.V2_PATH)); } @Override @@ -53,8 +59,6 @@ public class MetricsProxyContainer extends Container implements return METRICS_PROXY_CONTAINER; } - static public int BASEPORT = 19092; - @Override public int getWantedPort() { return BASEPORT; @@ -121,7 +125,21 @@ public class MetricsProxyContainer extends Container implements } } - private void addMetricsProxyComponent(Class<?> componentClass) { + @Override + public void getConfig(NodeInfoConfig.Builder builder) { + builder.role(getNodeRole()) + .hostname(getHostName()); + } + + private String getNodeRole() { + String hostConfigId = getHost().getConfigId(); + if (! isHostedVespa) return hostConfigId; + return getHostResource().spec().membership() + .map(ClusterMembership::stringValue) + .orElse(hostConfigId); + } + + private void addMetricsProxyComponent(Class<?> componentClass) { addSimpleComponent(componentClass.getName(), null, METRICS_PROXY_BUNDLE_NAME); } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainerCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainerCluster.java index fcc6c3279de..071666b5bc7 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainerCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainerCluster.java @@ -119,11 +119,16 @@ public class MetricsProxyContainerCluster extends ContainerCluster<MetricsProxyC } private void addHttpHandler(Class<? extends ThreadedHttpRequestHandler> clazz, String bindingPath) { + Handler<AbstractConfigProducer<?>> metricsHandler = createMetricsHandler(clazz, bindingPath); + addComponent(metricsHandler); + } + + static Handler<AbstractConfigProducer<?>> createMetricsHandler(Class<? extends ThreadedHttpRequestHandler> clazz, String bindingPath) { Handler<AbstractConfigProducer<?>> metricsHandler = new Handler<>( new ComponentModel(clazz.getName(), null, METRICS_PROXY_BUNDLE_NAME, null)); metricsHandler.addServerBindings("http://*" + bindingPath, "http://*" + bindingPath + "/*"); - addComponent(metricsHandler); + return metricsHandler; } @Override diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/VespaMetricSet.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/VespaMetricSet.java index 38f8cd67601..617e83bcc8e 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/VespaMetricSet.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/VespaMetricSet.java @@ -527,8 +527,10 @@ public class VespaMetricSet { metrics.add(new Metric("vds.visitor.allthreads.queuesize.count.sum")); metrics.add(new Metric("vds.visitor.allthreads.queuesize.count.count")); metrics.add(new Metric("vds.visitor.allthreads.queuesize.count.average")); // TODO: Remove in Vespa 8 - metrics.add(new Metric("vds.visitor.allthreads.completed.sum.average")); + metrics.add(new Metric("vds.visitor.allthreads.completed.sum.average")); // TODO: Remove in Vespa 8 + metrics.add(new Metric("vds.visitor.allthreads.completed.sum.rate")); metrics.add(new Metric("vds.visitor.allthreads.created.sum.rate")); + metrics.add(new Metric("vds.visitor.allthreads.failed.sum.rate")); metrics.add(new Metric("vds.visitor.allthreads.averagemessagesendtime.sum.max")); metrics.add(new Metric("vds.visitor.allthreads.averagemessagesendtime.sum.sum")); metrics.add(new Metric("vds.visitor.allthreads.averagemessagesendtime.sum.count")); @@ -537,19 +539,27 @@ public class VespaMetricSet { metrics.add(new Metric("vds.visitor.allthreads.averageprocessingtime.sum.sum")); metrics.add(new Metric("vds.visitor.allthreads.averageprocessingtime.sum.count")); metrics.add(new Metric("vds.visitor.allthreads.averageprocessingtime.sum.average")); // TODO: Remove in Vespa 8 - + + metrics.add(new Metric("vds.filestor.alldisks.allthreads.put.sum.count.rate")); + metrics.add(new Metric("vds.filestor.alldisks.allthreads.put.sum.failed.rate")); metrics.add(new Metric("vds.filestor.alldisks.allthreads.put.sum.latency.max")); metrics.add(new Metric("vds.filestor.alldisks.allthreads.put.sum.latency.sum")); metrics.add(new Metric("vds.filestor.alldisks.allthreads.put.sum.latency.count")); metrics.add(new Metric("vds.filestor.alldisks.allthreads.put.sum.latency.average")); // TODO: Remove in Vespa 8 + metrics.add(new Metric("vds.filestor.alldisks.allthreads.remove.sum.count.rate")); + metrics.add(new Metric("vds.filestor.alldisks.allthreads.remove.sum.failed.rate")); metrics.add(new Metric("vds.filestor.alldisks.allthreads.remove.sum.latency.max")); metrics.add(new Metric("vds.filestor.alldisks.allthreads.remove.sum.latency.sum")); metrics.add(new Metric("vds.filestor.alldisks.allthreads.remove.sum.latency.count")); metrics.add(new Metric("vds.filestor.alldisks.allthreads.remove.sum.latency.average")); // TODO: Remove in Vespa 8 + metrics.add(new Metric("vds.filestor.alldisks.allthreads.get.sum.count.rate")); + metrics.add(new Metric("vds.filestor.alldisks.allthreads.get.sum.failed.rate")); metrics.add(new Metric("vds.filestor.alldisks.allthreads.get.sum.latency.max")); metrics.add(new Metric("vds.filestor.alldisks.allthreads.get.sum.latency.sum")); metrics.add(new Metric("vds.filestor.alldisks.allthreads.get.sum.latency.count")); metrics.add(new Metric("vds.filestor.alldisks.allthreads.get.sum.latency.average")); // TODO: Remove in Vespa 8 + metrics.add(new Metric("vds.filestor.alldisks.allthreads.update.sum.count.rate")); + metrics.add(new Metric("vds.filestor.alldisks.allthreads.update.sum.failed.rate")); metrics.add(new Metric("vds.filestor.alldisks.allthreads.update.sum.latency.max")); metrics.add(new Metric("vds.filestor.alldisks.allthreads.update.sum.latency.sum")); metrics.add(new Metric("vds.filestor.alldisks.allthreads.update.sum.latency.count")); @@ -569,13 +579,13 @@ public class VespaMetricSet { metrics.add(new Metric("vds.filestor.alldisks.allthreads.splitbuckets.count.rate")); metrics.add(new Metric("vds.filestor.alldisks.allthreads.joinbuckets.count.rate")); metrics.add(new Metric("vds.filestor.alldisks.allthreads.deletebuckets.count.rate")); + metrics.add(new Metric("vds.filestor.alldisks.allthreads.deletebuckets.failed.rate")); metrics.add(new Metric("vds.filestor.alldisks.allthreads.deletebuckets.latency.max")); metrics.add(new Metric("vds.filestor.alldisks.allthreads.deletebuckets.latency.sum")); metrics.add(new Metric("vds.filestor.alldisks.allthreads.deletebuckets.latency.count")); metrics.add(new Metric("vds.filestor.alldisks.allthreads.deletebuckets.latency.average")); // TODO: Remove in Vespa 8 metrics.add(new Metric("vds.filestor.alldisks.allthreads.setbucketstates.count.rate")); - //Distributor metrics.add(new Metric("vds.idealstate.buckets_rechecking.average")); metrics.add(new Metric("vds.idealstate.idealstate_diff.average")); diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/search/QueryProfiles.java b/config-model/src/main/java/com/yahoo/vespa/model/container/search/QueryProfiles.java index 0abb0803405..0a9618e7b08 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/search/QueryProfiles.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/search/QueryProfiles.java @@ -232,8 +232,7 @@ public class QueryProfiles implements Serializable, QueryProfilesConfig.Producer return propB; } - private QueryProfilesConfig.Queryprofile.Queryprofilevariant.Property.Builder createVariantPropertyFieldConfig( - String fullName, Object value) { + private QueryProfilesConfig.Queryprofile.Queryprofilevariant.Property.Builder createVariantPropertyFieldConfig(String fullName, Object value) { QueryProfilesConfig.Queryprofile.Queryprofilevariant.Property.Builder propB = new QueryProfilesConfig.Queryprofile.Queryprofilevariant.Property.Builder(); if (value instanceof SubstituteString) value=value.toString(); // Send only types understood by configBuilder downwards @@ -251,7 +250,7 @@ public class QueryProfiles implements Serializable, QueryProfilesConfig.Producer qtB.matchaspath(true); for (QueryProfileType inherited : profileType.inherited()) qtB.inherit(inherited.getId().stringValue()); - List<FieldDescription> fields=new ArrayList<>(profileType.declaredFields().values()); + List<FieldDescription> fields = new ArrayList<>(profileType.declaredFields().values()); Collections.sort(fields); for (FieldDescription field : fields) qtB.field(createConfig(field)); @@ -260,22 +259,20 @@ public class QueryProfiles implements Serializable, QueryProfilesConfig.Producer private QueryProfilesConfig.Queryprofiletype.Field.Builder createConfig(FieldDescription field) { QueryProfilesConfig.Queryprofiletype.Field.Builder fB = new QueryProfilesConfig.Queryprofiletype.Field.Builder(); - fB. - name(field.getName()). - type(field.getType().stringValue()); + fB.name(field.getName()).type(field.getType().stringValue()); if ( ! field.isOverridable()) fB.overridable(false); if (field.isMandatory()) fB.mandatory(true); - String aliases=toSpaceSeparatedString(field.getAliases()); - if (!aliases.isEmpty()) + String aliases = toSpaceSeparatedString(field.getAliases()); + if ( ! aliases.isEmpty()) fB.alias(aliases); return fB; } public String toSpaceSeparatedString(List<String> list) { - StringBuilder b=new StringBuilder(); - for (Iterator<String> i=list.iterator(); i.hasNext(); ) { + StringBuilder b = new StringBuilder(); + for (Iterator<String> i = list.iterator(); i.hasNext(); ) { b.append(i.next()); if (i.hasNext()) b.append(" "); @@ -290,10 +287,7 @@ public class QueryProfiles implements Serializable, QueryProfilesConfig.Producer } } - /** - * The config produced by this - * @return query profiles config - */ + /** Returns the config produced by this */ public QueryProfilesConfig getConfig() { QueryProfilesConfig.Builder qB = new QueryProfilesConfig.Builder(); getConfig(qB); diff --git a/config-model/src/main/resources/schema/content.rnc b/config-model/src/main/resources/schema/content.rnc index 8b3868c132e..bd09902f929 100644 --- a/config-model/src/main/resources/schema/content.rnc +++ b/config-model/src/main/resources/schema/content.rnc @@ -57,6 +57,9 @@ PersistenceThread = element thread { ## Declare which storage threads each disk should have. PersistenceThreads = element persistence-threads { + ## The number of threads to create + attribute count { xsd:integer }? & + ## All of the below settings are deprecated. ## Operations with priority worse than this can be blocked attribute highest-priority-to-block { xsd:string } ? & ## Operations with priority better than this can block others diff --git a/config-model/src/test/derived/neuralnet/query-profiles.cfg b/config-model/src/test/derived/neuralnet/query-profiles.cfg new file mode 100644 index 00000000000..817640a89fd --- /dev/null +++ b/config-model/src/test/derived/neuralnet/query-profiles.cfg @@ -0,0 +1,54 @@ +queryprofile[].id "default" +queryprofile[].type "DefaultQueryProfileType" +queryprofiletype[].id "DefaultQueryProfileType" +queryprofiletype[].strict false +queryprofiletype[].matchaspath false +queryprofiletype[].inherit[] "native" +queryprofiletype[].field[].name "ranking" +queryprofiletype[].field[].type "query-profile:ranking_0_0" +queryprofiletype[].field[].overridable true +queryprofiletype[].field[].mandatory false +queryprofiletype[].field[].alias "" +queryprofiletype[].id "ranking_0_0" +queryprofiletype[].strict false +queryprofiletype[].matchaspath false +queryprofiletype[].inherit[] "ranking" +queryprofiletype[].field[].name "features" +queryprofiletype[].field[].type "query-profile:features_0_1" +queryprofiletype[].field[].overridable true +queryprofiletype[].field[].mandatory false +queryprofiletype[].field[].alias "rankfeature" +queryprofiletype[].id "features_0_1" +queryprofiletype[].strict false +queryprofiletype[].matchaspath false +queryprofiletype[].field[].name "query(W_0)" +queryprofiletype[].field[].type "tensor(hidden[9],x[9])" +queryprofiletype[].field[].overridable true +queryprofiletype[].field[].mandatory false +queryprofiletype[].field[].alias "" +queryprofiletype[].field[].name "query(W_1)" +queryprofiletype[].field[].type "tensor(hidden[9],out[9])" +queryprofiletype[].field[].overridable true +queryprofiletype[].field[].mandatory false +queryprofiletype[].field[].alias "" +queryprofiletype[].field[].name "query(W_out)" +queryprofiletype[].field[].type "tensor(out[9])" +queryprofiletype[].field[].overridable true +queryprofiletype[].field[].mandatory false +queryprofiletype[].field[].alias "" +queryprofiletype[].field[].name "query(b_0)" +queryprofiletype[].field[].type "tensor(hidden[9])" +queryprofiletype[].field[].overridable true +queryprofiletype[].field[].mandatory false +queryprofiletype[].field[].alias "" +queryprofiletype[].field[].name "query(b_1)" +queryprofiletype[].field[].type "tensor(out[9])" +queryprofiletype[].field[].overridable true +queryprofiletype[].field[].mandatory false +queryprofiletype[].field[].alias "" +queryprofiletype[].field[].name "query(b_out)" +queryprofiletype[].field[].type "tensor(out[1])" +queryprofiletype[].field[].overridable true +queryprofiletype[].field[].mandatory false +queryprofiletype[].field[].alias "" +enableGroupingSessionCache true diff --git a/config-model/src/test/derived/neuralnet/query-profiles/types/DefaultQueryProfileType.xml b/config-model/src/test/derived/neuralnet/query-profiles/types/DefaultQueryProfileType.xml index e74152638fb..42336098a9a 100644 --- a/config-model/src/test/derived/neuralnet/query-profiles/types/DefaultQueryProfileType.xml +++ b/config-model/src/test/derived/neuralnet/query-profiles/types/DefaultQueryProfileType.xml @@ -1,5 +1,5 @@ <!-- Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. --> -<query-profile-type id="DefaultQueryProfileType"> +<query-profile-type id="DefaultQueryProfileType" inherits="native"> <field name="ranking.features.query(W_0)" type="tensor(x[9],hidden[9])" /> <field name="ranking.features.query(b_0)" type="tensor(hidden[9])" /> <field name="ranking.features.query(W_1)" type="tensor(hidden[9],out[9])" /> diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/derived/AbstractExportingTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/derived/AbstractExportingTestCase.java index d67df3a5239..8ea53172200 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/derived/AbstractExportingTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/derived/AbstractExportingTestCase.java @@ -10,6 +10,8 @@ import com.yahoo.searchdefinition.parser.ParseException; import ai.vespa.rankingexpression.importer.configmodelview.ImportedMlModels; import com.yahoo.vespa.configmodel.producers.DocumentManager; import com.yahoo.vespa.configmodel.producers.DocumentTypes; +import com.yahoo.vespa.model.container.search.QueryProfiles; +import com.yahoo.vespa.model.test.utils.DeployLoggerStub; import java.io.File; import java.io.IOException; @@ -53,6 +55,7 @@ public abstract class AbstractExportingTestCase extends SearchDefinitionTestCase String path = exportConfig(name, config); DerivedConfiguration.exportDocuments(new DocumentManager().produce(builder.getModel(), new DocumentmanagerConfig.Builder()), path); DerivedConfiguration.exportDocuments(new DocumentTypes().produce(builder.getModel(), new DocumenttypesConfig.Builder()), path); + DerivedConfiguration.exportQueryProfiles(builder.getQueryProfileRegistry(), path); return config; } diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/derived/NeuralNetTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/derived/NeuralNetTestCase.java index b299c7fa299..a6171901d2d 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/derived/NeuralNetTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/derived/NeuralNetTestCase.java @@ -1,16 +1,34 @@ // Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.searchdefinition.derived; +import com.yahoo.search.Query; +import com.yahoo.search.query.profile.QueryProfileRegistry; +import com.yahoo.search.query.profile.compiled.CompiledQueryProfileRegistry; +import com.yahoo.search.query.profile.config.QueryProfileConfigurer; import com.yahoo.searchdefinition.parser.ParseException; +import com.yahoo.vespa.model.container.search.QueryProfiles; import org.junit.Test; import java.io.IOException; +import com.yahoo.component.ComponentId; + +import static org.junit.Assert.assertEquals; + public class NeuralNetTestCase extends AbstractExportingTestCase { @Test public void testNeuralNet() throws IOException, ParseException { - assertCorrectDeriving("neuralnet"); + ComponentId.resetGlobalCountersForTests(); + DerivedConfiguration c = assertCorrectDeriving("neuralnet"); + + // Verify that query profiles end up correct when passed through the same intermediate forms as a full system + CompiledQueryProfileRegistry queryProfiles = + QueryProfileConfigurer.createFromConfig(new QueryProfiles(c.getQueryProfiles(), (level, message) -> {}).getConfig()).compile(); + Query q = new Query("?test=foo&ranking.features.query(b_1)=[1,2,3,4,5,6,7,8,9]", + queryProfiles.getComponent("default")); + assertEquals("tensor(out[9]):[1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0, 9.0]", + q.properties().get("ranking.features.query(b_1)").toString()); } } diff --git a/config-model/src/test/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainerTest.java b/config-model/src/test/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainerTest.java index 621cebd6246..eddad6fce89 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainerTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainerTest.java @@ -1,6 +1,7 @@ // Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.model.admin.metricsproxy; +import ai.vespa.metricsproxy.http.metrics.NodeInfoConfig; import ai.vespa.metricsproxy.metric.dimensions.NodeDimensionsConfig; import ai.vespa.metricsproxy.metric.dimensions.PublicDimensions; import ai.vespa.metricsproxy.rpc.RpcConnectorConfig; @@ -10,10 +11,10 @@ import com.yahoo.vespa.model.test.VespaModelTester; import org.junit.Test; import static com.yahoo.config.model.api.container.ContainerServiceType.METRICS_PROXY_CONTAINER; -import static com.yahoo.vespa.model.admin.metricsproxy.MetricsProxyModelTester.CLUSTER_CONFIG_ID; import static com.yahoo.vespa.model.admin.metricsproxy.MetricsProxyModelTester.CONTAINER_CONFIG_ID; import static com.yahoo.vespa.model.admin.metricsproxy.MetricsProxyModelTester.TestMode.hosted; import static com.yahoo.vespa.model.admin.metricsproxy.MetricsProxyModelTester.TestMode.self_hosted; +import static com.yahoo.vespa.model.admin.metricsproxy.MetricsProxyModelTester.containerConfigId; import static com.yahoo.vespa.model.admin.metricsproxy.MetricsProxyModelTester.getModel; import static com.yahoo.vespa.model.admin.metricsproxy.MetricsProxyModelTester.getNodeDimensionsConfig; @@ -90,13 +91,28 @@ public class MetricsProxyContainerTest { String services = servicesWithContent(); VespaModel hostedModel = getModel(services, hosted); assertEquals(1, hostedModel.getHosts().size()); - String configId = CLUSTER_CONFIG_ID + "/" + hostedModel.getHosts().iterator().next().getHostname(); + String configId = containerConfigId(hostedModel, hosted); NodeDimensionsConfig config = getNodeDimensionsConfig(hostedModel, configId); assertEquals("content", config.dimensions(PublicDimensions.INTERNAL_CLUSTER_TYPE)); assertEquals("my-content", config.dimensions(PublicDimensions.INTERNAL_CLUSTER_ID)); } + @Test + public void metrics_v2_handler_is_set_up_with_node_info_config() { + String services = servicesWithContent(); + VespaModel hostedModel = getModel(services, hosted); + + var container = (MetricsProxyContainer)hostedModel.id2producer().get(containerConfigId(hostedModel, hosted)); + var handlers = container.getHandlers().getComponents(); + + assertEquals(1, handlers.size()); + var metricsV2Handler = handlers.iterator().next(); + + NodeInfoConfig config = hostedModel.getConfig(NodeInfoConfig.class, metricsV2Handler.getConfigId()); + assertTrue(config.role().startsWith("content/my-content/0/")); + assertTrue(config.hostname().startsWith("node-1-3-9-")); + } @Test public void vespa_services_config_has_all_services() { diff --git a/config-model/src/test/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyModelTester.java b/config-model/src/test/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyModelTester.java index f3140aafdaf..7cbc9db5eb2 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyModelTester.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyModelTester.java @@ -49,7 +49,7 @@ class MetricsProxyModelTester { return tester.createModel(servicesXml, true); } - static String configId(VespaModel model, MetricsProxyModelTester.TestMode mode) { + static String containerConfigId(VespaModel model, MetricsProxyModelTester.TestMode mode) { return (mode == hosted) ? CLUSTER_CONFIG_ID + "/" + model.getHosts().iterator().next().getHostname() : CONTAINER_CONFIG_ID; diff --git a/config-model/src/test/java/com/yahoo/vespa/model/content/StorageClusterTest.java b/config-model/src/test/java/com/yahoo/vespa/model/content/StorageClusterTest.java index 21c384dfc69..883d8b89765 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/content/StorageClusterTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/content/StorageClusterTest.java @@ -112,11 +112,7 @@ public class StorageClusterTest { "<cluster id=\"bees\">\n" + " <documents/>" + " <tuning>\n" + - " <persistence-threads>\n" + - " <thread lowest-priority=\"VERY_LOW\" count=\"2\"/>\n" + - " <thread lowest-priority=\"VERY_HIGH\" count=\"1\"/>\n" + - " <thread count=\"1\"/>\n" + - " </persistence-threads>\n" + + " <persistence-threads count=\"7\"/>\n" + " </tuning>\n" + " <group>" + " <node distribution-key=\"0\" hostalias=\"mockhost\"/>" + @@ -130,6 +126,44 @@ public class StorageClusterTest { stc.getConfig(builder); StorFilestorConfig config = new StorFilestorConfig(builder); + assertEquals(7, config.num_threads()); + assertEquals(false, config.enable_multibit_split_optimalization()); + } + { + assertEquals(1, stc.getChildren().size()); + StorageNode sn = stc.getChildren().values().iterator().next(); + StorFilestorConfig.Builder builder = new StorFilestorConfig.Builder(); + sn.getConfig(builder); + StorFilestorConfig config = new StorFilestorConfig(builder); + assertEquals(7, config.num_threads()); + } + } + + @Test + public void testPersistenceThreadsOld() throws Exception { + + StorageCluster stc = parse( + "<cluster id=\"bees\">\n" + + " <documents/>" + + " <tuning>\n" + + " <persistence-threads>\n" + + " <thread lowest-priority=\"VERY_LOW\" count=\"2\"/>\n" + + " <thread lowest-priority=\"VERY_HIGH\" count=\"1\"/>\n" + + " <thread count=\"1\"/>\n" + + " </persistence-threads>\n" + + " </tuning>\n" + + " <group>" + + " <node distribution-key=\"0\" hostalias=\"mockhost\"/>" + + " </group>" + + "</cluster>", + new Flavor(new FlavorsConfig.Flavor.Builder().name("test-flavor").minCpuCores(9).build()) + ); + + { + StorFilestorConfig.Builder builder = new StorFilestorConfig.Builder(); + stc.getConfig(builder); + StorFilestorConfig config = new StorFilestorConfig(builder); + assertEquals(4, config.num_threads()); assertEquals(false, config.enable_multibit_split_optimalization()); } @@ -161,7 +195,7 @@ public class StorageClusterTest { StorFilestorConfig.Builder builder = new StorFilestorConfig.Builder(); stc.getConfig(builder); StorFilestorConfig config = new StorFilestorConfig(builder); - assertEquals(6, config.num_threads()); + assertEquals(8, config.num_threads()); } { assertEquals(1, stc.getChildren().size()); diff --git a/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/RpcConfigSourceClient.java b/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/RpcConfigSourceClient.java index d809a3c97ed..ee843088086 100644 --- a/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/RpcConfigSourceClient.java +++ b/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/RpcConfigSourceClient.java @@ -69,7 +69,7 @@ class RpcConfigSourceClient implements ConfigSourceClient { private Map<ConfigSourceSet, JRTConfigRequester> createRequesterPool(ConfigSourceSet ccs, TimingValues timingValues) { Map<ConfigSourceSet, JRTConfigRequester> ret = new HashMap<>(); if (ccs.getSources().isEmpty()) return ret; // unit test, just skip creating any requester - ret.put(ccs, JRTConfigRequester.get(new JRTConnectionPool(ccs), timingValues)); + ret.put(ccs, new JRTConfigRequester(new JRTConnectionPool(ccs), timingValues)); return ret; } diff --git a/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/filedistribution/FileDistributionRpcServer.java b/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/filedistribution/FileDistributionRpcServer.java index c6f33a29410..7db9761d86a 100644 --- a/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/filedistribution/FileDistributionRpcServer.java +++ b/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/filedistribution/FileDistributionRpcServer.java @@ -35,7 +35,7 @@ class FileDistributionRpcServer { private final Supervisor supervisor; private final FileDownloader downloader; - private final ExecutorService rpcDownloadExecutor = Executors.newFixedThreadPool(Runtime.getRuntime().availableProcessors(), + private final ExecutorService rpcDownloadExecutor = Executors.newFixedThreadPool(Math.max(8, Runtime.getRuntime().availableProcessors()), new DaemonThreadFactory("Rpc executor")); FileDistributionRpcServer(Supervisor supervisor, FileDownloader downloader) { diff --git a/config-proxy/src/main/sh/vespa-config-ctl.sh b/config-proxy/src/main/sh/vespa-config-ctl.sh index a17f0abed92..3998d4f69d6 100755 --- a/config-proxy/src/main/sh/vespa-config-ctl.sh +++ b/config-proxy/src/main/sh/vespa-config-ctl.sh @@ -113,7 +113,7 @@ case $1 in if [ "$userargs" == "" ]; then userargs=$services__jvmargs_configproxy fi - jvmopts="-Xms32M -Xmx256M -XX:ThreadStackSize=256 -XX:MaxJavaStackTraceDepth=1000000" + jvmopts="-Xms32M -Xmx256M -XX:CompressedClassSpaceSize=32m -XX:MaxDirectMemorySize=32m -XX:ThreadStackSize=256 -XX:MaxJavaStackTraceDepth=1000" VESPA_SERVICE_NAME=configproxy export VESPA_SERVICE_NAME @@ -122,6 +122,7 @@ case $1 in java ${jvmopts} \ -XX:+ExitOnOutOfMemoryError $(getJavaOptionsIPV46) \ -Dproxyconfigsources="${configsources}" ${userargs} \ + -XX:ActiveProcessorCount=2 \ -cp $cp com.yahoo.vespa.config.proxy.ProxyServer 19090 echo "Waiting for config proxy to start" diff --git a/config/src/main/java/com/yahoo/config/subscription/impl/GenericConfigSubscriber.java b/config/src/main/java/com/yahoo/config/subscription/impl/GenericConfigSubscriber.java index 324546230d9..8c0a8f27555 100644 --- a/config/src/main/java/com/yahoo/config/subscription/impl/GenericConfigSubscriber.java +++ b/config/src/main/java/com/yahoo/config/subscription/impl/GenericConfigSubscriber.java @@ -24,7 +24,7 @@ public class GenericConfigSubscriber extends ConfigSubscriber { * Constructs a new subscriber using the given pool of requesters (JRTConfigRequester holds 1 connection which in * turn is subject to failover across the elems in the source set.) * The behaviour is undefined if the map key is different from the source set the requester was built with. - * See also {@link JRTConfigRequester#get(com.yahoo.vespa.config.ConnectionPool, com.yahoo.vespa.config.TimingValues)} + * See also {@link JRTConfigRequester#JRTConfigRequester(com.yahoo.vespa.config.ConnectionPool, com.yahoo.vespa.config.TimingValues)} * * @param requesters a map from config source set to config requester */ @@ -32,10 +32,6 @@ public class GenericConfigSubscriber extends ConfigSubscriber { this.requesters = requesters; } - public GenericConfigSubscriber() { - super(); - } - /** * Subscribes to config without using the class. For internal use in config proxy. * diff --git a/config/src/main/java/com/yahoo/config/subscription/impl/JRTConfigRequester.java b/config/src/main/java/com/yahoo/config/subscription/impl/JRTConfigRequester.java index e9daafb779c..989d8c6c8de 100644 --- a/config/src/main/java/com/yahoo/config/subscription/impl/JRTConfigRequester.java +++ b/config/src/main/java/com/yahoo/config/subscription/impl/JRTConfigRequester.java @@ -52,20 +52,11 @@ public class JRTConfigRequester implements RequestWaiter { /** * Returns a new requester - * @param connectionPool The connectionPool to use - * @param timingValues The timing values - * @return new requester object - */ - public static JRTConfigRequester get(ConnectionPool connectionPool, TimingValues timingValues) { - return new JRTConfigRequester(connectionPool, timingValues); - } - - /** - * New requester - * @param connectionPool the connectionPool this requester should use - * @param timingValues timeouts and delays used when sending JRT config requests + * + * @param connectionPool the connectionPool this requester should use + * @param timingValues timeouts and delays used when sending JRT config requests */ - JRTConfigRequester(ConnectionPool connectionPool, TimingValues timingValues) { + public JRTConfigRequester(ConnectionPool connectionPool, TimingValues timingValues) { this.connectionPool = connectionPool; this.timingValues = timingValues; } diff --git a/config/src/main/java/com/yahoo/vespa/config/TimingValues.java b/config/src/main/java/com/yahoo/vespa/config/TimingValues.java index 113d561bf87..780cf657009 100644 --- a/config/src/main/java/com/yahoo/vespa/config/TimingValues.java +++ b/config/src/main/java/com/yahoo/vespa/config/TimingValues.java @@ -38,7 +38,6 @@ public class TimingValues { long configuredErrorDelay, long fixedDelay, int maxDelayMultiplier) { - this.successTimeout = successTimeout; this.errorTimeout = errorTimeout; this.initialTimeout = initialTimeout; @@ -51,15 +50,14 @@ public class TimingValues { } private TimingValues(long successTimeout, - long errorTimeout, - long initialTimeout, - long subscribeTimeout, - long unconfiguredDelay, - long configuredErrorDelay, - long fixedDelay, - int maxDelayMultiplier, - Random rand) { - + long errorTimeout, + long initialTimeout, + long subscribeTimeout, + long unconfiguredDelay, + long configuredErrorDelay, + long fixedDelay, + int maxDelayMultiplier, + Random rand) { this.successTimeout = successTimeout; this.errorTimeout = errorTimeout; this.initialTimeout = initialTimeout; @@ -71,18 +69,6 @@ public class TimingValues { this.rand = rand; } - public TimingValues(TimingValues tv) { - this(tv.successTimeout, - tv.errorTimeout, - tv.initialTimeout, - tv.subscribeTimeout, - tv.unconfiguredDelay, - tv.configuredErrorDelay, - tv.fixedDelay, - tv.maxDelayMultiplier, - tv.getRandom()); - } - public TimingValues(TimingValues tv, Random random) { this(tv.successTimeout, tv.errorTimeout, diff --git a/config/src/test/java/com/yahoo/config/subscription/GenericConfigSubscriberTest.java b/config/src/test/java/com/yahoo/config/subscription/GenericConfigSubscriberTest.java index 08d215670db..e9dc9cf7b98 100644 --- a/config/src/test/java/com/yahoo/config/subscription/GenericConfigSubscriberTest.java +++ b/config/src/test/java/com/yahoo/config/subscription/GenericConfigSubscriberTest.java @@ -1,7 +1,10 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.config.subscription; -import java.util.*; +import java.util.HashMap; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; import com.yahoo.config.subscription.impl.GenericConfigHandle; import com.yahoo.config.subscription.impl.GenericConfigSubscriber; @@ -14,7 +17,10 @@ import com.yahoo.vespa.config.protocol.CompressionType; import org.junit.Test; import static org.hamcrest.CoreMatchers.is; -import static org.junit.Assert.*; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertThat; +import static org.junit.Assert.assertTrue; /** * @@ -28,9 +34,9 @@ public class GenericConfigSubscriberTest { public void testSubscribeGeneric() { Map<ConfigSourceSet, JRTConfigRequester> requesters = new HashMap<>(); ConfigSourceSet sourceSet = new ConfigSourceSet("blabla"); - requesters.put(sourceSet, JRTConfigRequester.get(new MockConnection(), JRTConfigRequesterTest.getTestTimingValues())); + requesters.put(sourceSet, new JRTConfigRequester(new MockConnection(), JRTConfigRequesterTest.getTestTimingValues())); GenericConfigSubscriber sub = new GenericConfigSubscriber(requesters); - final List<String> defContent = Arrays.asList("myVal int"); + final List<String> defContent = List.of("myVal int"); GenericConfigHandle handle = sub.subscribe(new ConfigKey<>("simpletypes", "id", "config"), defContent, sourceSet, JRTConfigRequesterTest.getTestTimingValues()); assertTrue(sub.nextConfig()); assertTrue(handle.isChanged()); @@ -43,8 +49,8 @@ public class GenericConfigSubscriberTest { public void testGenericRequesterPooling() { ConfigSourceSet source1 = new ConfigSourceSet("tcp/foo:78"); ConfigSourceSet source2 = new ConfigSourceSet("tcp/bar:79"); - JRTConfigRequester req1 = JRTConfigRequester.get(new JRTConnectionPool(source1), JRTConfigRequesterTest.getTestTimingValues()); - JRTConfigRequester req2 = JRTConfigRequester.get(new JRTConnectionPool(source2), JRTConfigRequesterTest.getTestTimingValues()); + JRTConfigRequester req1 = new JRTConfigRequester(new JRTConnectionPool(source1), JRTConfigRequesterTest.getTestTimingValues()); + JRTConfigRequester req2 = new JRTConfigRequester(new JRTConnectionPool(source2), JRTConfigRequesterTest.getTestTimingValues()); Map<ConfigSourceSet, JRTConfigRequester> requesters = new LinkedHashMap<>(); requesters.put(source1, req1); requesters.put(source2, req2); @@ -55,19 +61,23 @@ public class GenericConfigSubscriberTest { @Test(expected=UnsupportedOperationException.class) public void testOverriddenSubscribeInvalid1() { - GenericConfigSubscriber sub = new GenericConfigSubscriber(); - sub.subscribe(null, null); + createSubscriber().subscribe(null, null); } @Test(expected=UnsupportedOperationException.class) public void testOverriddenSubscribeInvalid2() { - GenericConfigSubscriber sub = new GenericConfigSubscriber(); - sub.subscribe(null, null, 0L); + createSubscriber().subscribe(null, null, 0L); } @Test(expected=UnsupportedOperationException.class) public void testOverriddenSubscribeInvalid3() { - GenericConfigSubscriber sub = new GenericConfigSubscriber(); - sub.subscribe(null, null, ""); + createSubscriber().subscribe(null, null, ""); } + + private GenericConfigSubscriber createSubscriber() { + return new GenericConfigSubscriber(Map.of( + new ConfigSourceSet("blabla"), + new JRTConfigRequester(new MockConnection(), JRTConfigRequesterTest.getTestTimingValues()))); + } + } diff --git a/config/src/test/java/com/yahoo/vespa/config/TimingValuesTest.java b/config/src/test/java/com/yahoo/vespa/config/TimingValuesTest.java deleted file mode 100644 index 0be5e98ff0a..00000000000 --- a/config/src/test/java/com/yahoo/vespa/config/TimingValuesTest.java +++ /dev/null @@ -1,25 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.config; - -import org.junit.Test; - -import static org.hamcrest.CoreMatchers.is; -import static org.hamcrest.CoreMatchers.not; -import static org.junit.Assert.assertThat; - -/** - * Note: Most of the functionality is tested implicitly by other tests - * - * @author hmusum - */ -public class TimingValuesTest { - - @Test - public void basic() { - TimingValues tv = new TimingValues(); - TimingValues tv2 = new TimingValues(1L, 1L, 1L, 1L, 1L, 1L, 1L, 1); - assertThat(tv.getRandom(), is(not(tv2.getRandom()))); - TimingValues copy = new TimingValues(tv2); - assertThat(copy.toString(), is(tv2.toString())); // No equals method, just using toString to compare - } -} diff --git a/config/src/test/java/com/yahoo/vespa/config/protocol/JRTConfigRequestV3Test.java b/config/src/test/java/com/yahoo/vespa/config/protocol/JRTConfigRequestV3Test.java index ac45554b065..4d1ba2f8793 100644 --- a/config/src/test/java/com/yahoo/vespa/config/protocol/JRTConfigRequestV3Test.java +++ b/config/src/test/java/com/yahoo/vespa/config/protocol/JRTConfigRequestV3Test.java @@ -195,7 +195,7 @@ public class JRTConfigRequestV3Test { }); ConfigSourceSet src = new ConfigSourceSet(); - ConfigSubscriber subscriber = new GenericConfigSubscriber(Collections.singletonMap(src, JRTConfigRequester.get(connection, new TimingValues()))); + ConfigSubscriber subscriber = new GenericConfigSubscriber(Collections.singletonMap(src, new JRTConfigRequester(connection, new TimingValues()))); JRTConfigSubscription<SimpletypesConfig> sub = new JRTConfigSubscription<>(new ConfigKey<>(SimpletypesConfig.class, configId), subscriber, src, new TimingValues()); sub.subscribe(120_0000); assertTrue(sub.nextConfig(120_0000)); diff --git a/configdefinitions/src/vespa/lb-services.def b/configdefinitions/src/vespa/lb-services.def index 8d5e7015947..33c568061fe 100644 --- a/configdefinitions/src/vespa/lb-services.def +++ b/configdefinitions/src/vespa/lb-services.def @@ -7,7 +7,6 @@ namespace=cloud.config # Active rotation given as flag 'active' for a prod region in deployment.xml # Default true for now (since code in config-model to set it is not ready yet), should have no default value tenants{}.applications{}.activeRotation bool default=true -tenants{}.applications{}.use4443Upstream bool default=false tenants{}.applications{}.hosts{}.hostname string default="(unknownhostname)" tenants{}.applications{}.hosts{}.services{}.type string default="(noservicetype)" diff --git a/configdefinitions/src/vespa/stor-filestor.def b/configdefinitions/src/vespa/stor-filestor.def index c02a9018064..7816ee74fde 100644 --- a/configdefinitions/src/vespa/stor-filestor.def +++ b/configdefinitions/src/vespa/stor-filestor.def @@ -24,7 +24,7 @@ disk_operation_timeout int default=0 restart ## PERFORMANCE PARAMETERS ## Number of threads to use for each mountpoint. -num_threads int default=6 restart +num_threads int default=8 restart ## When merging, if we find more than this number of documents that exist on all ## of the same copies, send a separate apply bucket diff with these entries diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/model/LbServicesProducer.java b/configserver/src/main/java/com/yahoo/vespa/config/server/model/LbServicesProducer.java index 5f4895c5ae8..6366576e163 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/model/LbServicesProducer.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/model/LbServicesProducer.java @@ -9,10 +9,7 @@ import com.yahoo.config.model.api.ServiceInfo; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.TenantName; import com.yahoo.config.provision.Zone; -import com.yahoo.vespa.flags.BooleanFlag; -import com.yahoo.vespa.flags.FetchVector; import com.yahoo.vespa.flags.FlagSource; -import com.yahoo.vespa.flags.Flags; import java.util.Collections; import java.util.Comparator; @@ -35,12 +32,10 @@ public class LbServicesProducer implements LbServicesConfig.Producer { private final Map<TenantName, Set<ApplicationInfo>> models; private final Zone zone; - private final BooleanFlag use4443Upstream; public LbServicesProducer(Map<TenantName, Set<ApplicationInfo>> models, Zone zone, FlagSource flagSource) { this.models = models; this.zone = zone; - this.use4443Upstream = Flags.USE_4443_UPSTREAM.bindTo(flagSource); } @Override @@ -72,8 +67,6 @@ public class LbServicesProducer implements LbServicesConfig.Producer { private LbServicesConfig.Tenants.Applications.Builder getAppConfig(ApplicationInfo app) { LbServicesConfig.Tenants.Applications.Builder ab = new LbServicesConfig.Tenants.Applications.Builder(); ab.activeRotation(getActiveRotation(app)); - ab.use4443Upstream( - use4443Upstream.with(FetchVector.Dimension.APPLICATION_ID, app.getApplicationId().serializedForm()).value()); app.getModel().getHosts().stream() .sorted((a, b) -> a.getHostname().compareTo(b.getHostname())) .forEach(hostInfo -> ab.hosts(hostInfo.getHostname(), getHostsConfig(hostInfo))); diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/monitoring/Metrics.java b/configserver/src/main/java/com/yahoo/vespa/config/server/monitoring/Metrics.java index 87e9fa287a4..249c8639ea9 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/monitoring/Metrics.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/monitoring/Metrics.java @@ -3,6 +3,8 @@ package com.yahoo.vespa.config.server.monitoring; import com.google.inject.Inject; import com.yahoo.cloud.config.ZookeeperServerConfig; +import com.yahoo.component.AbstractComponent; +import com.yahoo.concurrent.DaemonThreadFactory; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.TenantName; import com.yahoo.container.jdisc.config.HealthMonitorConfig; @@ -14,14 +16,17 @@ import com.yahoo.statistics.Counter; import java.util.*; import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.ScheduledThreadPoolExecutor; +import java.util.concurrent.TimeUnit; /** - * Statistics for server. The statistics framework takes care of logging. + * Metrics for config server. The statistics framework takes care of logging. * * @author Harald Musum - * @since 4.2 */ -public class Metrics extends TimerTask implements MetricUpdaterFactory { +public class Metrics extends AbstractComponent implements MetricUpdaterFactory, Runnable { private static final java.util.logging.Logger log = java.util.logging.Logger.getLogger(Metrics.class.getName()); private static final String METRIC_REQUESTS = getMetricName("requests"); @@ -33,23 +38,34 @@ public class Metrics extends TimerTask implements MetricUpdaterFactory { private final Counter failedRequests; private final Counter procTimeCounter; private final Metric metric; - private final ZKMetricUpdater zkMetricUpdater; + private final Optional<ZKMetricUpdater> zkMetricUpdater; // TODO The map is the key for now private final Map<Map<String, String>, MetricUpdater> metricUpdaters = new ConcurrentHashMap<>(); - private final Timer timer = new Timer(); + private final Optional<ScheduledExecutorService> executorService; @Inject public Metrics(Metric metric, Statistics statistics, HealthMonitorConfig healthMonitorConfig, ZookeeperServerConfig zkServerConfig) { + this(metric, statistics, healthMonitorConfig, zkServerConfig, true); + } + + private Metrics(Metric metric, Statistics statistics, HealthMonitorConfig healthMonitorConfig, + ZookeeperServerConfig zkServerConfig, boolean createZkMetricUpdater) { this.metric = metric; requests = createCounter(METRIC_REQUESTS, statistics); failedRequests = createCounter(METRIC_FAILED_REQUESTS, statistics); procTimeCounter = createCounter("procTime", statistics); - log.log(LogLevel.DEBUG, "Metric update interval is " + healthMonitorConfig.snapshot_interval() + " seconds"); - long intervalMs = (long) (healthMonitorConfig.snapshot_interval() * 1000); - timer.scheduleAtFixedRate(this, 20000, intervalMs); - zkMetricUpdater = new ZKMetricUpdater(zkServerConfig, 19500, intervalMs); + if (createZkMetricUpdater) { + log.log(LogLevel.DEBUG, "Metric update interval is " + healthMonitorConfig.snapshot_interval() + " seconds"); + long intervalMs = (long) (healthMonitorConfig.snapshot_interval() * 1000); + executorService = Optional.of(new ScheduledThreadPoolExecutor(1, new DaemonThreadFactory("configserver-metrics"))); + executorService.get().scheduleAtFixedRate(this, 20000, intervalMs, TimeUnit.MILLISECONDS); + zkMetricUpdater = Optional.of(new ZKMetricUpdater(zkServerConfig, 19500, intervalMs)); + } else { + executorService = Optional.empty(); + zkMetricUpdater = Optional.empty(); + } } public static Metrics createTestMetrics() { @@ -58,14 +74,13 @@ public class Metrics extends TimerTask implements MetricUpdaterFactory { HealthMonitorConfig.Builder builder = new HealthMonitorConfig.Builder(); builder.snapshot_interval(60.0); ZookeeperServerConfig.Builder zkBuilder = new ZookeeperServerConfig.Builder().myid(1); - return new Metrics(metric, statistics, new HealthMonitorConfig(builder), new ZookeeperServerConfig(zkBuilder)); + return new Metrics(metric, statistics, new HealthMonitorConfig(builder), new ZookeeperServerConfig(zkBuilder), false); } private Counter createCounter(String name, Statistics statistics) { return new Counter(name, statistics, false); } - void incrementRequests(Metric.Context metricContext) { requests.increment(1); metric.add(METRIC_REQUESTS, 1, metricContext); @@ -126,8 +141,12 @@ public class Metrics extends TimerTask implements MetricUpdaterFactory { } } setRegularMetrics(); - zkMetricUpdater.getZKMetrics().forEach((attr, val) -> metric.set(attr, val, null)); - timer.purge(); + zkMetricUpdater.ifPresent(updater -> updater.getZKMetrics().forEach((attr, val) -> metric.set(attr, val, null))); + } + + public void deconstruct() { + executorService.ifPresent(ExecutorService::shutdown); + zkMetricUpdater.ifPresent(ZKMetricUpdater::shutdown); } private void setRegularMetrics() { diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/monitoring/ZKMetricUpdater.java b/configserver/src/main/java/com/yahoo/vespa/config/server/monitoring/ZKMetricUpdater.java index b2813be5456..62da6fcffbe 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/monitoring/ZKMetricUpdater.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/monitoring/ZKMetricUpdater.java @@ -2,6 +2,7 @@ package com.yahoo.vespa.config.server.monitoring; import com.yahoo.cloud.config.ZookeeperServerConfig; +import com.yahoo.concurrent.DaemonThreadFactory; import java.io.ByteArrayOutputStream; import java.io.IOException; @@ -17,6 +18,8 @@ import java.util.Timer; import java.util.TimerTask; import java.util.concurrent.ExecutionException; import java.util.concurrent.Future; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.ScheduledThreadPoolExecutor; import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeoutException; import java.util.concurrent.atomic.AtomicReference; @@ -26,7 +29,7 @@ import java.util.regex.Pattern; import static com.yahoo.vespa.config.server.monitoring.Metrics.getMetricName; -public class ZKMetricUpdater extends TimerTask { +public class ZKMetricUpdater implements Runnable { private static final Logger log = Logger.getLogger(ZKMetricUpdater.class.getName()); public static final String METRIC_ZK_ZNODES = getMetricName("zkZNodes"); @@ -35,19 +38,19 @@ public class ZKMetricUpdater extends TimerTask { public static final String METRIC_ZK_CONNECTIONS = getMetricName("zkConnections"); public static final String METRIC_ZK_OUTSTANDING_REQUESTS = getMetricName("zkOutstandingRequests"); - private final int CONNECTION_TIMEOUT_MS = 500; - private final int WRITE_TIMEOUT_MS = 250; - private final int READ_TIMEOUT_MS = 500; + private static final int CONNECTION_TIMEOUT_MS = 500; + private static final int WRITE_TIMEOUT_MS = 250; + private static final int READ_TIMEOUT_MS = 500; private AtomicReference<Map<String, Long>> zkMetrics = new AtomicReference<>(new HashMap<>()); - private final Timer timer = new Timer(); + private final ScheduledExecutorService executorService; private final int zkPort; public ZKMetricUpdater(ZookeeperServerConfig zkServerConfig, long delayMS, long intervalMS) { this.zkPort = zkServerConfig.clientPort(); - if (intervalMS > 0) { - timer.scheduleAtFixedRate(this, delayMS, intervalMS); - } + if (intervalMS <= 0 ) throw new IllegalArgumentException("interval must be positive, was " + intervalMS + " ms"); + this.executorService = new ScheduledThreadPoolExecutor(1, new DaemonThreadFactory("zkmetricupdater")); + this.executorService.scheduleAtFixedRate(this, delayMS, intervalMS, TimeUnit.MILLISECONDS); } private void setMetricAttribute(String attribute, long value, Map<String, Long> data) { @@ -74,7 +77,10 @@ public class ZKMetricUpdater extends TimerTask { public void run() { Optional<String> report = retrieveReport(); report.ifPresent(this::parseReport); - timer.purge(); + } + + public void shutdown() { + executorService.shutdown(); } private Optional<String> retrieveReport() { diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/monitoring/ZKMetricUpdaterTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/monitoring/ZKMetricUpdaterTest.java index ed089109759..607a2dca6c6 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/monitoring/ZKMetricUpdaterTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/monitoring/ZKMetricUpdaterTest.java @@ -57,12 +57,14 @@ public class ZKMetricUpdaterTest { assertThat(reportedMetrics.get(ZKMetricUpdater.METRIC_ZK_LATENCY_MAX), equalTo(1234L)); assertThat(reportedMetrics.get(ZKMetricUpdater.METRIC_ZK_OUTSTANDING_REQUESTS), equalTo(12L)); assertThat(reportedMetrics.get(ZKMetricUpdater.METRIC_ZK_ZNODES), equalTo(4L)); + + updater.shutdown(); } private ZKMetricUpdater buildUpdater() { ZookeeperServerConfig zkServerConfig = new ZookeeperServerConfig( new ZookeeperServerConfig.Builder().clientPort(serverPort).myid(12345)); - return new ZKMetricUpdater(zkServerConfig, 0, -1); + return new ZKMetricUpdater(zkServerConfig, 0, 100000); } private void setupTcpServer(Supplier<String> reportProvider) throws IOException { diff --git a/container-accesslogging/src/main/java/com/yahoo/container/logging/AccessLogEntry.java b/container-accesslogging/src/main/java/com/yahoo/container/logging/AccessLogEntry.java index b943c03f48f..d8085cc808b 100644 --- a/container-accesslogging/src/main/java/com/yahoo/container/logging/AccessLogEntry.java +++ b/container-accesslogging/src/main/java/com/yahoo/container/logging/AccessLogEntry.java @@ -2,6 +2,7 @@ package com.yahoo.container.logging; import com.yahoo.collections.ListMap; +import com.yahoo.yolean.trace.TraceNode; import javax.security.auth.x500.X500Principal; import java.net.InetAddress; @@ -69,6 +70,7 @@ public class AccessLogEntry { private X500Principal sslPrincipal; private String rawPath; private String rawQuery; + private TraceNode traceNode; private ListMap<String,String> keyValues=null; @@ -452,6 +454,18 @@ public class AccessLogEntry { } } + public void setTrace(TraceNode traceNode) { + synchronized (monitor) { + requireNull(this.traceNode); + this.traceNode = traceNode; + } + } + public TraceNode getTrace() { + synchronized (monitor) { + return traceNode; + } + } + @Override public String toString() { synchronized (monitor) { @@ -481,6 +495,7 @@ public class AccessLogEntry { ", sslPrincipal=" + sslPrincipal + ", rawPath='" + rawPath + '\'' + ", rawQuery='" + rawQuery + '\'' + + ", trace='" + traceNode + '\'' + ", keyValues=" + keyValues + '}'; } diff --git a/container-accesslogging/src/main/java/com/yahoo/container/logging/JSONFormatter.java b/container-accesslogging/src/main/java/com/yahoo/container/logging/JSONFormatter.java index 556b97ced62..ae794e5b60a 100644 --- a/container-accesslogging/src/main/java/com/yahoo/container/logging/JSONFormatter.java +++ b/container-accesslogging/src/main/java/com/yahoo/container/logging/JSONFormatter.java @@ -5,6 +5,7 @@ import com.fasterxml.jackson.core.JsonEncoding; import com.fasterxml.jackson.core.JsonFactory; import com.fasterxml.jackson.core.JsonGenerator; import com.fasterxml.jackson.databind.ObjectMapper; +import com.yahoo.yolean.trace.TraceNode; import java.io.ByteArrayOutputStream; import java.io.IOException; @@ -55,8 +56,7 @@ public class JSONFormatter { generator.writeStartObject(); generator.writeStringField("ip", accessLogEntry.getIpV4Address()); generator.writeNumberField("time", toTimestampInSeconds(accessLogEntry.getTimeStampMillis())); - generator.writeNumberField("duration", - durationAsSeconds(accessLogEntry.getDurationBetweenRequestResponseMillis())); + generator.writeNumberField("duration", durationAsSeconds(accessLogEntry.getDurationBetweenRequestResponseMillis())); generator.writeNumberField("responsesize", accessLogEntry.getReturnedContentSize()); generator.writeNumberField("code", accessLogEntry.getStatusCode()); generator.writeStringField("method", accessLogEntry.getHttpMethod()); @@ -95,6 +95,15 @@ public class JSONFormatter { } } + TraceNode trace = accessLogEntry.getTrace(); + if (trace != null) { + long timestamp = trace.timestamp(); + if (timestamp == 0L) { + timestamp = accessLogEntry.getTimeStampMillis(); + } + trace.accept(new TraceRenderer(generator, timestamp)); + } + // Only add search sub block of this is a search request if (isSearchRequest(accessLogEntry)) { generator.writeObjectFieldStart("search"); diff --git a/container-accesslogging/src/main/java/com/yahoo/container/logging/TraceRenderer.java b/container-accesslogging/src/main/java/com/yahoo/container/logging/TraceRenderer.java new file mode 100644 index 00000000000..295786aa15d --- /dev/null +++ b/container-accesslogging/src/main/java/com/yahoo/container/logging/TraceRenderer.java @@ -0,0 +1,185 @@ +package com.yahoo.container.logging; + +import com.yahoo.data.access.Inspectable; +import com.yahoo.data.access.Inspector; +import com.yahoo.data.access.simple.JsonRender; +import com.yahoo.yolean.trace.TraceNode; +import com.yahoo.yolean.trace.TraceVisitor; +import com.fasterxml.jackson.core.JsonGenerator; + +import java.io.IOException; + +public class TraceRenderer extends TraceVisitor { + private static final String TRACE_CHILDREN = "children"; + private static final String TRACE_MESSAGE = "message"; + private static final String TRACE_TIMESTAMP = "timestamp"; + private static final String TRACE = "trace"; + + private final long basetime; + private final JsonGenerator generator; + private final FieldConsumer fieldConsumer; + private boolean hasFieldName = false; + int emittedChildNesting = 0; + int currentChildNesting = 0; + private boolean insideOpenObject = false; + + public interface FieldConsumer { + void accept(Object object) throws IOException; + } + + private static class Consumer implements FieldConsumer { + private final JsonGenerator generator; + + Consumer(JsonGenerator generator) { + this.generator = generator; + } + + @Override + public void accept(Object object) throws IOException { + if (object instanceof Inspectable) { + renderInspectorDirect(((Inspectable) object).inspect()); + } else { + generator.writeObject(object); + } + } + private void renderInspectorDirect(Inspector data) throws IOException { + StringBuilder intermediate = new StringBuilder(); + JsonRender.render(data, intermediate, true); + generator.writeRawValue(intermediate.toString()); + } + } + + TraceRenderer(JsonGenerator generator, long basetime) { + this(generator, new Consumer(generator), basetime); + } + public TraceRenderer(JsonGenerator generator, FieldConsumer consumer, long basetime) { + this.generator = generator; + this.fieldConsumer = consumer; + this.basetime = basetime; + } + + @Override + public void entering(TraceNode node) { + ++currentChildNesting; + } + + @Override + public void leaving(TraceNode node) { + conditionalEndObject(); + if (currentChildNesting == emittedChildNesting) { + try { + generator.writeEndArray(); + generator.writeEndObject(); + } catch (IOException e) { + throw new TraceRenderWrapper(e); + } + --emittedChildNesting; + } + --currentChildNesting; + } + + @Override + public void visit(TraceNode node) { + try { + doVisit(node.timestamp(), node.payload(), node.children().iterator().hasNext()); + } catch (IOException e) { + throw new TraceRenderWrapper(e); + } + } + + private void doVisit(long timestamp, Object payload, boolean hasChildren) throws IOException { + boolean dirty = false; + if (timestamp != 0L) { + header(); + generator.writeStartObject(); + generator.writeNumberField(TRACE_TIMESTAMP, timestamp - basetime); + dirty = true; + } + if (payload != null) { + if (!dirty) { + header(); + generator.writeStartObject(); + } + generator.writeFieldName(TRACE_MESSAGE); + fieldConsumer.accept(payload); + dirty = true; + } + if (dirty) { + if (!hasChildren) { + generator.writeEndObject(); + } else { + setInsideOpenObject(true); + } + } + } + private void header() { + fieldName(); + for (int i = 0; i < (currentChildNesting - emittedChildNesting); ++i) { + startChildArray(); + } + emittedChildNesting = currentChildNesting; + } + + private void startChildArray() { + try { + conditionalStartObject(); + generator.writeArrayFieldStart(TRACE_CHILDREN); + } catch (IOException e) { + throw new TraceRenderWrapper(e); + } + } + + private void conditionalStartObject() throws IOException { + if (!isInsideOpenObject()) { + generator.writeStartObject(); + } else { + setInsideOpenObject(false); + } + } + + private void conditionalEndObject() { + if (isInsideOpenObject()) { + // This triggers if we were inside a data node with payload and + // subnodes, but none of the subnodes contained data + try { + generator.writeEndObject(); + setInsideOpenObject(false); + } catch (IOException e) { + throw new TraceRenderWrapper(e); + } + } + } + + private void fieldName() { + if (hasFieldName) { + return; + } + + try { + generator.writeFieldName(TRACE); + } catch (IOException e) { + throw new TraceRenderWrapper(e); + } + hasFieldName = true; + } + + boolean isInsideOpenObject() { + return insideOpenObject; + } + + void setInsideOpenObject(boolean insideOpenObject) { + this.insideOpenObject = insideOpenObject; + } + public static final class TraceRenderWrapper extends RuntimeException { + + /** + * Should never be serialized, but this is still needed. + */ + private static final long serialVersionUID = 2L; + + TraceRenderWrapper(IOException wrapped) { + super(wrapped); + } + + } +} diff --git a/container-accesslogging/src/test/java/com/yahoo/container/logging/JSONLogTestCase.java b/container-accesslogging/src/test/java/com/yahoo/container/logging/JSONLogTestCase.java index 8bbb8500cfd..6c7878f99ed 100644 --- a/container-accesslogging/src/test/java/com/yahoo/container/logging/JSONLogTestCase.java +++ b/container-accesslogging/src/test/java/com/yahoo/container/logging/JSONLogTestCase.java @@ -1,8 +1,10 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.container.logging; +import com.yahoo.yolean.trace.TraceNode; import org.junit.Test; +import java.io.IOException; import java.net.URI; import static org.junit.Assert.assertEquals; @@ -70,6 +72,36 @@ public class JSONLogTestCase { } @Test + public void test_json_of_trace() throws IOException { + TraceNode root = new TraceNode("root", 7); + AccessLogEntry entry = newAccessLogEntry("test"); + entry.setTrace(root); + + String expectedOutput = + "{\"ip\":\"152.200.54.243\"," + + "\"time\":920880005.023," + + "\"duration\":0.122," + + "\"responsesize\":9875," + + "\"code\":200," + + "\"method\":\"GET\"," + + "\"uri\":\"?query=test\"," + + "\"version\":\"HTTP/1.1\"," + + "\"agent\":\"Mozilla/4.05 [en] (Win95; I)\"," + + "\"host\":\"localhost\"," + + "\"scheme\":null," + + "\"localport\":0," + + "\"trace\":{\"timestamp\":0,\"message\":\"root\"}," + + "\"search\":{" + + "\"totalhits\":1234," + + "\"hits\":0," + + "\"coverage\":{\"coverage\":100,\"documents\":100}" + + "}" + + "}"; + + assertEquals(expectedOutput, new JSONFormatter(entry).format()); + } + + @Test public void test_with_keyvalues() { AccessLogEntry entry = newAccessLogEntry("test"); entry.addKeyValue("singlevalue", "value1"); diff --git a/container-core/src/main/java/com/yahoo/restapi/ResourceResponse.java b/container-core/src/main/java/com/yahoo/restapi/ResourceResponse.java index ff301d44798..0188136addb 100644 --- a/container-core/src/main/java/com/yahoo/restapi/ResourceResponse.java +++ b/container-core/src/main/java/com/yahoo/restapi/ResourceResponse.java @@ -1,46 +1,41 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.restapi; import com.yahoo.container.jdisc.HttpRequest; -import com.yahoo.container.jdisc.HttpResponse; -import com.yahoo.slime.Cursor; -import com.yahoo.slime.JsonFormat; import com.yahoo.slime.Slime; -import java.io.IOException; -import java.io.OutputStream; import java.net.URI; +import java.util.List; /** * Returns a response containing an array of links to sub-resources * * @author bratseth */ -public class ResourceResponse extends HttpResponse { +public class ResourceResponse extends SlimeJsonResponse { - private final Slime slime = new Slime(); + public ResourceResponse(URI parentUrl, List<String> subResources) { + super(200, toSlime(parentUrl, subResources)); + } public ResourceResponse(URI parentUrl, String ... subResources) { - super(200); - Cursor resourceArray = slime.setObject().setArray("resources"); - for (String subResource : subResources) { - Cursor resourceEntry = resourceArray.addObject(); - resourceEntry.setString("url", new Uri(parentUrl).append(subResource) - .withTrailingSlash() - .toString()); - } + this(parentUrl, List.of(subResources)); } public ResourceResponse(HttpRequest request, String ... subResources) { this(request.getUri(), subResources); } - @Override - public void render(OutputStream stream) throws IOException { - new JsonFormat(true).encode(stream, slime); + private static Slime toSlime(URI parentUrl, List<String> subResources) { + var slime = new Slime(); + var resourceArray = slime.setObject().setArray("resources"); + for (var subResource : subResources) { + var resourceEntry = resourceArray.addObject(); + resourceEntry.setString("url", new Uri(parentUrl).append(subResource) + .withTrailingSlash() + .toString()); + } + return slime; } - @Override - public String getContentType() { return "application/json"; } - } diff --git a/container-search/abi-spec.json b/container-search/abi-spec.json index 437db99c45b..82d3223c8fe 100644 --- a/container-search/abi-spec.json +++ b/container-search/abi-spec.json @@ -5790,6 +5790,7 @@ "public com.yahoo.search.query.profile.compiled.CompiledQueryProfile getQueryProfile()", "public java.lang.Object get(com.yahoo.processing.request.CompoundName, java.util.Map, com.yahoo.processing.request.Properties)", "public void set(com.yahoo.processing.request.CompoundName, java.lang.Object, java.util.Map)", + "public void clearAll(com.yahoo.processing.request.CompoundName, java.util.Map)", "public java.util.Map listProperties(com.yahoo.processing.request.CompoundName, java.util.Map, com.yahoo.processing.request.Properties)", "public boolean isComplete(java.lang.StringBuilder, java.util.Map)", "public com.yahoo.search.query.profile.QueryProfileProperties clone()", @@ -6285,6 +6286,7 @@ "public boolean isOverridable(java.lang.String)", "public java.lang.Class getValueClass(java.lang.String)", "public com.yahoo.search.query.profile.types.QueryProfileType getType(java.lang.String)", + "public com.yahoo.search.query.profile.types.FieldType getFieldType(com.yahoo.processing.request.CompoundName)", "public com.yahoo.search.query.profile.types.FieldDescription getField(java.lang.String)", "public com.yahoo.search.query.profile.types.FieldDescription removeField(java.lang.String)", "public void addField(com.yahoo.search.query.profile.types.FieldDescription)", @@ -6937,7 +6939,8 @@ "com.yahoo.search.rendering.JsonRenderer$FieldConsumer": { "superClass": "java.lang.Object", "interfaces": [ - "com.yahoo.search.result.Hit$RawUtf8Consumer" + "com.yahoo.search.result.Hit$RawUtf8Consumer", + "com.yahoo.container.logging.TraceRenderer$FieldConsumer" ], "attributes": [ "public" @@ -6949,6 +6952,7 @@ "protected boolean shouldRender(java.lang.String, java.lang.Object)", "protected boolean shouldRenderUtf8Value(java.lang.String, int)", "protected void renderFieldContents(java.lang.Object)", + "public void accept(java.lang.Object)", "public bridge synthetic void accept(java.lang.Object, java.lang.Object)" ], "fields": [] diff --git a/container-search/src/main/java/com/yahoo/prelude/searcher/ValidateSortingSearcher.java b/container-search/src/main/java/com/yahoo/prelude/searcher/ValidateSortingSearcher.java index bbdb3b796a2..82148cf54e6 100644 --- a/container-search/src/main/java/com/yahoo/prelude/searcher/ValidateSortingSearcher.java +++ b/container-search/src/main/java/com/yahoo/prelude/searcher/ValidateSortingSearcher.java @@ -119,6 +119,9 @@ public class ValidateSortingSearcher extends Searcher { String name = f.getFieldName(); if ("[rank]".equals(name) || "[docid]".equals(name)) { // built-in constants - ok + } else if ("[relevance]".equals(name)) { + // built-in constant '[relevance]' must map to '[rank]' + f.getSorter().setName("[rank]"); } else if ("[relevancy]".equals(name)) { // built-in constant '[relevancy]' must map to '[rank]' f.getSorter().setName("[rank]"); diff --git a/container-search/src/main/java/com/yahoo/search/handler/HttpSearchResponse.java b/container-search/src/main/java/com/yahoo/search/handler/HttpSearchResponse.java index 3602d21f7d8..d636d3bc925 100644 --- a/container-search/src/main/java/com/yahoo/search/handler/HttpSearchResponse.java +++ b/container-search/src/main/java/com/yahoo/search/handler/HttpSearchResponse.java @@ -23,6 +23,7 @@ import com.yahoo.processing.rendering.Renderer; import com.yahoo.search.Query; import com.yahoo.search.Result; import com.yahoo.search.query.context.QueryContext; +import com.yahoo.yolean.trace.TraceNode; /** * Wrap the result of a query as an HTTP response. @@ -36,8 +37,13 @@ public class HttpSearchResponse extends ExtendedResponse { private final Renderer<Result> rendererCopy; private final Timing timing; private final HitCounts hitCounts; + private final TraceNode trace; public HttpSearchResponse(int status, Result result, Query query, Renderer renderer) { + this(status, result, query, renderer, null); + } + + HttpSearchResponse(int status, Result result, Query query, Renderer renderer, TraceNode trace) { super(status); this.query = query; this.result = result; @@ -45,6 +51,7 @@ public class HttpSearchResponse extends ExtendedResponse { this.timing = SearchResponse.createTiming(query, result); this.hitCounts = SearchResponse.createHitCounts(query, result); + this.trace = trace; populateHeaders(headers(), result.getHeaders(false)); } @@ -107,6 +114,9 @@ public class HttpSearchResponse extends ExtendedResponse { @Override public void populateAccessLogEntry(final AccessLogEntry accessLogEntry) { super.populateAccessLogEntry(accessLogEntry); + if (trace != null) { + accessLogEntry.setTrace(trace); + } populateAccessLogEntry(accessLogEntry, getHitCounts()); } diff --git a/container-search/src/main/java/com/yahoo/search/handler/SearchHandler.java b/container-search/src/main/java/com/yahoo/search/handler/SearchHandler.java index 0f0df3aa93e..dad106570ab 100644 --- a/container-search/src/main/java/com/yahoo/search/handler/SearchHandler.java +++ b/container-search/src/main/java/com/yahoo/search/handler/SearchHandler.java @@ -296,7 +296,10 @@ public class SearchHandler extends LoggingRequestHandler { // Transform result to response Renderer renderer = toRendererCopy(query.getPresentation().getRenderer()); HttpSearchResponse response = new HttpSearchResponse(getHttpResponseStatus(request, result), - result, query, renderer); + result, query, renderer, + log.isLoggable(Level.FINE) + ? query.getContext(false).getTrace().traceNode() + : null); if (hostResponseHeaderKey.isPresent()) response.headers().add(hostResponseHeaderKey.get(), selfHostname); @@ -410,7 +413,7 @@ public class SearchHandler extends LoggingRequestHandler { } catch (ParseException e) { ErrorMessage error = ErrorMessage.createIllegalQuery("Could not parse query [" + request + "]: " + Exceptions.toMessageString(e)); - log.log(LogLevel.DEBUG, () -> error.getDetailedMessage()); + log.log(LogLevel.DEBUG, error::getDetailedMessage); return new Result(query, error); } catch (IllegalArgumentException e) { if ("Comparison method violates its general contract!".equals(e.getMessage())) { @@ -422,7 +425,7 @@ public class SearchHandler extends LoggingRequestHandler { else { ErrorMessage error = ErrorMessage.createBadRequest("Invalid search request [" + request + "]: " + Exceptions.toMessageString(e)); - log.log(LogLevel.DEBUG, () -> error.getDetailedMessage()); + log.log(LogLevel.DEBUG, error::getDetailedMessage); return new Result(query, error); } } catch (LinkageError | StackOverflowError e) { diff --git a/container-search/src/main/java/com/yahoo/search/query/Ranking.java b/container-search/src/main/java/com/yahoo/search/query/Ranking.java index 7444c94f491..830a3f4ef81 100644 --- a/container-search/src/main/java/com/yahoo/search/query/Ranking.java +++ b/container-search/src/main/java/com/yahoo/search/query/Ranking.java @@ -47,7 +47,7 @@ public class Ranking implements Cloneable { public static final String PROPERTIES = "properties"; static { - argumentType =new QueryProfileType(RANKING); + argumentType = new QueryProfileType(RANKING); argumentType.setStrict(true); argumentType.setBuiltin(true); argumentType.addField(new FieldDescription(LOCATION, "string", "location")); @@ -63,7 +63,7 @@ public class Ranking implements Cloneable { argumentType.addField(new FieldDescription(FEATURES, "query-profile", "rankfeature")); argumentType.addField(new FieldDescription(PROPERTIES, "query-profile", "rankproperty")); argumentType.freeze(); - argumentTypeName=new CompoundName(argumentType.getId().getName()); + argumentTypeName = new CompoundName(argumentType.getId().getName()); } public static QueryProfileType getArgumentType() { return argumentType; } diff --git a/container-search/src/main/java/com/yahoo/search/query/profile/BackedOverridableQueryProfile.java b/container-search/src/main/java/com/yahoo/search/query/profile/BackedOverridableQueryProfile.java index 99f1e26b221..11864e60cec 100644 --- a/container-search/src/main/java/com/yahoo/search/query/profile/BackedOverridableQueryProfile.java +++ b/container-search/src/main/java/com/yahoo/search/query/profile/BackedOverridableQueryProfile.java @@ -135,8 +135,8 @@ public class BackedOverridableQueryProfile extends OverridableQueryProfile imple @Override public List<String> getDimensions() { - List<String> dimensions=super.getDimensions(); - if (dimensions!=null) return dimensions; + List<String> dimensions = super.getDimensions(); + if (dimensions != null) return dimensions; return backingProfile.getDimensions(); } diff --git a/container-search/src/main/java/com/yahoo/search/query/profile/ModelObjectMap.java b/container-search/src/main/java/com/yahoo/search/query/profile/ModelObjectMap.java index 4dc9ade62e5..e32d2dc226d 100644 --- a/container-search/src/main/java/com/yahoo/search/query/profile/ModelObjectMap.java +++ b/container-search/src/main/java/com/yahoo/search/query/profile/ModelObjectMap.java @@ -22,8 +22,7 @@ public class ModelObjectMap extends PropertyMap { */ @Override protected boolean shouldSet(CompoundName name, Object value) { - if (value == null) return true; - return ! FieldType.isLegalFieldValue(value); + return value != null && ! FieldType.isLegalFieldValue(value); } } diff --git a/container-search/src/main/java/com/yahoo/search/query/profile/QueryProfile.java b/container-search/src/main/java/com/yahoo/search/query/profile/QueryProfile.java index e9ccdd22f98..7ae18f96d86 100644 --- a/container-search/src/main/java/com/yahoo/search/query/profile/QueryProfile.java +++ b/container-search/src/main/java/com/yahoo/search/query/profile/QueryProfile.java @@ -100,7 +100,7 @@ public class QueryProfile extends FreezableSimpleComponent implements Cloneable public QueryProfileType getType() { return type; } /** Sets the type of this, or set to null to not use any type checking in this profile */ - public void setType(QueryProfileType type) { this.type=type; } + public void setType(QueryProfileType type) { this.type = type; } /** Returns the virtual variants of this, or null if none */ public QueryProfileVariants getVariants() { return variants; } diff --git a/container-search/src/main/java/com/yahoo/search/query/profile/QueryProfileProperties.java b/container-search/src/main/java/com/yahoo/search/query/profile/QueryProfileProperties.java index 5d4f39cecbf..701ea7690f4 100644 --- a/container-search/src/main/java/com/yahoo/search/query/profile/QueryProfileProperties.java +++ b/container-search/src/main/java/com/yahoo/search/query/profile/QueryProfileProperties.java @@ -9,6 +9,7 @@ import com.yahoo.search.query.Properties; import com.yahoo.search.query.profile.compiled.CompiledQueryProfile; import com.yahoo.search.query.profile.compiled.DimensionalValue; import com.yahoo.search.query.profile.types.FieldDescription; +import com.yahoo.search.query.profile.types.QueryProfileFieldType; import com.yahoo.search.query.profile.types.QueryProfileType; import java.util.ArrayList; @@ -31,7 +32,12 @@ public class QueryProfileProperties extends Properties { /** Values which has been overridden at runtime, or null if none */ private Map<CompoundName, Object> values = null; - /** Query profile references which has been overridden at runtime, or null if none. Earlier values has precedence */ + + /** + * Query profile references which has been overridden at runtime, possibly to the null value to clear values, + * or null if none (i.e this is lazy). + * Earlier values has precedence + */ private List<Pair<CompoundName, CompiledQueryProfile>> references = null; /** Creates an instance from a profile, throws an exception if the given profile is null */ @@ -48,20 +54,21 @@ public class QueryProfileProperties extends Properties { public Object get(CompoundName name, Map<String, String> context, com.yahoo.processing.request.Properties substitution) { name = unalias(name, context); - Object value = null; - if (values != null) - value = values.get(name); - if (value == null) { - Pair<CompoundName, CompiledQueryProfile> reference = findReference(name); - if (reference != null) - return reference.getSecond().get(name.rest(reference.getFirst().size()), context, substitution); // yes; even if null + if (values != null && values.containsKey(name)) + return values.get(name); // Returns this value, even if null + + Pair<CompoundName, CompiledQueryProfile> reference = findReference(name); + if (reference != null) { + if (reference.getSecond() == null) + return null; // cleared + else + return reference.getSecond().get(name.rest(reference.getFirst().size()), context, substitution); // even if null } - if (value == null) - value = profile.get(name, context, substitution); - if (value == null) - value = super.get(name, context, substitution); - return value; + Object value = profile.get(name, context, substitution); + if (value != null) + return value; + return super.get(name, context, substitution); } /** @@ -87,8 +94,10 @@ public class QueryProfileProperties extends Properties { // Check types if ( ! profile.getTypes().isEmpty()) { - for (int i = 0; i<name.size(); i++) { - QueryProfileType type = profile.getType(name.first(i), context); + QueryProfileType type = null; + for (int i = 0; i < name.size(); i++) { + if (type == null) // We're on the first iteration, or no type is explicitly specified + type = profile.getType(name.first(i), context); if (type == null) continue; String localName = name.get(i); FieldDescription fieldDescription = type.getField(localName); @@ -97,12 +106,19 @@ public class QueryProfileProperties extends Properties { // TODO: In addition to strictness, check legality along the way - if (i == name.size()-1 && fieldDescription != null) { // at the end of the path, check the assignment type - value = fieldDescription.getType().convertFrom(value, profile.getRegistry()); - if (value == null) - throw new IllegalArgumentException("'" + value + "' is not a " + - fieldDescription.getType().toInstanceDescription()); + if (fieldDescription != null) { + if (i == name.size() - 1) { // at the end of the path, check the assignment type + value = fieldDescription.getType().convertFrom(value, profile.getRegistry()); + if (value == null) + throw new IllegalArgumentException("'" + value + "' is not a " + + fieldDescription.getType().toInstanceDescription()); + } + else if (fieldDescription.getType() instanceof QueryProfileFieldType) { + // If a type is specified, use that instead of the type implied by the name + type = ((QueryProfileFieldType) fieldDescription.getType()).getQueryProfileType(); + } } + } } @@ -133,12 +149,26 @@ public class QueryProfileProperties extends Properties { } @Override - public Map<String, Object> listProperties(CompoundName path, Map<String,String> context, + public void clearAll(CompoundName name, Map<String, String> context) { + if (references == null) + references = new ArrayList<>(); + references.add(new Pair<>(name, null)); + + if (values != null) + values.keySet().removeIf(key -> key.hasPrefix(name)); + } + + @Override + public Map<String, Object> listProperties(CompoundName path, Map<String, String> context, com.yahoo.processing.request.Properties substitution) { path = unalias(path, context); if (context == null) context = Collections.emptyMap(); - Map<String, Object> properties = profile.listValues(path, context, substitution); + Map<String, Object> properties = new HashMap<>(); + for (var entry : profile.listValues(path, context, substitution).entrySet()) { + if (references != null && containsNullParentOf(path, references)) continue; + properties.put(entry.getKey(), entry.getValue()); + } properties.putAll(super.listProperties(path, context, substitution)); if (references != null) { @@ -155,8 +185,14 @@ public class QueryProfileProperties extends Properties { pathInReference = path.rest(refEntry.getFirst().size()); prefixToReferenceKeys = CompoundName.empty; } - for (Map.Entry<String, Object> valueEntry : refEntry.getSecond().listValues(pathInReference, context, substitution).entrySet()) { - properties.put(prefixToReferenceKeys.append(new CompoundName(valueEntry.getKey())).toString(), valueEntry.getValue()); + if (refEntry.getSecond() == null) { + if (refEntry.getFirst().hasPrefix(path)) + properties.put(prefixToReferenceKeys.toString(), null); + } + else { + for (Map.Entry<String, Object> valueEntry : refEntry.getSecond().listValues(pathInReference, context, substitution).entrySet()) { + properties.put(prefixToReferenceKeys.append(new CompoundName(valueEntry.getKey())).toString(), valueEntry.getValue()); + } } } @@ -231,6 +267,12 @@ public class QueryProfileProperties extends Properties { return null; } + private boolean containsNullParentOf(CompoundName path, List<Pair<CompoundName, CompiledQueryProfile>> properties) { + if (properties.contains(new Pair<>(path, (CompiledQueryProfile)null))) return true; + if (path.size() > 0 && containsNullParentOf(path.first(path.size() - 1), properties)) return true; + return false; + } + CompoundName unalias(CompoundName name, Map<String,String> context) { if (profile.getTypes().isEmpty()) return name; diff --git a/container-search/src/main/java/com/yahoo/search/query/profile/compiled/CompiledQueryProfile.java b/container-search/src/main/java/com/yahoo/search/query/profile/compiled/CompiledQueryProfile.java index 644d366e7d0..94f4e4747a0 100644 --- a/container-search/src/main/java/com/yahoo/search/query/profile/compiled/CompiledQueryProfile.java +++ b/container-search/src/main/java/com/yahoo/search/query/profile/compiled/CompiledQueryProfile.java @@ -134,7 +134,6 @@ public class CompiledQueryProfile extends AbstractComponent implements Cloneable public Map<String, Object> listValues(CompoundName prefix, Map<String, String> context, Properties substitution) { Map<String, Object> values = new HashMap<>(); for (Map.Entry<CompoundName, DimensionalValue<ValueWithSource>> entry : entries.entrySet()) { - if ( entry.getKey().size() <= prefix.size()) continue; if ( ! entry.getKey().hasPrefix(prefix)) continue; ValueWithSource valueWithSource = entry.getValue().get(context); diff --git a/container-search/src/main/java/com/yahoo/search/query/profile/config/QueryProfileXMLReader.java b/container-search/src/main/java/com/yahoo/search/query/profile/config/QueryProfileXMLReader.java index 33f07a58195..1b1cdce5890 100644 --- a/container-search/src/main/java/com/yahoo/search/query/profile/config/QueryProfileXMLReader.java +++ b/container-search/src/main/java/com/yahoo/search/query/profile/config/QueryProfileXMLReader.java @@ -33,7 +33,7 @@ public class QueryProfileXMLReader { * Reads all query profile xml files in a given directory, * and all type xml files from the immediate subdirectory "types/" (if any) * - * @throws RuntimeException if <code>directory</code> is not a readable directory, or if there is some error in the XML + * @throws IllegalArgumentException if the directory is not readable, or if there is some error in the XML */ public QueryProfileRegistry read(String directory) { List<NamedReader> queryProfileReaders = new ArrayList<>(); @@ -58,7 +58,7 @@ public class QueryProfileXMLReader { return read(queryProfileTypeReaders,queryProfileReaders); } catch (IOException e) { - throw new IllegalArgumentException("Could not read query profiles from '" + directory + "'",e); + throw new IllegalArgumentException("Could not read query profiles from '" + directory + "'", e); } finally { closeAll(queryProfileReaders); @@ -105,14 +105,14 @@ public class QueryProfileXMLReader { "' must be 'query-profile-type', not '" + root.getNodeName() + "'"); } - String idString=root.getAttribute("id"); + String idString = root.getAttribute("id"); if (idString == null || idString.equals("")) throw new IllegalArgumentException("'" + reader.getName() + "' has no 'id' attribute in the root element"); ComponentId id = new ComponentId(idString); - validateFileNameToId(reader.getName(),id,"query profile type"); + validateFileNameToId(reader.getName(), id,"query profile type"); QueryProfileType type = new QueryProfileType(id); - type.setMatchAsPath(XML.getChild(root,"match") != null); - type.setStrict(XML.getChild(root,"strict") != null); + type.setMatchAsPath(XML.getChild(root, "match") != null); + type.setStrict(XML.getChild(root, "strict") != null); registry.register(type); queryProfileTypeElements.add(root); } @@ -145,7 +145,7 @@ public class QueryProfileXMLReader { queryProfile.setType(type); } - Element dimensions = XML.getChild(root,"dimensions"); + Element dimensions = XML.getChild(root, "dimensions"); if (dimensions != null) queryProfile.setDimensions(toArray(XML.getValue(dimensions))); @@ -215,7 +215,7 @@ public class QueryProfileXMLReader { try { String fieldTypeName = field.getAttribute("type"); if (fieldTypeName == null) throw new IllegalArgumentException("Field '" + field + "' has no 'type' attribute"); - FieldType fieldType=FieldType.fromString(fieldTypeName,registry); + FieldType fieldType = FieldType.fromString(fieldTypeName, registry); type.addField(new FieldDescription(name, fieldType, field.getAttribute("alias"), diff --git a/container-search/src/main/java/com/yahoo/search/query/profile/types/FieldDescription.java b/container-search/src/main/java/com/yahoo/search/query/profile/types/FieldDescription.java index b8290fa092b..6c30f1a8b05 100644 --- a/container-search/src/main/java/com/yahoo/search/query/profile/types/FieldDescription.java +++ b/container-search/src/main/java/com/yahoo/search/query/profile/types/FieldDescription.java @@ -97,7 +97,7 @@ public class FieldDescription implements Comparable<FieldDescription> { this.type = type; // Forbidden until we can figure out the right semantics - if (name.isCompound() && ! aliases.isEmpty()) throw new IllegalArgumentException("Aliases is not allowed with compound names"); + if (name.isCompound() && ! aliases.isEmpty()) throw new IllegalArgumentException("Aliases are not allowed with compound names"); this.aliases = ImmutableList.copyOf(aliases); this.mandatory = mandatory; diff --git a/container-search/src/main/java/com/yahoo/search/query/profile/types/QueryProfileType.java b/container-search/src/main/java/com/yahoo/search/query/profile/types/QueryProfileType.java index 07c9e4475ec..c02aada2062 100644 --- a/container-search/src/main/java/com/yahoo/search/query/profile/types/QueryProfileType.java +++ b/container-search/src/main/java/com/yahoo/search/query/profile/types/QueryProfileType.java @@ -6,6 +6,7 @@ import com.google.common.collect.ImmutableMap; import com.yahoo.component.ComponentId; import com.yahoo.component.provider.FreezableSimpleComponent; import com.yahoo.processing.request.CompoundName; +import com.yahoo.search.query.profile.OverridableQueryProfile; import com.yahoo.search.query.profile.QueryProfile; import java.util.ArrayList; @@ -24,6 +25,7 @@ import static com.yahoo.text.Lowercase.toLowerCase; public class QueryProfileType extends FreezableSimpleComponent { private final CompoundName componentIdAsCompoundName; + /** The fields of this query profile type */ private Map<String, FieldDescription> fields; @@ -217,25 +219,38 @@ public class QueryProfileType extends FreezableSimpleComponent { /** Returns the type of the given query profile type declared as a field in this */ public QueryProfileType getType(String localName) { - FieldDescription fieldDescription=getField(localName); - if (fieldDescription ==null) return null; + FieldDescription fieldDescription = getField(localName); + if (fieldDescription == null) return null; if ( ! (fieldDescription.getType() instanceof QueryProfileFieldType)) return null; return ((QueryProfileFieldType) fieldDescription.getType()).getQueryProfileType(); } + /** Returns the field type of the given name under this, of null if none */ + public FieldType getFieldType(CompoundName name) { + FieldDescription field = getField(name.first()); + if (field == null) return null; + + FieldType fieldType = field.getType(); + if (name.size() == 1) return fieldType; + + if ( ! (fieldType instanceof QueryProfileFieldType)) return null; + + return ((QueryProfileFieldType)fieldType).getQueryProfileType().getFieldType(name.rest()); + } + /** * Returns the description of the field with the given name in this type or an inherited type * (depth first left to right search). Returns null if the field is not defined in this or an inherited profile. */ public FieldDescription getField(String name) { - FieldDescription field=fields.get(name); - if ( field!=null ) return field; + FieldDescription field = fields.get(name); + if ( field != null ) return field; if ( isFrozen() ) return null; // Inherited are collapsed into this for (QueryProfileType inheritedType : this.inherited() ) { - field=inheritedType.getField(name); - if (field!=null) return field; + field = inheritedType.getField(name); + if (field != null) return field; } return null; @@ -276,7 +291,7 @@ public class QueryProfileType extends FreezableSimpleComponent { // Add (/to) a query profile type containing the rest of the name. // (we do not need the field description settings for intermediate query profile types // as the leaf entry will enforce them) - QueryProfileType type = getOrCreateQueryProfileType(name.first(), registry); + QueryProfileType type = extendOrCreateQueryProfileType(name.first(), registry); type.addField(fieldDescription.withName(name.rest()), registry); } else { @@ -288,27 +303,42 @@ public class QueryProfileType extends FreezableSimpleComponent { addAlias(alias, fieldDescription.getName()); } - private QueryProfileType getOrCreateQueryProfileType(String name, QueryProfileTypeRegistry registry) { + private QueryProfileType extendOrCreateQueryProfileType(String name, QueryProfileTypeRegistry registry) { + QueryProfileType type = null; FieldDescription fieldDescription = getField(name); if (fieldDescription != null) { - if ( ! ( fieldDescription.getType() instanceof QueryProfileFieldType)) + if ( ! (fieldDescription.getType() instanceof QueryProfileFieldType)) throw new IllegalArgumentException("Cannot use name '" + name + "' as a prefix because it is " + "already a " + fieldDescription.getType()); QueryProfileFieldType fieldType = (QueryProfileFieldType) fieldDescription.getType(); - QueryProfileType type = fieldType.getQueryProfileType(); - if (type == null) { // an as-yet untyped reference; add type - type = new QueryProfileType(name); - registry.register(type.getId(), type); - fields.put(name, fieldDescription.withType(new QueryProfileFieldType(type))); - } - return type; + type = fieldType.getQueryProfileType(); + } + + if (type == null) { + type = registry.getComponent(name); + } + + // found in registry but not already added in *this* type (getField also checks parents): extend it + if (type != null && ! fields.containsKey(name)) { + type = new QueryProfileType(ComponentId.createAnonymousComponentId(type.getIdString()), + new HashMap<>(), + List.of(type)); + } + + if (type == null) { // create it + type = new QueryProfileType(ComponentId.createAnonymousComponentId(name)); + } + + if (fieldDescription == null) { + fieldDescription = new FieldDescription(name, new QueryProfileFieldType(type)); } else { - QueryProfileType type = new QueryProfileType(name); - registry.register(type.getId(), type); - fields.put(name, new FieldDescription(name, new QueryProfileFieldType(type))); - return type; + fieldDescription = fieldDescription.withType(new QueryProfileFieldType(type)); } + + registry.register(type); + fields.put(name, fieldDescription); + return type; } private void addAlias(String alias, String field) { @@ -362,6 +392,7 @@ public class QueryProfileType extends FreezableSimpleComponent { return other.getId().equals(this.getId()); } + @Override public String toString() { return "query profile type '" + getId() + "'"; } diff --git a/container-search/src/main/java/com/yahoo/search/query/properties/PropertyMap.java b/container-search/src/main/java/com/yahoo/search/query/properties/PropertyMap.java index 4f30331e738..643e215daef 100644 --- a/container-search/src/main/java/com/yahoo/search/query/properties/PropertyMap.java +++ b/container-search/src/main/java/com/yahoo/search/query/properties/PropertyMap.java @@ -26,7 +26,10 @@ public class PropertyMap extends Properties { /** The properties of this */ private Map<CompoundName, Object> properties = new LinkedHashMap<>(); - public void set(CompoundName name, Object value, Map<String,String> context) { + public void set(CompoundName name, Object value, Map<String, String> context) { + if (value == null) // Both clear and forward + properties.remove(name); + if (shouldSet(name, value)) properties.put(name, value); else diff --git a/container-search/src/main/java/com/yahoo/search/query/properties/QueryProperties.java b/container-search/src/main/java/com/yahoo/search/query/properties/QueryProperties.java index dfe6c2af44b..96f73e925af 100644 --- a/container-search/src/main/java/com/yahoo/search/query/properties/QueryProperties.java +++ b/container-search/src/main/java/com/yahoo/search/query/properties/QueryProperties.java @@ -294,7 +294,7 @@ public class QueryProperties extends Properties { super.set(key,value,context); } catch (Exception e) { // Make sure error messages are informative. This should be moved out of this properties implementation - if (e.getMessage().startsWith("Could not set")) + if (e.getMessage() != null && e.getMessage().startsWith("Could not set")) throw e; else throw new IllegalArgumentException("Could not set '" + key + "' to '" + value + "'", e); diff --git a/container-search/src/main/java/com/yahoo/search/rendering/JsonRenderer.java b/container-search/src/main/java/com/yahoo/search/rendering/JsonRenderer.java index af453983f89..31f8194b3b7 100644 --- a/container-search/src/main/java/com/yahoo/search/rendering/JsonRenderer.java +++ b/container-search/src/main/java/com/yahoo/search/rendering/JsonRenderer.java @@ -7,6 +7,7 @@ import com.fasterxml.jackson.core.JsonGenerator; import com.fasterxml.jackson.core.TreeNode; import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.base.Preconditions; +import com.yahoo.container.logging.TraceRenderer; import com.yahoo.data.JsonProducer; import com.yahoo.data.access.Inspectable; import com.yahoo.data.access.Inspector; @@ -44,8 +45,6 @@ import com.yahoo.search.result.Hit; import com.yahoo.search.result.HitGroup; import com.yahoo.search.result.NanNumber; import com.yahoo.tensor.Tensor; -import com.yahoo.yolean.trace.TraceNode; -import com.yahoo.yolean.trace.TraceVisitor; import org.json.JSONArray; import org.json.JSONObject; @@ -111,10 +110,6 @@ public class JsonRenderer extends AsynchronousSectionedRenderer<Result> { private static final String ROOT = "root"; private static final String SOURCE = "source"; private static final String TOTAL_COUNT = "totalCount"; - private static final String TRACE = "trace"; - private static final String TRACE_CHILDREN = "children"; - private static final String TRACE_MESSAGE = "message"; - private static final String TRACE_TIMESTAMP = "timestamp"; private static final String TIMING = "timing"; private static final String QUERY_TIME = "querytime"; private static final String SUMMARY_FETCH_TIME = "summaryfetchtime"; @@ -132,145 +127,6 @@ public class JsonRenderer extends AsynchronousSectionedRenderer<Result> { private LongSupplier timeSource; private OutputStream stream; - private class TraceRenderer extends TraceVisitor { - private final long basetime; - private boolean hasFieldName = false; - int emittedChildNesting = 0; - int currentChildNesting = 0; - private boolean insideOpenObject = false; - - TraceRenderer(long basetime) { - this.basetime = basetime; - } - - @Override - public void entering(TraceNode node) { - ++currentChildNesting; - } - - @Override - public void leaving(TraceNode node) { - conditionalEndObject(); - if (currentChildNesting == emittedChildNesting) { - try { - generator.writeEndArray(); - generator.writeEndObject(); - } catch (IOException e) { - throw new TraceRenderWrapper(e); - } - --emittedChildNesting; - } - --currentChildNesting; - } - - @Override - public void visit(TraceNode node) { - try { - doVisit(node.timestamp(), node.payload(), node.children().iterator().hasNext()); - } catch (IOException e) { - throw new TraceRenderWrapper(e); - } - } - - private void doVisit(long timestamp, Object payload, boolean hasChildren) throws IOException { - boolean dirty = false; - if (timestamp != 0L) { - header(); - generator.writeStartObject(); - generator.writeNumberField(TRACE_TIMESTAMP, timestamp - basetime); - dirty = true; - } - if (payload != null) { - if (!dirty) { - header(); - generator.writeStartObject(); - } - generator.writeFieldName(TRACE_MESSAGE); - fieldConsumer.renderFieldContentsDirect(payload); - dirty = true; - } - if (dirty) { - if (!hasChildren) { - generator.writeEndObject(); - } else { - setInsideOpenObject(true); - } - } - } - - private void header() { - fieldName(); - for (int i = 0; i < (currentChildNesting - emittedChildNesting); ++i) { - startChildArray(); - } - emittedChildNesting = currentChildNesting; - } - - private void startChildArray() { - try { - conditionalStartObject(); - generator.writeArrayFieldStart(TRACE_CHILDREN); - } catch (IOException e) { - throw new TraceRenderWrapper(e); - } - } - - private void conditionalStartObject() throws IOException { - if (!isInsideOpenObject()) { - generator.writeStartObject(); - } else { - setInsideOpenObject(false); - } - } - - private void conditionalEndObject() { - if (isInsideOpenObject()) { - // This triggers if we were inside a data node with payload and - // subnodes, but none of the subnodes contained data - try { - generator.writeEndObject(); - setInsideOpenObject(false); - } catch (IOException e) { - throw new TraceRenderWrapper(e); - } - } - } - - private void fieldName() { - if (hasFieldName) { - return; - } - - try { - generator.writeFieldName(TRACE); - } catch (IOException e) { - throw new TraceRenderWrapper(e); - } - hasFieldName = true; - } - - boolean isInsideOpenObject() { - return insideOpenObject; - } - - void setInsideOpenObject(boolean insideOpenObject) { - this.insideOpenObject = insideOpenObject; - } - } - - private static final class TraceRenderWrapper extends RuntimeException { - - /** - * Should never be serialized, but this is still needed. - */ - private static final long serialVersionUID = 2L; - - TraceRenderWrapper(IOException wrapped) { - super(wrapped); - } - - } - public JsonRenderer() { this(null); } @@ -352,8 +208,8 @@ public class JsonRenderer extends AsynchronousSectionedRenderer<Result> { long basetime = trace.traceNode().timestamp(); if (basetime == 0L) basetime = getResult().getElapsedTime().first(); - trace.accept(new TraceRenderer(basetime)); - } catch (TraceRenderWrapper e) { + trace.accept(new TraceRenderer(generator, fieldConsumer, basetime)); + } catch (TraceRenderer.TraceRenderWrapper e) { throw new IOException(e); } } @@ -641,11 +497,9 @@ public class JsonRenderer extends AsynchronousSectionedRenderer<Result> { private String getJsonCallback() { Result result = getResult(); - if (result != null) { - Query query = result.getQuery(); - if (query != null) { - return query.properties().getString(JSON_CALLBACK, null); - } + Query query = result.getQuery(); + if (query != null) { + return query.properties().getString(JSON_CALLBACK, null); } return null; } @@ -671,7 +525,7 @@ public class JsonRenderer extends AsynchronousSectionedRenderer<Result> { * This instance is reused for all hits of a Result since we are in a single-threaded context * and want to limit object creation. */ - public static class FieldConsumer implements Hit.RawUtf8Consumer { + public static class FieldConsumer implements Hit.RawUtf8Consumer, TraceRenderer.FieldConsumer { private final JsonGenerator generator; private final boolean debugRendering; @@ -788,11 +642,12 @@ public class JsonRenderer extends AsynchronousSectionedRenderer<Result> { if (field instanceof Inspectable && ! (field instanceof FeatureData)) { renderInspector(((Inspectable)field).inspect()); } else { - renderFieldContentsDirect(field); + accept(field); } } - private void renderFieldContentsDirect(Object field) throws IOException { + @Override + public void accept(Object field) throws IOException { if (field == null) { generator.writeNull(); } else if (field instanceof Boolean) { diff --git a/container-search/src/test/java/com/yahoo/prelude/searcher/test/ValidateSortingSearcherTestCase.java b/container-search/src/test/java/com/yahoo/prelude/searcher/test/ValidateSortingSearcherTestCase.java index 65011ffb562..f4bf957e29a 100644 --- a/container-search/src/test/java/com/yahoo/prelude/searcher/test/ValidateSortingSearcherTestCase.java +++ b/container-search/src/test/java/com/yahoo/prelude/searcher/test/ValidateSortingSearcherTestCase.java @@ -52,6 +52,7 @@ public class ValidateSortingSearcherTestCase { assertEquals("[ASCENDING:[rank]]", quoteAndTransform("+[rank]")); assertEquals("[ASCENDING:[docid]]", quoteAndTransform("+[docid]")); assertEquals("[ASCENDING:[rank]]", quoteAndTransform("+[relevancy]")); + assertEquals("[ASCENDING:[rank]]", quoteAndTransform("+[relevance]")); } @Test diff --git a/container-search/src/test/java/com/yahoo/search/query/profile/config/test/XmlReadingTestCase.java b/container-search/src/test/java/com/yahoo/search/query/profile/config/test/XmlReadingTestCase.java index 0f4a22ef368..e9f7ff24d42 100644 --- a/container-search/src/test/java/com/yahoo/search/query/profile/config/test/XmlReadingTestCase.java +++ b/container-search/src/test/java/com/yahoo/search/query/profile/config/test/XmlReadingTestCase.java @@ -384,43 +384,64 @@ public class XmlReadingTestCase { assertNull(query.properties().get("profileRef.myProfile1Only")); // later assignment - query.properties().set("profileRef.name","newName"); - assertEquals("newName",query.properties().get("profileRef.name")); + query.properties().set("profileRef.name", "newName"); + assertEquals("newName", query.properties().get("profileRef.name")); // ...will not impact others - query=new Query(HttpRequest.createTestRequest("?query=test&profileRef=ref:MyProfile2", Method.GET),registry.getComponent("default")); - assertEquals("MyProfile2",query.properties().get("profileRef.name")); + query=new Query(HttpRequest.createTestRequest("?query=test&profileRef=ref:MyProfile2", Method.GET), registry.getComponent("default")); + assertEquals("MyProfile2", query.properties().get("profileRef.name")); } } @Test public void testRefOverrideTyped() { - CompiledQueryProfileRegistry registry=new QueryProfileXMLReader().read("src/test/java/com/yahoo/search/query/profile/config/test/refoverridetyped").compile(); + CompiledQueryProfileRegistry registry = new QueryProfileXMLReader().read("src/test/java/com/yahoo/search/query/profile/config/test/refoverridetyped").compile(); { // Original reference - Query query=new Query(HttpRequest.createTestRequest("?query=test", Method.GET),registry.getComponent("default")); - assertEquals(null,query.properties().get("profileRef")); - assertEquals("MyProfile1",query.properties().get("profileRef.name")); - assertEquals("myProfile1Only",query.properties().get("profileRef.myProfile1Only")); + Query query = new Query(HttpRequest.createTestRequest("?query=test", Method.GET), registry.getComponent("default")); + assertEquals(null, query.properties().get("profileRef")); + assertEquals("MyProfile1", query.properties().get("profileRef.name")); + assertEquals("myProfile1Only", query.properties().get("profileRef.myProfile1Only")); assertNull(query.properties().get("profileRef.myProfile2Only")); } { // Overridden reference - Query query=new Query(HttpRequest.createTestRequest("?query=test&profileRef=MyProfile2", Method.GET),registry.getComponent("default")); - assertEquals(null,query.properties().get("profileRef")); - assertEquals("MyProfile2",query.properties().get("profileRef.name")); - assertEquals("myProfile2Only",query.properties().get("profileRef.myProfile2Only")); + Query query = new Query(HttpRequest.createTestRequest("?query=test&profileRef=MyProfile2", Method.GET), registry.getComponent("default")); + assertEquals(null, query.properties().get("profileRef")); + assertEquals("MyProfile2", query.properties().get("profileRef.name")); + assertEquals("myProfile2Only", query.properties().get("profileRef.myProfile2Only")); assertNull(query.properties().get("profileRef.myProfile1Only")); // later assignment - query.properties().set("profileRef.name","newName"); - assertEquals("newName",query.properties().get("profileRef.name")); + query.properties().set("profileRef.name", "newName"); + assertEquals("newName", query.properties().get("profileRef.name")); // ...will not impact others - query=new Query(HttpRequest.createTestRequest("?query=test&profileRef=ref:MyProfile2", Method.GET),registry.getComponent("default")); - assertEquals("MyProfile2",query.properties().get("profileRef.name")); + query = new Query(HttpRequest.createTestRequest("?query=test&profileRef=ref:MyProfile2", Method.GET), registry.getComponent("default")); + assertEquals("MyProfile2", query.properties().get("profileRef.name")); } } + @Test + public void testTensorTypes() { + CompiledQueryProfileRegistry registry = new QueryProfileXMLReader().read("src/test/java/com/yahoo/search/query/profile/config/test/tensortypes").compile(); + + QueryProfileType type1 = registry.getTypeRegistry().getComponent("type1"); + assertEquals("tensor<float>(x[1])", type1.getFieldType(new CompoundName("ranking.features.query(tensor_1)")).stringValue()); + assertNull(type1.getFieldType(new CompoundName("ranking.features.query(tensor_2)"))); + assertNull(type1.getFieldType(new CompoundName("ranking.features.query(tensor_3)"))); + + QueryProfileType type2 = registry.getTypeRegistry().getComponent("type2"); + assertNull(type2.getFieldType(new CompoundName("ranking.features.query(tensor_1)"))); + assertEquals("tensor<float>(x[2])", type2.getFieldType(new CompoundName("ranking.features.query(tensor_2)")).stringValue()); + assertEquals("tensor<float>(x[3])", type2.getFieldType(new CompoundName("ranking.features.query(tensor_3)")).stringValue()); + + Query queryProfile1 = new Query("?query=test&ranking.features.query(tensor_1)=[1.200]", registry.getComponent("profile1")); + assertEquals("Is received as a tensor tensor", "tensor<float>(x[1]):[1.2]", queryProfile1.properties().get("ranking.features.query(tensor_1)").toString()); + + Query queryProfile2 = new Query("?query=test&ranking.features.query(tensor_1)=[1.200]", registry.getComponent("profile2")); + assertEquals("Is received as a string", "[1.200]", queryProfile2.properties().get("ranking.features.query(tensor_1)").toString()); + } + } diff --git a/container-search/src/test/java/com/yahoo/search/query/profile/config/test/tensortypes/profile1.xml b/container-search/src/test/java/com/yahoo/search/query/profile/config/test/tensortypes/profile1.xml new file mode 100644 index 00000000000..000fd3e1c5b --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/query/profile/config/test/tensortypes/profile1.xml @@ -0,0 +1,2 @@ +<query-profile id="profile1" type="type1"> +</query-profile> diff --git a/container-search/src/test/java/com/yahoo/search/query/profile/config/test/tensortypes/profile2.xml b/container-search/src/test/java/com/yahoo/search/query/profile/config/test/tensortypes/profile2.xml new file mode 100644 index 00000000000..f6539da23e8 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/query/profile/config/test/tensortypes/profile2.xml @@ -0,0 +1,2 @@ +<query-profile id="profile2" type="type2"> +</query-profile> diff --git a/container-search/src/test/java/com/yahoo/search/query/profile/config/test/tensortypes/types/type1.xml b/container-search/src/test/java/com/yahoo/search/query/profile/config/test/tensortypes/types/type1.xml new file mode 100644 index 00000000000..3dfaab9c5f2 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/query/profile/config/test/tensortypes/types/type1.xml @@ -0,0 +1,3 @@ +<query-profile-type id="type1"> + <field name="ranking.features.query(tensor_1)" type="tensor<float>(x[1])" /> +</query-profile-type> diff --git a/container-search/src/test/java/com/yahoo/search/query/profile/config/test/tensortypes/types/type2.xml b/container-search/src/test/java/com/yahoo/search/query/profile/config/test/tensortypes/types/type2.xml new file mode 100644 index 00000000000..ed7cf23e464 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/query/profile/config/test/tensortypes/types/type2.xml @@ -0,0 +1,4 @@ +<query-profile-type id="type2"> + <field name="ranking.features.query(tensor_2)" type="tensor<float>(x[2])" /> + <field name="ranking.features.query(tensor_3)" type="tensor<float>(x[3])" /> +</query-profile-type> diff --git a/container-search/src/test/java/com/yahoo/search/query/profile/test/QueryProfileTestCase.java b/container-search/src/test/java/com/yahoo/search/query/profile/test/QueryProfileTestCase.java index bda191ee910..eb1584efe84 100644 --- a/container-search/src/test/java/com/yahoo/search/query/profile/test/QueryProfileTestCase.java +++ b/container-search/src/test/java/com/yahoo/search/query/profile/test/QueryProfileTestCase.java @@ -504,7 +504,8 @@ public class QueryProfileTestCase { p.set("a","a-value", null); p.set("a.b","a.b-value", null); Map<String, Object> values = p.compile(null).listValues("a"); - assertEquals(1, values.size()); + assertEquals(2, values.size()); + p.set("a","a-value", null); assertEquals("a.b-value", values.get("b")); } diff --git a/container-search/src/test/java/com/yahoo/search/query/profile/types/test/QueryProfileTypeTestCase.java b/container-search/src/test/java/com/yahoo/search/query/profile/types/test/QueryProfileTypeTestCase.java index c05c3589a30..3c200debcaf 100644 --- a/container-search/src/test/java/com/yahoo/search/query/profile/types/test/QueryProfileTypeTestCase.java +++ b/container-search/src/test/java/com/yahoo/search/query/profile/types/test/QueryProfileTypeTestCase.java @@ -397,7 +397,7 @@ public class QueryProfileTypeTestCase { @Test public void testTensorRankFeatureInRequest() throws UnsupportedEncodingException { - QueryProfile profile=new QueryProfile("test"); + QueryProfile profile = new QueryProfile("test"); profile.setType(type); registry.register(profile); @@ -447,25 +447,25 @@ public class QueryProfileTypeTestCase { */ @Test public void testTypedOverridingOfQueryProfileReferencesNonStrictThroughQueryNestedInAnUntypedProfile() { - QueryProfile topMap=new QueryProfile("topMap"); + QueryProfile topMap = new QueryProfile("topMap"); - QueryProfile subMap=new QueryProfile("topSubMap"); - topMap.set("subMap",subMap, registry); + QueryProfile subMap = new QueryProfile("topSubMap"); + topMap.set("subMap", subMap, registry); - QueryProfile test=new QueryProfile("test"); + QueryProfile test = new QueryProfile("test"); test.setType(type); - subMap.set("typeProfile",test, registry); + subMap.set("typeProfile", test, registry); - QueryProfile myUser=new QueryProfile("myUser"); + QueryProfile myUser = new QueryProfile("myUser"); myUser.setType(user); - myUser.set("myUserString","userValue1", registry); - myUser.set("myUserInteger",442, registry); - test.set("myUserQueryProfile",myUser, registry); + myUser.set("myUserString", "userValue1", registry); + myUser.set("myUserInteger", 442, registry); + test.set("myUserQueryProfile", myUser, registry); - QueryProfile newUser=new QueryProfile("newUser"); + QueryProfile newUser = new QueryProfile("newUser"); newUser.setType(user); - newUser.set("myUserString","newUserValue1", registry); - newUser.set("myUserInteger",845, registry); + newUser.set("myUserString", "newUserValue1", registry); + newUser.set("myUserInteger", 845, registry); registry.register(topMap); registry.register(subMap); diff --git a/container-search/src/test/java/com/yahoo/search/test/QueryTestCase.java b/container-search/src/test/java/com/yahoo/search/test/QueryTestCase.java index e71ed308aaf..34c3da395b7 100644 --- a/container-search/src/test/java/com/yahoo/search/test/QueryTestCase.java +++ b/container-search/src/test/java/com/yahoo/search/test/QueryTestCase.java @@ -45,6 +45,8 @@ import org.junit.Test; import java.io.UnsupportedEncodingException; import java.net.URLEncoder; import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; import java.util.HashSet; import java.util.List; import java.util.Map; @@ -345,6 +347,61 @@ public class QueryTestCase { } @Test + public void testQueryProfileClearAndSet() { + QueryProfile profile = new QueryProfile("myProfile"); + profile.set("b", "b-value", null); + Query q = new Query(QueryTestCase.httpEncode("/search?queryProfile=myProfile"), profile.compile(null)); + assertEquals("b-value", q.properties().get("b")); + assertContains(q.properties().listProperties("b"), "b-value"); + + q.properties().set("b", null, null); + assertContains(q.properties().listProperties("b"), (Object)null); + + q.properties().set("b", "b-value", null); + assertEquals("b-value", q.properties().get("b")); + assertContains(q.properties().listProperties("b"), "b-value"); + } + + @Test + public void testQueryProfileClearValue() { + QueryProfile profile = new QueryProfile("myProfile"); + profile.set("a", "a-value", null); + profile.set("b", "b-value", null); + profile.set("b.c", "b.c-value", null); + profile.set("b.d", "b.d-value", null); + Query q = new Query(QueryTestCase.httpEncode("/search?queryProfile=myProfile"), profile.compile(null)); + assertEquals("a-value", q.properties().get("a")); + assertEquals("b-value", q.properties().get("b")); + assertEquals("b.c-value", q.properties().get("b.c")); + assertEquals("b.d-value", q.properties().get("b.d")); + assertContains(q.properties().listProperties("b"), "b-value", "b.c-value", "b.d-value"); + + q.properties().set("a", null, null); + assertEquals(null, q.properties().get("a")); + + q.properties().set("b", null, null); + assertEquals(null, q.properties().get("b")); + assertEquals("b.c-value", q.properties().get("b.c")); + assertEquals("b.d-value", q.properties().get("b.d")); + assertContains(q.properties().listProperties("b"), null, "b.c-value", "b.d-value"); + + q.properties().set("b", "b-value", null); + q.properties().set("b.e", "b.e-value", null); + q.properties().set("b.f", "b.f-value", null); + assertEquals("b-value", q.properties().get("b")); + assertEquals("b.e-value", q.properties().get("b.e")); + assertContains(q.properties().listProperties("b"), "b-value", "b.c-value", "b.d-value", "b.e-value", "b.f-value"); + + q.properties().clearAll("b"); + assertEquals(null, q.properties().get("b")); + assertEquals(null, q.properties().get("b.c")); + assertEquals(null, q.properties().get("b.d")); + assertEquals(null, q.properties().get("b.e")); + assertEquals(null, q.properties().get("b.f")); + assertContains(q.properties().listProperties("b"), (Object)null); + } + + @Test public void testNotEqual() { Query q = new Query("/?query=something+test&nocache"); Query p = new Query("/?query=something+test"); @@ -985,6 +1042,19 @@ public class QueryTestCase { assertEquals(expectedDetectionText, mockLinguistics.detector.lastDetectionText); } + private void assertContains(Map<String, Object> properties, Object ... expectedValues) { + if (expectedValues == null) { + assertEquals(1, properties.size()); + assertTrue("Contains value null", properties.containsValue(null)); + } + else { + assertEquals(properties + " contains values " + Arrays.toString(expectedValues), + expectedValues.length, properties.size()); + for (Object expectedValue : expectedValues) + assertTrue("Contains value " + expectedValue, properties.containsValue(expectedValue)); + } + } + /** A linguistics instance which records the last language detection text passed to it */ private static class MockLinguistics extends SimpleLinguistics { diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/user/Roles.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/user/Roles.java index 77bd589f23b..f5f3ebe8f35 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/user/Roles.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/user/Roles.java @@ -35,6 +35,7 @@ public class Roles { public static Role toRole(String value) { String[] parts = value.split("\\."); if (parts.length == 1 && parts[0].equals("hostedOperator")) return Role.hostedOperator(); + if (parts.length == 1 && parts[0].equals("hostedSupporter")) return Role.hostedSupporter(); if (parts.length == 2) return toRole(TenantName.from(parts[0]), parts[1]); if (parts.length == 3) return toRole(TenantName.from(parts[0]), ApplicationName.from(parts[1]), parts[2]); throw new IllegalArgumentException("Malformed or illegal role value '" + value + "'."); diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/role/PathGroup.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/role/PathGroup.java index 998af030b6b..67a6faac606 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/role/PathGroup.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/role/PathGroup.java @@ -31,7 +31,9 @@ enum PathGroup { "/os/v1/{*}", "/provision/v2/{*}", "/zone/v2/{*}", - "/routing/v1/{*}"), + "/routing/v1/", + "/routing/v1/status/environment/{*}", + "/routing/v1/inactive/environment/{*}"), /** Paths used for creating and reading user resources. */ user(Optional.of("/api"), @@ -53,7 +55,8 @@ enum PathGroup { Optional.of("/api"), "/application/v4/tenant/{tenant}/application/", "/application/v4/tenant/{tenant}/cost", - "/application/v4/tenant/{tenant}/cost/{date}"), + "/application/v4/tenant/{tenant}/cost/{date}", + "/routing/v1/status/tenant/{tenant}/{*}"), tenantKeys(Matcher.tenant, Optional.of("/api"), @@ -97,7 +100,8 @@ enum PathGroup { "/application/v4/tenant/{tenant}/application/{application}/environment/{environment}/region/{region}/instance/{ignored}/suspended", "/application/v4/tenant/{tenant}/application/{application}/environment/{environment}/region/{region}/instance/{ignored}/service/{*}", "/application/v4/tenant/{tenant}/application/{application}/environment/{environment}/region/{region}/instance/{ignored}/global-rotation/{*}", - "/application/v4/tenant/{tenant}/application/{application}/metering"), + "/application/v4/tenant/{tenant}/application/{application}/metering", + "/routing/v1/inactive/tenant/{tenant}/application/{application}/instance/{ignored}/environment/prod/region/{region}"), // TODO jonmv: remove /** Path used to restart development nodes. */ diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/role/Policy.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/role/Policy.java index e27fb0fbf27..0e8e3a13f9f 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/role/Policy.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/role/Policy.java @@ -23,6 +23,11 @@ enum Policy { .on(PathGroup.all()) .in(SystemName.all())), + /** Full access to everything. */ + supporter(Privilege.grant(Action.read) + .on(PathGroup.all()) + .in(SystemName.all())), + /** Full access to user management for a tenant in select systems. */ tenantManager(Privilege.grant(Action.all()) .on(PathGroup.tenantUsers) diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/role/Role.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/role/Role.java index b53cf9162e7..263e3284dbd 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/role/Role.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/role/Role.java @@ -28,6 +28,11 @@ public abstract class Role { return new UnboundRole(RoleDefinition.hostedOperator); } + /** Returns a {@link RoleDefinition#hostedSupporter} for the current system. */ + public static UnboundRole hostedSupporter() { + return new UnboundRole(RoleDefinition.hostedSupporter); + } + /** Returns a {@link RoleDefinition#everyone} for the current system. */ public static UnboundRole everyone() { return new UnboundRole(RoleDefinition.everyone); diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/role/RoleDefinition.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/role/RoleDefinition.java index 58d69512feb..848866f7c33 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/role/RoleDefinition.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/role/RoleDefinition.java @@ -21,6 +21,9 @@ public enum RoleDefinition { /** Deus ex machina. */ hostedOperator(Policy.operator), + /** Machina autem exspiravit. */ + hostedSupporter(Policy.supporter), + /** Base role which every user is part of. */ everyone(Policy.classifiedRead, Policy.classifiedApiRead, diff --git a/controller-api/src/test/java/com/yahoo/vespa/hosted/controller/api/integration/user/RolesTest.java b/controller-api/src/test/java/com/yahoo/vespa/hosted/controller/api/integration/user/RolesTest.java index cfb5462e50a..22baedd16b4 100644 --- a/controller-api/src/test/java/com/yahoo/vespa/hosted/controller/api/integration/user/RolesTest.java +++ b/controller-api/src/test/java/com/yahoo/vespa/hosted/controller/api/integration/user/RolesTest.java @@ -27,6 +27,8 @@ public class RolesTest { assertEquals(Role.hostedOperator(), Roles.toRole("hostedOperator")); + assertEquals(Role.hostedSupporter(), + Roles.toRole("hostedSupporter")); assertEquals(Role.tenantOperator(tenant), Roles.toRole("my-tenant.tenantOperator")); assertEquals(Role.applicationReader(tenant, application), diff --git a/controller-api/src/test/java/com/yahoo/vespa/hosted/controller/api/role/RoleTest.java b/controller-api/src/test/java/com/yahoo/vespa/hosted/controller/api/role/RoleTest.java index d153e218640..5348185c276 100644 --- a/controller-api/src/test/java/com/yahoo/vespa/hosted/controller/api/role/RoleTest.java +++ b/controller-api/src/test/java/com/yahoo/vespa/hosted/controller/api/role/RoleTest.java @@ -8,6 +8,7 @@ import com.yahoo.config.provision.TenantName; import org.junit.Test; import java.net.URI; +import java.util.List; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; @@ -30,6 +31,31 @@ public class RoleTest { assertTrue(mainEnforcer.allows(role, Action.update, URI.create("/os/v1/bar"))); assertTrue(mainEnforcer.allows(role, Action.update, URI.create("/application/v4/tenant/t1/application/a1"))); assertTrue(mainEnforcer.allows(role, Action.update, URI.create("/application/v4/tenant/t2/application/a2"))); + assertTrue(mainEnforcer.allows(role, Action.read, URI.create("/routing/v1/"))); + assertTrue(mainEnforcer.allows(role, Action.read, URI.create("/routing/v1/status/environment/"))); + assertTrue(mainEnforcer.allows(role, Action.read, URI.create("/routing/v1/status/environment/prod"))); + assertTrue(mainEnforcer.allows(role, Action.create, URI.create("/routing/v1/inactive/environment/prod/region/us-north-1"))); + } + + @Test + public void supporter_membership() { + Role role = Role.hostedSupporter(); + + // No create update or delete + assertFalse(mainEnforcer.allows(role, Action.create, URI.create("/not/explicitly/defined"))); + assertFalse(mainEnforcer.allows(role, Action.create, URI.create("/controller/v1/foo"))); + assertFalse(mainEnforcer.allows(role, Action.update, URI.create("/os/v1/bar"))); + assertFalse(mainEnforcer.allows(role, Action.update, URI.create("/application/v4/tenant/t1/application/a1"))); + assertFalse(mainEnforcer.allows(role, Action.update, URI.create("/application/v4/tenant/t2/application/a2"))); + assertFalse(mainEnforcer.allows(role, Action.delete, URI.create("/application/v4/tenant/t8/application/a6/instance/i1/environment/dev/region/r1"))); + + // But reads is ok (but still only for valid paths) + assertFalse(mainEnforcer.allows(role, Action.read, URI.create("/not/explicitly/defined"))); + assertTrue(mainEnforcer.allows(role, Action.read, URI.create("/controller/v1/foo"))); + assertTrue(mainEnforcer.allows(role, Action.read, URI.create("/os/v1/bar"))); + assertTrue(mainEnforcer.allows(role, Action.read, URI.create("/application/v4/tenant/t1/application/a1"))); + assertTrue(mainEnforcer.allows(role, Action.read, URI.create("/application/v4/tenant/t2/application/a2"))); + assertFalse(mainEnforcer.allows(role, Action.delete, URI.create("/application/v4/tenant/t8/application/a6/instance/i1/environment/dev/region/r1"))); } @Test @@ -133,13 +159,42 @@ public class RoleTest { Action action = Action.update; assertTrue(mainEnforcer.allows(Role.systemFlagsDeployer(), action, deployUri)); assertTrue(mainEnforcer.allows(Role.hostedOperator(), action, deployUri)); + assertFalse(mainEnforcer.allows(Role.hostedSupporter(), action, deployUri)); assertFalse(mainEnforcer.allows(Role.systemFlagsDryrunner(), action, deployUri)); assertFalse(mainEnforcer.allows(Role.everyone(), action, deployUri)); URI dryrunUri = URI.create("/system-flags/v1/dryrun"); assertTrue(mainEnforcer.allows(Role.systemFlagsDeployer(), action, dryrunUri)); assertTrue(mainEnforcer.allows(Role.hostedOperator(), action, dryrunUri)); + assertFalse(mainEnforcer.allows(Role.hostedSupporter(), action, dryrunUri)); assertTrue(mainEnforcer.allows(Role.systemFlagsDryrunner(), action, dryrunUri)); assertFalse(mainEnforcer.allows(Role.everyone(), action, dryrunUri)); } + + @Test + public void routing() { + var tenantUrl = URI.create("/routing/v1/status/tenant/t1"); + var applicationUrl = URI.create("/routing/v1/status/tenant/t1/application/a1"); + var instanceUrl = URI.create("/routing/v1/status/tenant/t1/application/a1/instance/i1"); + var deploymentUrl = URI.create("/routing/v1/status/tenant/t1/application/a1/instance/i1/environment/prod/region/us-north-1"); + // Read + for (var url : List.of(tenantUrl, applicationUrl, instanceUrl, deploymentUrl)) { + var allowedRole = Role.reader(TenantName.from("t1")); + var disallowedRole = Role.reader(TenantName.from("t2")); + assertTrue(allowedRole + " can read " + url, mainEnforcer.allows(allowedRole, Action.read, url)); + assertFalse(disallowedRole + " cannot read " + url, mainEnforcer.allows(disallowedRole, Action.read, url)); + } + + // Write + { + var url = URI.create("/routing/v1/inactive/tenant/t1/application/a1/instance/i1/environment/prod/region/us-north-1"); + var allowedRole = Role.applicationAdmin(TenantName.from("t1"), ApplicationName.from("a1")); + var disallowedRole = Role.applicationAdmin(TenantName.from("t2"), ApplicationName.from("a2")); + assertTrue(allowedRole + " can override status at " + url, mainEnforcer.allows(allowedRole, Action.create, url)); + assertTrue(allowedRole + " can clear status at " + url, mainEnforcer.allows(allowedRole, Action.delete, url)); + assertFalse(disallowedRole + " cannot override status at " + url, mainEnforcer.allows(disallowedRole, Action.create, url)); + assertFalse(disallowedRole + " cannot clear status at " + url, mainEnforcer.allows(disallowedRole, Action.delete, url)); + } + } + } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Application.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Application.java index e37d6accd89..9815fbca093 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Application.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Application.java @@ -107,6 +107,12 @@ public class Application { /** Returns the instances of this application */ public Map<InstanceName, Instance> instances() { return instances; } + /** Returns the instances of this application which are defined in its deployment spec. */ + public Map<InstanceName, Instance> productionInstances() { + return deploymentSpec.instanceNames().stream() + .collect(Collectors.toUnmodifiableMap(Function.identity(), instances::get)); + } + /** Returns the instance with the given name, if it exists. */ public Optional<Instance> get(InstanceName instance) { return Optional.ofNullable(instances.get(instance)); } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/impl/AthenzFacade.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/impl/AthenzFacade.java index 7ace62ab44d..628d7f48c85 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/impl/AthenzFacade.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/impl/AthenzFacade.java @@ -215,6 +215,10 @@ public class AthenzFacade implements AccessControl { return hasAccess("modify", service.getDomain().getName() + ":hosted-vespa", identity); } + public boolean hasHostedSupporterAccess(AthenzIdentity identity) { + return hasAccess("read", service.getDomain().getName() + ":hosted-vespa", identity); + } + public boolean canLaunch(AthenzIdentity principal, AthenzService service) { return hasAccess("launch", service.getDomain().getName() + ":service."+service.getName(), principal); } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentStatus.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentStatus.java index 3a60c480100..6aebae66bad 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentStatus.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentStatus.java @@ -17,7 +17,6 @@ import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobId; import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType; import com.yahoo.vespa.hosted.controller.application.Change; import com.yahoo.vespa.hosted.controller.application.Deployment; -import com.yahoo.vespa.hosted.controller.versions.VersionStatus; import java.time.Duration; import java.time.Instant; @@ -579,8 +578,12 @@ public class DeploymentStatus { Versions versions = Versions.from(change, status.application, status.deploymentFor(job.id()), status.systemVersion); return job.lastSuccess() .filter(run -> versions.targetsMatch(run.versions())) - .filter(run -> status.instanceJobs(instance).get(prodType).lastCompleted() - .map(last -> ! last.end().get().isAfter(run.start())).orElse(false)) + .filter(run -> ! status.jobs() + .instance(instance) + .type(prodType) + .successOn(versions) + .lastCompleted().endedNoLaterThan(run.start()) + .isEmpty()) .map(run -> run.end().get()); } }; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentStatusList.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentStatusList.java index c14493a0b72..efa21b71936 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentStatusList.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentStatusList.java @@ -3,15 +3,9 @@ package com.yahoo.vespa.hosted.controller.deployment; import com.yahoo.collections.AbstractFilteringList; import com.yahoo.component.Version; -import com.yahoo.config.application.api.DeploymentSpec; -import com.yahoo.vespa.hosted.controller.Instance; -import com.yahoo.vespa.hosted.controller.application.ApplicationList; -import com.yahoo.vespa.hosted.controller.application.TenantAndApplicationId; import java.time.Instant; import java.util.Collection; -import java.util.List; -import java.util.Optional; /** * List for filtering deployment status of applications, for inspection and decision making. @@ -48,14 +42,14 @@ public class DeploymentStatusList extends AbstractFilteringList<DeploymentStatus private static boolean failingUpgradeToVersionSince(JobList jobs, Version version, Instant threshold) { return ! jobs.not().failingApplicationChange() - .firstFailing().endedBefore(threshold) + .firstFailing().endedNoLaterThan(threshold) .lastCompleted().on(version) .isEmpty(); } private static boolean failingApplicationChangeSince(JobList jobs, Instant threshold) { return ! jobs.failingApplicationChange() - .firstFailing().endedBefore(threshold) + .firstFailing().endedNoLaterThan(threshold) .isEmpty(); } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java index 61dc249feaa..060ffd63fb3 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java @@ -13,6 +13,7 @@ import com.yahoo.config.provision.AthenzService; import com.yahoo.config.provision.ClusterSpec; import com.yahoo.config.provision.NodeResources; import com.yahoo.config.provision.zone.ZoneId; +import com.yahoo.log.LogLevel; import com.yahoo.security.KeyAlgorithm; import com.yahoo.security.KeyUtils; import com.yahoo.security.SignatureAlgorithm; @@ -25,7 +26,9 @@ import com.yahoo.vespa.hosted.controller.api.ActivateResult; import com.yahoo.vespa.hosted.controller.api.application.v4.model.DeployOptions; import com.yahoo.vespa.hosted.controller.api.identifiers.DeploymentId; import com.yahoo.vespa.hosted.controller.api.identifiers.Hostname; +import com.yahoo.vespa.hosted.controller.api.integration.LogEntry; import com.yahoo.vespa.hosted.controller.api.integration.configserver.ConfigServerException; +import com.yahoo.vespa.hosted.controller.api.integration.configserver.Log; import com.yahoo.vespa.hosted.controller.api.integration.configserver.Node; import com.yahoo.vespa.hosted.controller.api.integration.configserver.PrepareResponse; import com.yahoo.vespa.hosted.controller.api.integration.configserver.ServiceConvergence; @@ -224,6 +227,13 @@ public class InternalStepRunner implements StepRunner { Instant startTime, DualLogger logger) { try { PrepareResponse prepareResponse = deployment.get().prepareResponse(); + if (prepareResponse.log != null) + logger.logAll(prepareResponse.log.stream() + .map(entry -> new LogEntry(0, // Sequenced by BufferedLogStore. + Instant.ofEpochMilli(entry.time), + LogEntry.typeOf(LogLevel.parse(entry.level)), + entry.message)) + .collect(toList())); if ( ! prepareResponse.configChangeActions.refeedActions.stream().allMatch(action -> action.allowed)) { List<String> messages = new ArrayList<>(); messages.add("Deploy failed due to non-compatible changes that require re-feed."); @@ -237,10 +247,6 @@ public class InternalStepRunner implements StepRunner { .filter(action -> ! action.allowed) .flatMap(action -> action.messages.stream()) .forEach(messages::add); - messages.add("Details:"); - prepareResponse.log.stream() - .map(entry -> entry.message) - .forEach(messages::add); logger.log(messages); return Optional.of(deploymentFailed); } @@ -255,7 +261,7 @@ public class InternalStepRunner implements StepRunner { .map(Hostname::new) .forEach(hostname -> { controller.applications().restart(new DeploymentId(id, type.zone(controller.system())), Optional.of(hostname)); - logger.log("Restarting services on host " + hostname.id() + "."); + logger.log("Schedule service restart on host " + hostname.id() + "."); }); logger.log("Deployment successful."); if (prepareResponse.message != null) @@ -830,6 +836,10 @@ public class InternalStepRunner implements StepRunner { log(List.of(messages)); } + private void logAll(List<LogEntry> messages) { + controller.jobController().log(id, step, messages); + } + private void log(List<String> messages) { controller.jobController().log(id, step, INFO, messages); } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobList.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobList.java index f353910163f..525eadb6eaf 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobList.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobList.java @@ -117,7 +117,7 @@ public class JobList extends AbstractFilteringList<JobStatus, JobList> { return new RunFilter(JobStatus::firstFailing); } - /** Allows sub-filters for runs of the given kind */ + /** Allows sub-filters for runs of the indicated kind */ public class RunFilter { private final Function<JobStatus, Optional<Run>> which; @@ -126,47 +126,32 @@ public class JobList extends AbstractFilteringList<JobStatus, JobList> { this.which = which; } - /** Returns the subset of jobs where the run of the given type exists */ + /** Returns the subset of jobs where the run of the indicated type exists */ public JobList present() { return matching(run -> true); } - /** Returns the runs of the given kind, mapped by the given function, as a list. */ + /** Returns the runs of the indicated kind, mapped by the given function, as a list. */ public <OtherType> List<OtherType> mapToList(Function<? super Run, OtherType> mapper) { return present().mapToList(which.andThen(Optional::get).andThen(mapper)); } - /** Returns the runs of the given kind. */ + /** Returns the runs of the indicated kind. */ public List<Run> asList() { return mapToList(Function.identity()); } - /** Returns the subset of jobs where the run of the given type occurred before the given instant */ - public JobList endedBefore(Instant threshold) { - return matching(run -> run.end().orElse(Instant.MAX).isBefore(threshold)); + /** Returns the subset of jobs where the run of the indicated type ended no later than the given instant */ + public JobList endedNoLaterThan(Instant threshold) { + return matching(run -> ! run.end().orElse(Instant.MAX).isAfter(threshold)); } - /** Returns the subset of jobs where the run of the given type occurred after the given instant */ - public JobList endedAfter(Instant threshold) { - return matching(run -> run.end().orElse(Instant.MIN).isAfter(threshold)); - } - - /** Returns the subset of jobs where the run of the given type occurred before the given instant */ - public JobList startedBefore(Instant threshold) { - return matching(run -> run.start().isBefore(threshold)); - } - - /** Returns the subset of jobs where the run of the given type occurred after the given instant */ - public JobList startedAfter(Instant threshold) { - return matching(run -> run.start().isAfter(threshold)); - } - - /** Returns the subset of jobs where the run of the given type was on the given version */ + /** Returns the subset of jobs where the run of the indicated type was on the given version */ public JobList on(ApplicationVersion version) { return matching(run -> run.versions().targetApplication().equals(version)); } - /** Returns the subset of jobs where the run of the given type was on the given version */ + /** Returns the subset of jobs where the run of the indicated type was on the given version */ public JobList on(Version version) { return matching(run -> run.versions().targetPlatform().equals(version)); } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/endpointcertificates/EndpointCertificateManager.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/endpointcertificates/EndpointCertificateManager.java index 071c1217b14..1bb449b0a16 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/endpointcertificates/EndpointCertificateManager.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/endpointcertificates/EndpointCertificateManager.java @@ -129,12 +129,7 @@ public class EndpointCertificateManager { throw new RuntimeException("Backfill failed - provider metadata missing request_id"); if (providerMetadata.requestedDnsSans().isEmpty()) throw new RuntimeException("Backfill failed - provider metadata missing DNS SANs for " + providerMetadata.request_id().get()); - providerMetadata.requestedDnsSans().get().forEach(san -> { - var previous = sanToEndpointCertificate.put(san, providerMetadata); - if (previous != null) - throw new RuntimeException("Backfill failed - Overlapping SANs in certificates " + - providerMetadata.request_id() + " and " + previous.request_id()); - } + providerMetadata.requestedDnsSans().get().forEach(san -> sanToEndpointCertificate.put(san, providerMetadata) ); })); diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/CloudEventReporter.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/CloudEventReporter.java index bd8faaed2e2..199e9e4e7ae 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/CloudEventReporter.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/CloudEventReporter.java @@ -49,6 +49,9 @@ public class CloudEventReporter extends Maintainer { for (var awsRegion : zonesByCloudNativeRegion.keySet()) { List<CloudEvent> events = eventFetcher.getEvents(awsRegion); for (var event : events) { + log.info(String.format("Retrieved event %s, affecting the following instances: %s", + event.instanceEventId, + event.affectedInstances)); List<String> deprovisionedHosts = deprovisionHosts(awsRegion, event); submitIssue(event, deprovisionedHosts); } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java index f37304025ac..0f4eca6fd58 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java @@ -13,6 +13,7 @@ import com.yahoo.config.provision.Environment; import com.yahoo.config.provision.InstanceName; import com.yahoo.config.provision.NodeResources; import com.yahoo.config.provision.TenantName; +import com.yahoo.config.provision.zone.RoutingMethod; import com.yahoo.config.provision.zone.ZoneId; import com.yahoo.container.jdisc.HttpRequest; import com.yahoo.container.jdisc.HttpResponse; @@ -227,6 +228,7 @@ public class ApplicationApiHandler extends LoggingRequestHandler { if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/deploying/pin")) return deploying(path.get("tenant"), path.get("application"), path.get("instance"), request); if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/job")) return JobControllerApiHandlerHelper.jobTypeResponse(controller, appIdFromPath(path), request.getUri()); if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/job/{jobtype}")) return JobControllerApiHandlerHelper.runResponse(controller.jobController().runs(appIdFromPath(path), jobTypeFromPath(path)), request.getUri()); + if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/job/{jobtype}/package")) return devApplicationPackage(appIdFromPath(path), jobTypeFromPath(path)); if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/job/{jobtype}/test-config")) return testConfig(appIdFromPath(path), jobTypeFromPath(path)); if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/job/{jobtype}/run/{number}")) return JobControllerApiHandlerHelper.runDetailsResponse(controller.jobController(), runIdFromPath(path), request.getProperty("after")); if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/environment/{environment}/region/{region}")) return deployment(path.get("tenant"), path.get("application"), path.get("instance"), path.get("environment"), path.get("region"), request); @@ -439,6 +441,15 @@ public class ApplicationApiHandler extends LoggingRequestHandler { return new SlimeJsonResponse(slime); } + private HttpResponse devApplicationPackage(ApplicationId id, JobType type) { + if ( ! type.environment().isManuallyDeployed()) + throw new IllegalArgumentException("Only manually deployed zones have dev packages"); + + ZoneId zone = type.zone(controller.system()); + byte[] applicationPackage = controller.applications().applicationStore().getDev(id, zone); + return new ZipResponse(id.toFullString() + "." + zone.value() + ".zip", applicationPackage); + } + private HttpResponse applicationPackage(String tenantName, String applicationName, HttpRequest request) { var tenantAndApplication = TenantAndApplicationId.from(tenantName, applicationName); var applicationId = ApplicationId.from(tenantName, applicationName, InstanceName.defaultName().value()); @@ -733,7 +744,8 @@ public class ApplicationApiHandler extends LoggingRequestHandler { application.majorVersion().ifPresent(majorVersion -> object.setLong("majorVersion", majorVersion)); Cursor instancesArray = object.setArray("instances"); - for (Instance instance : application.instances().values()) + for (Instance instance : showOnlyProductionInstances(request) ? application.productionInstances().values() + : application.instances().values()) toSlime(instancesArray.addObject(), status, instance, application.deploymentSpec(), request); application.deployKeys().stream().map(KeyUtils::toPem).forEach(object.setArray("pemDeployKeys")::addString); @@ -1040,6 +1052,7 @@ public class ApplicationApiHandler extends LoggingRequestHandler { endpointObject.setBool("tls", endpoint.tls()); endpointObject.setString("url", endpoint.url().toString()); endpointObject.setString("scope", endpointScopeString(endpoint.scope())); + endpointObject.setString("routingMethod", routingMethodString(RoutingMethod.exclusive)); } // Add global endpoints that point to this policy for (var endpoint : policy.globalEndpointsIn(controller.system()).asList()) { @@ -1048,6 +1061,7 @@ public class ApplicationApiHandler extends LoggingRequestHandler { endpointObject.setBool("tls", endpoint.tls()); endpointObject.setString("url", endpoint.url().toString()); endpointObject.setString("scope", endpointScopeString(endpoint.scope())); + endpointObject.setString("routingMethod", routingMethodString(RoutingMethod.exclusive)); } } // Add zone endpoints served by shared routing layer @@ -1057,6 +1071,7 @@ public class ApplicationApiHandler extends LoggingRequestHandler { endpointObject.setBool("tls", true); endpointObject.setString("url", clusterAndUrl.getValue().toString()); endpointObject.setString("scope", endpointScopeString(Endpoint.Scope.zone)); + endpointObject.setString("routingMethod", routingMethodString(RoutingMethod.shared)); } // Add global endpoints served by shared routing layer var application = controller.applications().requireApplication(TenantAndApplicationId.from(deploymentId.applicationId())); @@ -1073,6 +1088,7 @@ public class ApplicationApiHandler extends LoggingRequestHandler { endpointObject.setBool("tls", true); endpointObject.setString("url", endpoint.url().toString()); endpointObject.setString("scope", endpointScopeString(endpoint.scope())); + endpointObject.setString("routingMethod", routingMethodString(RoutingMethod.shared)); } } } @@ -1682,23 +1698,22 @@ public class ApplicationApiHandler extends LoggingRequestHandler { private HttpResponse deleteInstance(String tenantName, String applicationName, String instanceName, HttpRequest request) { TenantAndApplicationId id = TenantAndApplicationId.from(tenantName, applicationName); - Optional<Credentials> credentials = controller.tenants().require(id.tenant()).type() == Tenant.Type.user - ? Optional.empty() - : Optional.of(accessControlRequests.credentials(id.tenant(), toSlime(request.getData()).get(), request.getJDiscRequest())); controller.applications().deleteInstance(id.instance(instanceName)); - if (controller.applications().requireApplication(id).instances().isEmpty()) + if (controller.applications().requireApplication(id).instances().isEmpty()) { + Optional<Credentials> credentials = controller.tenants().require(id.tenant()).type() == Tenant.Type.user + ? Optional.empty() + : Optional.of(accessControlRequests.credentials(id.tenant(), toSlime(request.getData()).get(), request.getJDiscRequest())); controller.applications().deleteApplication(id, credentials); + } return new MessageResponse("Deleted instance " + id.instance(instanceName).toFullString()); } private HttpResponse deactivate(String tenantName, String applicationName, String instanceName, String environment, String region, HttpRequest request) { - Instance instance = controller.applications().requireInstance(ApplicationId.from(tenantName, applicationName, instanceName)); - + DeploymentId id = new DeploymentId(ApplicationId.from(tenantName, applicationName, instanceName), + ZoneId.from(environment, region)); // Attempt to deactivate application even if the deployment is not known by the controller - DeploymentId deploymentId = new DeploymentId(instance.id(), ZoneId.from(environment, region)); - controller.applications().deactivate(deploymentId.applicationId(), deploymentId.zoneId()); - - return new MessageResponse("Deactivated " + deploymentId); + controller.applications().deactivate(id.applicationId(), id.zoneId()); + return new MessageResponse("Deactivated " + id); } /** Returns test config for indicated job, with production deployments of the default instance. */ @@ -1779,7 +1794,8 @@ public class ApplicationApiHandler extends LoggingRequestHandler { Cursor applicationArray = object.setArray("applications"); for (Application application : applications) { DeploymentStatus status = controller.jobController().deploymentStatus(application); - for (Instance instance : application.instances().values()) + for (Instance instance : showOnlyProductionInstances(request) ? application.productionInstances().values() + : application.instances().values()) if (recurseOverApplications(request)) toSlime(applicationArray.addObject(), instance, status, request); else @@ -2011,6 +2027,10 @@ public class ApplicationApiHandler extends LoggingRequestHandler { return ImmutableSet.of("all", "true", "deployment").contains(request.getProperty("recursive")); } + private static boolean showOnlyProductionInstances(HttpRequest request) { + return "true".equals(request.getProperty("production")); + } + private static String tenantType(Tenant tenant) { switch (tenant.type()) { case user: return "USER"; @@ -2036,7 +2056,7 @@ public class ApplicationApiHandler extends LoggingRequestHandler { private HttpResponse submit(String tenant, String application, HttpRequest request) { Map<String, byte[]> dataParts = parseDataParts(request); Inspector submitOptions = SlimeUtils.jsonToSlime(dataParts.get(EnvironmentResource.SUBMIT_OPTIONS)).get(); - long projectId = Math.max(1, submitOptions.field("projectId").asLong()); + long projectId = Math.max(1, submitOptions.field("projectId").asLong()); // Absence of this means it's not a prod app :/ Optional<String> repository = optional("repository", submitOptions); Optional<String> branch = optional("branch", submitOptions); Optional<String> commit = optional("commit", submitOptions); @@ -2117,6 +2137,14 @@ public class ApplicationApiHandler extends LoggingRequestHandler { throw new IllegalArgumentException("Unknown endpoint scope " + scope); } + private static String routingMethodString(RoutingMethod method) { + switch (method) { + case exclusive: return "exclusive"; + case shared: return "shared"; + } + throw new IllegalArgumentException("Unknown routing method " + method); + } + private static <T> T getAttribute(HttpRequest request, String attributeName, Class<T> cls) { return Optional.ofNullable(request.getJDiscRequest().context().get(attributeName)) .filter(cls::isInstance) diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/JobControllerApiHandlerHelper.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/JobControllerApiHandlerHelper.java index 73cdf28c366..32c2d6ec3d1 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/JobControllerApiHandlerHelper.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/JobControllerApiHandlerHelper.java @@ -5,6 +5,7 @@ import com.google.common.base.Joiner; import com.yahoo.component.Version; import com.yahoo.config.application.api.DeploymentInstanceSpec; import com.yahoo.config.application.api.DeploymentSpec; +import com.yahoo.config.application.api.DeploymentSpec.ChangeBlocker; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.SystemName; import com.yahoo.config.provision.zone.ZoneId; @@ -41,14 +42,20 @@ import com.yahoo.vespa.hosted.controller.deployment.Versions; import com.yahoo.vespa.hosted.controller.versions.VespaVersion; import java.net.URI; +import java.time.Instant; +import java.time.format.TextStyle; import java.util.ArrayDeque; +import java.util.Arrays; +import java.util.Collection; import java.util.Comparator; import java.util.Deque; import java.util.LinkedHashMap; import java.util.List; +import java.util.Locale; import java.util.Map; import java.util.Optional; import java.util.stream.Collectors; +import java.util.stream.Stream; import static com.yahoo.config.application.api.DeploymentSpec.UpgradePolicy.conservative; import static com.yahoo.config.application.api.DeploymentSpec.UpgradePolicy.defaultPolicy; @@ -175,6 +182,20 @@ class JobControllerApiHandlerHelper { devJobObject.setString("url", baseUriForJobs.resolve(baseUriForJobs.getPath() + "/" + type.jobName()).normalize().toString()); }); + Cursor jobsArray = responseObject.setArray("deployment"); + Arrays.stream(JobType.values()) + .filter(type -> type.environment().isManuallyDeployed()) + .map(devType -> new JobId(instance.id(), devType)) + .forEach(job -> { + Collection<Run> runs = controller.jobController().runs(job).descendingMap().values(); + if (runs.isEmpty()) + return; + + Cursor jobObject = jobsArray.addObject(); + jobObject.setString("jobName", job.type().jobName()); + toSlime(jobObject.setArray("runs"), runs, baseUriForJobs); + }); + return new SlimeJsonResponse(slime); } @@ -569,12 +590,48 @@ class JobControllerApiHandlerHelper { stepObject.setBool("declared", stepStatus.isDeclared()); stepObject.setString("instance", stepStatus.instance().value()); + stepStatus.readyAt(change).ifPresent(ready -> stepObject.setLong("readyAt", ready.toEpochMilli())); + stepStatus.readyAt(change) + .filter(controller.clock().instant()::isBefore) + .ifPresent(until -> stepObject.setLong("delayedUntil", until.toEpochMilli())); + stepStatus.pausedUntil().ifPresent(until -> stepObject.setLong("pausedUntil", until.toEpochMilli())); + stepStatus.coolingDownUntil(change).ifPresent(until -> stepObject.setLong("coolingDownUntil", until.toEpochMilli())); + stepStatus.blockedUntil(change).ifPresent(until -> stepObject.setLong("blockedUntil", until.toEpochMilli())); + + if (stepStatus.type() == DeploymentStatus.StepType.instance) { + Cursor deployingObject = stepObject.setObject("deploying"); + if ( ! change.isEmpty()) { + change.platform().ifPresent(version -> deployingObject.setString("platform", version.toString())); + change.application().ifPresent(version -> toSlime(deployingObject.setObject("application"), version)); + } + + Cursor latestVersionsObject = stepObject.setObject("latestVersions"); + List<ChangeBlocker> blockers = application.deploymentSpec().requireInstance(stepStatus.instance()).changeBlocker(); + latestVersionPreferablyWithNormalConfidenceAndNotNewerThanSystem(controller.versionStatus().versions()) + .ifPresent(latestPlatform -> { + Cursor latestPlatformObject = latestVersionsObject.setObject("platform"); + latestPlatformObject.setString("platform", latestPlatform.versionNumber().toFullString()); + latestPlatformObject.setLong("at", latestPlatform.committedAt().toEpochMilli()); + latestPlatformObject.setBool("upgrade", application.require(stepStatus.instance()).productionDeployments().values().stream() + .anyMatch(deployment -> deployment.version().isBefore(latestPlatform.versionNumber()))); + toSlime(latestPlatformObject.setArray("blockers"), blockers.stream().filter(ChangeBlocker::blocksVersions)); + }); + application.latestVersion().ifPresent(latestApplication -> { + Cursor latestApplicationObject = latestVersionsObject.setObject("application"); + toSlime(latestApplicationObject.setObject("application"), latestApplication); + latestApplicationObject.setLong("at", latestApplication.buildTime().orElse(Instant.EPOCH).toEpochMilli()); + latestApplicationObject.setBool("upgrade", application.require(stepStatus.instance()).productionDeployments().values().stream() + .anyMatch(deployment -> deployment.applicationVersion().compareTo(latestApplication) < 0)); + toSlime(latestApplicationObject.setArray("blockers"), blockers.stream().filter(ChangeBlocker::blocksRevisions)); + }); + } + stepStatus.job().ifPresent(job -> { stepObject.setString("jobName", job.type().jobName()); - String baseUriForJob = baseUriForDeployments.resolve(baseUriForDeployments.getPath() + + URI baseUriForJob = baseUriForDeployments.resolve(baseUriForDeployments.getPath() + "/../instance/" + job.application().instance().value() + - "/job/" + job.type().jobName()).normalize().toString(); - stepObject.setString("url", baseUriForJob); + "/job/" + job.type().jobName()).normalize(); + stepObject.setString("url", baseUriForJob.toString()); stepObject.setString("environment", job.type().environment().value()); stepObject.setString("region", job.type().zone(controller.system()).value()); @@ -599,41 +656,16 @@ class JobControllerApiHandlerHelper { Cursor runObject = toRunArray.addObject(); toSlime(runObject.setObject("versions"), versions); } - stepStatus.readyAt(change).ifPresent(ready -> stepObject.setLong("readyAt", ready.toEpochMilli())); - stepStatus.readyAt(change) - .filter(controller.clock().instant()::isBefore) - .ifPresent(until -> stepObject.setLong("delayedUntil", until.toEpochMilli())); - stepStatus.pausedUntil().ifPresent(until -> stepObject.setLong("pausedUntil", until.toEpochMilli())); - stepStatus.coolingDownUntil(change).ifPresent(until -> stepObject.setLong("coolingDownUntil", until.toEpochMilli())); - stepStatus.blockedUntil(change).ifPresent(until -> stepObject.setLong("blockedUntil", until.toEpochMilli())); - - Cursor runsArray = stepObject.setArray("runs"); - jobStatus.runs().descendingMap().values().stream().limit(10).forEach(run -> { - Cursor runObject = runsArray.addObject(); - runObject.setLong("id", run.id().number()); - runObject.setString("url", baseUriForJob + "/run/" + run.id()); - runObject.setLong("start", run.start().toEpochMilli()); - run.end().ifPresent(end -> runObject.setLong("end", end.toEpochMilli())); - runObject.setString("status", run.status().name()); - toSlime(runObject.setObject("versions"), run.versions()); - Cursor runStepsArray = runObject.setArray("steps"); - run.steps().forEach((step, info) -> { - Cursor runStepObject = runStepsArray.addObject(); - runStepObject.setString("name", step.name()); - runStepObject.setString("status", info.status().name()); - }); - }); + + toSlime(stepObject.setArray("runs"), jobStatus.runs().descendingMap().values(), baseUriForJob); }); } - // TODO jonmv: Add latest platform and application status. - return new SlimeJsonResponse(slime); } private static void toSlime(Cursor versionObject, ApplicationVersion version) { - version.buildNumber().ifPresent(id -> versionObject.setLong("id", id)); - version.source().ifPresent(source -> versionObject.setString("commit", source.commit())); + version.buildNumber().ifPresent(id -> versionObject.setLong("build", id)); version.compileVersion().ifPresent(platform -> versionObject.setString("compileVersion", platform.toFullString())); version.sourceUrl().ifPresent(url -> versionObject.setString("sourceUrl", url)); version.commit().ifPresent(commit -> versionObject.setString("commit", commit)); @@ -646,5 +678,50 @@ class JobControllerApiHandlerHelper { versions.sourceApplication().ifPresent(application -> toSlime(versionsObject.setObject("sourceApplication"), application)); } -} + private static void toSlime(Cursor blockersArray, Stream<ChangeBlocker> blockers) { + blockers.forEach(blocker -> { + Cursor blockerObject = blockersArray.addObject(); + blocker.window().days().stream() + .map(day -> day.getDisplayName(TextStyle.SHORT, Locale.ENGLISH)) + .forEach(blockerObject.setArray("days")::addString); + blocker.window().hours() + .forEach(blockerObject.setArray("hours")::addLong); + blockerObject.setString("zone", blocker.window().zone().toString()); + }); + } + + private static Optional<VespaVersion> latestVersionPreferablyWithNormalConfidenceAndNotNewerThanSystem(List<VespaVersion> versions) { + int i; + for (i = versions.size(); i-- > 0; ) + if (versions.get(i).isSystemVersion()) + break; + + if (i < 0) + return Optional.empty(); + + for (int j = i; j >= 0; j--) + if (versions.get(j).confidence().equalOrHigherThan(normal)) + return Optional.of(versions.get(j)); + + return Optional.of(versions.get(i)); + } + + private static void toSlime(Cursor runsArray, Collection<Run> runs, URI baseUriForJob) { + runs.stream().limit(10).forEach(run -> { + Cursor runObject = runsArray.addObject(); + runObject.setLong("id", run.id().number()); + runObject.setString("url", baseUriForJob.resolve(baseUriForJob.getPath() + "/run/" + run.id().number()).toString()); + runObject.setLong("start", run.start().toEpochMilli()); + run.end().ifPresent(end -> runObject.setLong("end", end.toEpochMilli())); + runObject.setString("status", run.status().name()); + toSlime(runObject.setObject("versions"), run.versions()); + Cursor runStepsArray = runObject.setArray("steps"); + run.steps().forEach((step, info) -> { + Cursor runStepObject = runStepsArray.addObject(); + runStepObject.setString("name", step.name()); + runStepObject.setString("status", info.status().name()); + }); + }); + } +} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/filter/AthenzRoleFilter.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/filter/AthenzRoleFilter.java index 1aaecb58a8d..ba974521278 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/filter/AthenzRoleFilter.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/filter/AthenzRoleFilter.java @@ -87,6 +87,9 @@ public class AthenzRoleFilter extends JsonSecurityRequestFilterBase { if (athenz.hasHostedOperatorAccess(identity)) roleMemberships.add(Role.hostedOperator()); + if (athenz.hasHostedSupporterAccess(identity)) + roleMemberships.add(Role.hostedSupporter()); + // Add all tenants that are accessible for this request athenz.accessibleTenants(tenants.asList(), new Credentials(principal)) .forEach(accessibleTenant -> roleMemberships.add(Role.athenzTenantAdmin(accessibleTenant.name()))); diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/routing/RoutingApiHandler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/routing/RoutingApiHandler.java index 1815628a1ee..673cb7e82e9 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/routing/RoutingApiHandler.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/routing/RoutingApiHandler.java @@ -2,6 +2,9 @@ package com.yahoo.vespa.hosted.controller.restapi.routing; import com.yahoo.config.provision.ApplicationId; +import com.yahoo.config.provision.ApplicationName; +import com.yahoo.config.provision.InstanceName; +import com.yahoo.config.provision.TenantName; import com.yahoo.config.provision.zone.RoutingMethod; import com.yahoo.config.provision.zone.ZoneId; import com.yahoo.container.jdisc.HttpRequest; @@ -9,21 +12,27 @@ import com.yahoo.container.jdisc.HttpResponse; import com.yahoo.restapi.ErrorResponse; import com.yahoo.restapi.MessageResponse; import com.yahoo.restapi.Path; +import com.yahoo.restapi.ResourceResponse; import com.yahoo.restapi.SlimeJsonResponse; import com.yahoo.slime.Cursor; import com.yahoo.slime.Slime; +import com.yahoo.vespa.hosted.controller.Application; import com.yahoo.vespa.hosted.controller.Controller; import com.yahoo.vespa.hosted.controller.Instance; import com.yahoo.vespa.hosted.controller.api.application.v4.model.EndpointStatus; import com.yahoo.vespa.hosted.controller.api.identifiers.DeploymentId; +import com.yahoo.vespa.hosted.controller.application.TenantAndApplicationId; import com.yahoo.vespa.hosted.controller.auditlog.AuditLoggingRequestHandler; import com.yahoo.vespa.hosted.controller.routing.GlobalRouting; import com.yahoo.vespa.hosted.controller.routing.RoutingPolicy; import com.yahoo.yolean.Exceptions; +import java.net.URI; import java.time.Instant; +import java.util.List; import java.util.Objects; import java.util.logging.Level; +import java.util.stream.Collectors; /** * This implements the /routing/v1 API, which provides operator with global routing control at both zone- and @@ -45,7 +54,7 @@ public class RoutingApiHandler extends AuditLoggingRequestHandler { try { var path = new Path(request.getUri()); switch (request.getMethod()) { - case GET: return get(path); + case GET: return get(path, request); case POST: return post(path); case DELETE: return delete(path); default: return ErrorResponse.methodNotAllowed("Method '" + request.getMethod() + "' is not supported"); @@ -70,12 +79,92 @@ public class RoutingApiHandler extends AuditLoggingRequestHandler { return ErrorResponse.notFoundError("Nothing at " + path); } - private HttpResponse get(Path path) { - if (path.matches("/routing/v1/status/tenant/{tenant}/application/{application}/instance/{instance}/environment/{environment}/region/{region}")) return deploymentStatus(path); - if (path.matches("/routing/v1/status/environment/{environment}/region/{region}")) return zoneStatus(path); + private HttpResponse get(Path path, HttpRequest request) { + if (path.matches("/routing/v1/")) return status(request.getUri()); + if (path.matches("/routing/v1/status/tenant/{tenant}")) return tenant(path, request); + if (path.matches("/routing/v1/status/tenant/{tenant}/application/{application}")) return application(path, request); + if (path.matches("/routing/v1/status/tenant/{tenant}/application/{application}/instance/{instance}")) return instance(path, request); + if (path.matches("/routing/v1/status/tenant/{tenant}/application/{application}/instance/{instance}/environment/{environment}/region/{region}")) return deployment(path); + if (path.matches("/routing/v1/status/environment")) return environment(request); + if (path.matches("/routing/v1/status/environment/{environment}/region/{region}")) return zone(path); return ErrorResponse.notFoundError("Nothing at " + path); } + private HttpResponse environment(HttpRequest request) { + var zones = controller.zoneRegistry().zones().all().ids(); + if (isRecursive(request)) { + var slime = new Slime(); + var root = slime.setObject(); + var zonesArray = root.setArray("zones"); + for (var zone : zones) { + toSlime(zone, zonesArray.addObject()); + } + return new SlimeJsonResponse(slime); + } + var resources = controller.zoneRegistry().zones().all().ids().stream() + .map(zone -> zone.environment().value() + + "/region/" + zone.region().value()) + .sorted() + .collect(Collectors.toList()); + return new ResourceResponse(request.getUri(), resources); + } + + private HttpResponse status(URI requestUrl) { + return new ResourceResponse(requestUrl, "status/tenant", "status/environment"); + } + + private HttpResponse tenant(Path path, HttpRequest request) { + var tenantName = tenantFrom(path); + if (isRecursive(request)) { + var slime = new Slime(); + var root = slime.setObject(); + toSlime(controller.applications().asList(tenantName), null, null, root); + return new SlimeJsonResponse(slime); + } + var resources = controller.applications().asList(tenantName).stream() + .map(Application::id) + .map(TenantAndApplicationId::application) + .map(ApplicationName::value) + .map(application -> "application/" + application) + .sorted() + .collect(Collectors.toList()); + return new ResourceResponse(request.getUri(), resources); + } + + private HttpResponse application(Path path, HttpRequest request) { + var tenantAndApplicationId = tenantAndApplicationIdFrom(path); + if (isRecursive(request)) { + var slime = new Slime(); + var root = slime.setObject(); + toSlime(List.of(controller.applications().requireApplication(tenantAndApplicationId)), null, + null, root); + return new SlimeJsonResponse(slime); + } + var resources = controller.applications().requireApplication(tenantAndApplicationId).instances().keySet().stream() + .map(InstanceName::value) + .map(instance -> "instance/" + instance) + .sorted() + .collect(Collectors.toList()); + return new ResourceResponse(request.getUri(), resources); + } + + private HttpResponse instance(Path path, HttpRequest request) { + var instanceId = instanceFrom(path); + if (isRecursive(request)) { + var slime = new Slime(); + var root = slime.setObject(); + toSlime(List.of(controller.applications().requireApplication(TenantAndApplicationId.from(instanceId))), + instanceId, null, root); + return new SlimeJsonResponse(slime); + } + var resources = controller.applications().requireInstance(instanceId).deployments().keySet().stream() + .map(zone -> "environment/" + zone.environment().value() + + "/region/" + zone.region().value()) + .sorted() + .collect(Collectors.toList()); + return new ResourceResponse(request.getUri(), resources); + } + private HttpResponse setZoneStatus(Path path, boolean in) { var zone = zoneFrom(path); if (controller.zoneRegistry().zones().directlyRouted().ids().contains(zone)) { @@ -88,21 +177,25 @@ public class RoutingApiHandler extends AuditLoggingRequestHandler { (in ? "IN" : "OUT")); } - private HttpResponse zoneStatus(Path path) { + private HttpResponse zone(Path path) { var zone = zoneFrom(path); var slime = new Slime(); var root = slime.setObject(); + toSlime(zone, root); + return new SlimeJsonResponse(slime); + } + + private void toSlime(ZoneId zone, Cursor zoneObject) { if (controller.zoneRegistry().zones().directlyRouted().ids().contains(zone)) { var zonePolicy = controller.routingController().policies().get(zone); - zoneStatusToSlime(root, zonePolicy.zone(), zonePolicy.globalRouting(), RoutingMethod.exclusive); + zoneStatusToSlime(zoneObject, zonePolicy.zone(), zonePolicy.globalRouting(), RoutingMethod.exclusive); } else { // Rotation status per zone only exposes in/out status, no agent or time of change. var in = controller.serviceRegistry().configServer().getGlobalRotationStatus(zone); var globalRouting = new GlobalRouting(in ? GlobalRouting.Status.in : GlobalRouting.Status.out, GlobalRouting.Agent.operator, Instant.EPOCH); - zoneStatusToSlime(root, zone, globalRouting, RoutingMethod.shared); + zoneStatusToSlime(zoneObject, zone, globalRouting, RoutingMethod.shared); } - return new SlimeJsonResponse(slime); } private HttpResponse setDeploymentStatus(Path path, boolean in) { @@ -124,41 +217,56 @@ public class RoutingApiHandler extends AuditLoggingRequestHandler { return new MessageResponse("Set global routing status for " + deployment + " to " + (in ? "IN" : "OUT")); } - private HttpResponse deploymentStatus(Path path) { - var deployment = deploymentFrom(path); - var instance = controller.applications().requireInstance(deployment.applicationId()); + private HttpResponse deployment(Path path) { var slime = new Slime(); - var deploymentsObject = slime.setObject().setArray("deployments"); + var root = slime.setObject(); + var deploymentId = deploymentFrom(path); + var application = controller.applications().requireApplication(TenantAndApplicationId.from(deploymentId.applicationId())); + toSlime(List.of(application), deploymentId.applicationId(), deploymentId.zoneId(), root); + return new SlimeJsonResponse(slime); + } - // Include status from rotation - if (rotationCanRouteTo(deployment.zoneId(), instance)) { - var rotationStatus = controller.routingController().globalRotationStatus(deployment); - // Status is equal across all global endpoints, as the status is per deployment, not per endpoint. - var endpointStatus = rotationStatus.values().stream().findFirst(); - if (endpointStatus.isPresent()) { - var changedAt = Instant.ofEpochSecond(endpointStatus.get().getEpoch()); - GlobalRouting.Agent agent; - try { - agent = GlobalRouting.Agent.valueOf(endpointStatus.get().getAgent()); - } catch (IllegalArgumentException e) { - agent = GlobalRouting.Agent.unknown; + private void toSlime(List<Application> applications, ApplicationId instanceId, ZoneId zoneId, Cursor root) { + var deploymentsArray = root.setArray("deployments"); + for (var application : applications) { + var instances = instanceId == null + ? application.instances().values() + : List.of(application.instances().get(instanceId.instance())); + for (var instance : instances) { + var zones = zoneId == null ? instance.deployments().keySet() : List.of(zoneId); + for (var zone : zones) { + var deploymentId = new DeploymentId(instance.id(), zone); + // Include status from rotation + if (rotationCanRouteTo(zone, instance)) { + var rotationStatus = controller.routingController().globalRotationStatus(deploymentId); + // Status is equal across all global endpoints, as the status is per deployment, not per endpoint. + var endpointStatus = rotationStatus.values().stream().findFirst(); + if (endpointStatus.isPresent()) { + var changedAt = Instant.ofEpochSecond(endpointStatus.get().getEpoch()); + GlobalRouting.Agent agent; + try { + agent = GlobalRouting.Agent.valueOf(endpointStatus.get().getAgent()); + } catch (IllegalArgumentException e) { + agent = GlobalRouting.Agent.unknown; + } + var status = endpointStatus.get().getStatus() == EndpointStatus.Status.in + ? GlobalRouting.Status.in + : GlobalRouting.Status.out; + deploymentStatusToSlime(deploymentsArray.addObject(), deploymentId, + new GlobalRouting(status, agent, changedAt), + RoutingMethod.shared); + } + } + + // Include status from routing policies + var routingPolicies = controller.routingController().policies().get(deploymentId); + for (var policy : routingPolicies.values()) { + deploymentStatusToSlime(deploymentsArray.addObject(), policy); + } } - var status = endpointStatus.get().getStatus() == EndpointStatus.Status.in - ? GlobalRouting.Status.in - : GlobalRouting.Status.out; - deploymentStatusToSlime(deploymentsObject.addObject(), deployment, - new GlobalRouting(status, agent, changedAt), - RoutingMethod.shared); } } - // Include status from routing policies - var routingPolicies = controller.routingController().policies().get(deployment); - for (var policy : routingPolicies.values()) { - deploymentStatusToSlime(deploymentsObject.addObject(), policy); - } - - return new SlimeJsonResponse(slime); } /** Returns whether instance has an assigned rotation and a deployment in given zone */ @@ -190,9 +298,24 @@ public class RoutingApiHandler extends AuditLoggingRequestHandler { policy.status().globalRouting(), RoutingMethod.exclusive); } + private TenantName tenantFrom(Path path) { + return TenantName.from(path.get("tenant")); + } + + private ApplicationName applicationFrom(Path path) { + return ApplicationName.from(path.get("application")); + } + + private TenantAndApplicationId tenantAndApplicationIdFrom(Path path) { + return TenantAndApplicationId.from(tenantFrom(path), applicationFrom(path)); + } + + private ApplicationId instanceFrom(Path path) { + return ApplicationId.from(tenantFrom(path), applicationFrom(path), InstanceName.from(path.get("instance"))); + } + private DeploymentId deploymentFrom(Path path) { - return new DeploymentId(ApplicationId.from(path.get("tenant"), path.get("application"), path.get("instance")), - zoneFrom(path)); + return new DeploymentId(instanceFrom(path), zoneFrom(path)); } private ZoneId zoneFrom(Path path) { @@ -203,6 +326,10 @@ public class RoutingApiHandler extends AuditLoggingRequestHandler { return zone; } + private static boolean isRecursive(HttpRequest request) { + return "true".equals(request.getProperty("recursive")); + } + private static String asString(GlobalRouting.Status status) { switch (status) { case in: return "in"; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/user/UserApiHandler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/user/UserApiHandler.java index d9ffe8a251c..847a6c96a53 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/user/UserApiHandler.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/user/UserApiHandler.java @@ -134,10 +134,11 @@ public class UserApiHandler extends LoggingRequestHandler { // List of operator roles, currently only one available, but possible to extend List<Role> operatorRoles = roles.stream() - .filter(role -> role.definition().equals(RoleDefinition.hostedOperator)) + .filter(role -> role.definition().equals(RoleDefinition.hostedOperator) || + role.definition().equals(RoleDefinition.hostedSupporter)) + .sorted(Comparator.comparing(Role::definition)) .collect(Collectors.toList()); - Slime slime = new Slime(); Cursor root = slime.setObject(); diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/ApplicationPackageBuilder.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/ApplicationPackageBuilder.java index 9b0706d184f..ed7ae12168f 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/ApplicationPackageBuilder.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/ApplicationPackageBuilder.java @@ -201,11 +201,11 @@ public class ApplicationPackageBuilder { xml.append("'/>\n"); } xml.append(notifications); - xml.append(blockChange); if (explicitSystemTest) xml.append(" <test />\n"); if (explicitStagingTest) xml.append(" <staging />\n"); + xml.append(blockChange); xml.append(" <"); xml.append(environment.value()); if (globalServiceId != null) { diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/testdata/application-without-project-id.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/testdata/application-without-project-id.json deleted file mode 100644 index 31949cce282..00000000000 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/testdata/application-without-project-id.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "id": "tenant1:app1:default", - "deploymentSpecField": "<deployment version='1.0'>\n <test />\n <staging />\n <prod>\n <region active=\"true\">us-central-1</region>\n <region active=\"true\">us-west-1</region>\n </prod>\n</deployment>\n", - "validationOverrides": "<deployment version='1.0'/>", - "deployments": [], - "deploymentJobs": { - "jobStatus": [ - { - "jobType": "system-test", - "lastTriggered": { - "version": "6.42.1", - "upgrade": false, - "at": 1506330088050 - } - } - ] - }, - "outstandingChangeField": false -} diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java index fb7d54759f8..1bdc3a22c2e 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java @@ -3,6 +3,7 @@ package com.yahoo.vespa.hosted.controller.restapi.application; import ai.vespa.hosted.api.MultiPartStreamer; import ai.vespa.hosted.api.Signatures; +import com.yahoo.application.container.handler.Headers; import com.yahoo.application.container.handler.Request; import com.yahoo.component.Version; import com.yahoo.config.application.api.ValidationId; @@ -75,6 +76,7 @@ import org.junit.Test; import java.io.File; import java.math.BigDecimal; import java.net.URI; +import java.net.URLEncoder; import java.nio.charset.StandardCharsets; import java.time.Duration; import java.time.Instant; @@ -90,12 +92,16 @@ import java.util.Optional; import java.util.Set; import java.util.TreeSet; import java.util.function.Supplier; +import java.util.stream.Collectors; import static com.yahoo.application.container.handler.Request.Method.DELETE; import static com.yahoo.application.container.handler.Request.Method.GET; import static com.yahoo.application.container.handler.Request.Method.PATCH; import static com.yahoo.application.container.handler.Request.Method.POST; import static com.yahoo.application.container.handler.Request.Method.PUT; +import static java.net.URLEncoder.encode; +import static java.nio.charset.StandardCharsets.UTF_8; +import static java.util.stream.Collectors.joining; import static org.junit.Assert.assertArrayEquals; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; @@ -235,6 +241,18 @@ public class ApplicationApiTest extends ControllerContainerTest { // GET tenant applications (instances of "application1" only) tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/instance/", GET).userIdentity(USER_ID), new File("instance-list.json")); + // GET at a tenant, with "&recursive=true&production=true", recurses over no instances yet, as they are not in deployment spec. + tester.assertResponse(request("/application/v4/tenant/tenant1/", GET) + .userIdentity(USER_ID) + .properties(Map.of("recursive", "true", + "production", "true")), + new File("tenant-without-applications.json")); + // GET at an application, with "&recursive=true&production=true", recurses over no instances yet, as they are not in deployment spec. + tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1", GET) + .userIdentity(USER_ID) + .properties(Map.of("recursive", "true", + "production", "true")), + new File("application-without-instances.json")); addUserToHostedOperatorRole(HostedAthenzIdentities.from(HOSTED_VESPA_OPERATOR)); @@ -249,6 +267,15 @@ public class ApplicationApiTest extends ControllerContainerTest { "{\"message\":\"Deployment started in run 1 of dev-us-east-1 for tenant1.application1.instance1. This may take about 15 minutes the first time.\",\"run\":1}"); app1.runJob(JobType.devUsEast1); + // GET dev application package + tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/instance/instance1/job/dev-us-east-1/package", GET) + .userIdentity(USER_ID), + (response) -> { + assertEquals("attachment; filename=\"tenant1.application1.instance1.dev.us-east-1.zip\"", response.getHeaders().getFirst("Content-Disposition")); + assertArrayEquals(applicationPackageInstance1.zippedContent(), response.getBody()); + }, + 200); + // POST an application package is not generally allowed under user instance tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/instance/otheruser/deploy/dev-us-east-1", POST) .userIdentity(OTHER_USER_ID) @@ -482,7 +509,7 @@ public class ApplicationApiTest extends ControllerContainerTest { "{\"message\":\"Triggered pin to 6.1 for tenant1.application1.instance1\"}"); assertTrue("Action is logged to audit log", tester.controller().auditLogger().readLog().entries().stream() - .anyMatch(entry -> entry.resource().equals("/application/v4/tenant/tenant1/application/application1/instance/instance1/deploying/pin"))); + .anyMatch(entry -> entry.resource().equals("/application/v4/tenant/tenant1/application/application1/instance/instance1/deploying/pin?"))); tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/instance/instance1/deploying", GET) .userIdentity(USER_ID), "{\"platform\":\"6.1\",\"pinned\":true}"); tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/instance/instance1/deploying/pin", GET) @@ -558,7 +585,8 @@ public class ApplicationApiTest extends ControllerContainerTest { "{\"message\":\"Requested restart of tenant1.application1.instance1 in dev.us-central-1\"}"); // POST a 'restart application' command with a host filter (other filters not supported yet) - tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/environment/prod/region/us-central-1/instance/instance1/restart?hostname=node-1-tenant-host-prod.us-central-1", POST) + tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/environment/prod/region/us-central-1/instance/instance1/restart", POST) + .properties(Map.of("hostname", "node-1-tenant-host-prod.us-central-1")) .screwdriverIdentity(SCREWDRIVER_ID), "{\"message\":\"Requested restart of tenant1.application1.instance1 in prod.us-central-1\"}", 200); @@ -662,7 +690,9 @@ public class ApplicationApiTest extends ControllerContainerTest { 200); // GET application package for previous build - tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/package?build=1", GET).userIdentity(HOSTED_VESPA_OPERATOR), + tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/package", GET) + .properties(Map.of("build", "1")) + .userIdentity(HOSTED_VESPA_OPERATOR), (response) -> { assertEquals("attachment; filename=\"tenant1.application1-build1.zip\"", response.getHeaders().getFirst("Content-Disposition")); assertArrayEquals(applicationPackageInstance1.zippedContent(), response.getBody()); @@ -862,19 +892,22 @@ public class ApplicationApiTest extends ControllerContainerTest { 400); // GET global rotation status for us-west-1 in default endpoint - tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/instance/instance1/environment/prod/region/us-west-1/global-rotation?endpointId=default", GET) + tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/instance/instance1/environment/prod/region/us-west-1/global-rotation", GET) + .properties(Map.of("endpointId", "default")) .userIdentity(USER_ID), "{\"bcpStatus\":{\"rotationStatus\":\"IN\"}}", 200); // GET global rotation status for us-west-1 in eu endpoint - tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/instance/instance1/environment/prod/region/us-west-1/global-rotation?endpointId=eu", GET) + tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/instance/instance1/environment/prod/region/us-west-1/global-rotation", GET) + .properties(Map.of("endpointId", "eu")) .userIdentity(USER_ID), "{\"bcpStatus\":{\"rotationStatus\":\"UNKNOWN\"}}", 200); // GET global rotation status for eu-west-1 in eu endpoint - tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/instance/instance1/environment/prod/region/eu-west-1/global-rotation?endpointId=eu", GET) + tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/instance/instance1/environment/prod/region/eu-west-1/global-rotation", GET) + .properties(Map.of("endpointId", "eu")) .userIdentity(USER_ID), "{\"bcpStatus\":{\"rotationStatus\":\"IN\"}}", 200); @@ -1089,12 +1122,16 @@ public class ApplicationApiTest extends ControllerContainerTest { 404); // GET non-existent application package of specific build - tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/package?build=42", GET).userIdentity(HOSTED_VESPA_OPERATOR), + tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/package", GET) + .properties(Map.of("build", "42")) + .userIdentity(HOSTED_VESPA_OPERATOR), "{\"error-code\":\"NOT_FOUND\",\"message\":\"No application package found for 'tenant1.application1' with build number 42\"}", 404); // GET non-existent application package of invalid build - tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/package?build=foobar", GET).userIdentity(HOSTED_VESPA_OPERATOR), + tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/package", GET) + .properties(Map.of("build", "foobar")) + .userIdentity(HOSTED_VESPA_OPERATOR), "{\"error-code\":\"BAD_REQUEST\",\"message\":\"Invalid build number: For input string: \\\"foobar\\\"\"}", 400); @@ -1628,7 +1665,7 @@ public class ApplicationApiTest extends ControllerContainerTest { private OktaAccessToken oktaAccessToken; private String contentType = "application/json"; private Map<String, List<String>> headers = new HashMap<>(); - private String recursive; + private Map<String, String> properties = new HashMap<>(); private RequestBuilder(String path, Request.Method method) { this.path = path; @@ -1636,7 +1673,7 @@ public class ApplicationApiTest extends ControllerContainerTest { } private RequestBuilder data(byte[] data) { this.data = data; return this; } - private RequestBuilder data(String data) { return data(data.getBytes(StandardCharsets.UTF_8)); } + private RequestBuilder data(String data) { return data(data.getBytes(UTF_8)); } private RequestBuilder data(MultiPartStreamer streamer) { return Exceptions.uncheck(() -> data(streamer.data().readAllBytes()).contentType(streamer.contentType())); } @@ -1646,7 +1683,8 @@ public class ApplicationApiTest extends ControllerContainerTest { private RequestBuilder oktaIdentityToken(OktaIdentityToken oktaIdentityToken) { this.oktaIdentityToken = oktaIdentityToken; return this; } private RequestBuilder oktaAccessToken(OktaAccessToken oktaAccessToken) { this.oktaAccessToken = oktaAccessToken; return this; } private RequestBuilder contentType(String contentType) { this.contentType = contentType; return this; } - private RequestBuilder recursive(String recursive) { this.recursive = recursive; return this; } + private RequestBuilder recursive(String recursive) {return properties(Map.of("recursive", recursive)); } + private RequestBuilder properties(Map<String, String> properties) { this.properties.putAll(properties); return this; } private RequestBuilder header(String name, String value) { this.headers.putIfAbsent(name, new ArrayList<>()); this.headers.get(name).add(value); @@ -1656,11 +1694,13 @@ public class ApplicationApiTest extends ControllerContainerTest { @Override public Request get() { Request request = new Request("http://localhost:8080" + path + - // user and domain parameters are translated to a Principal by MockAuthorizer as we do not run HTTP filters - (recursive == null ? "" : "?recursive=" + recursive), + properties.entrySet().stream() + .map(entry -> encode(entry.getKey(), UTF_8) + "=" + encode(entry.getValue(), UTF_8)) + .collect(joining("&", "?", "")), data, method); request.getHeaders().addAll(headers); request.getHeaders().put("Content-Type", contentType); + // user and domain parameters are translated to a Principal by MockAuthorizer as we do not run HTTP filters if (identity != null) { addIdentityToRequest(request, identity); } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/JobControllerApiHandlerHelperTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/JobControllerApiHandlerHelperTest.java index 186534dd288..1c96f46dd31 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/JobControllerApiHandlerHelperTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/JobControllerApiHandlerHelperTest.java @@ -5,15 +5,12 @@ import com.yahoo.component.Version; import com.yahoo.config.provision.zone.ZoneId; import com.yahoo.container.jdisc.HttpResponse; import com.yahoo.vespa.hosted.controller.api.application.v4.model.DeployOptions; -import com.yahoo.vespa.hosted.controller.api.identifiers.DeploymentId; import com.yahoo.vespa.hosted.controller.api.integration.configserver.ConfigServerException; import com.yahoo.vespa.hosted.controller.api.integration.deployment.ApplicationVersion; import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType; import com.yahoo.vespa.hosted.controller.application.ApplicationPackage; import com.yahoo.vespa.hosted.controller.deployment.ApplicationPackageBuilder; import com.yahoo.vespa.hosted.controller.deployment.DeploymentTester; -import com.yahoo.vespa.hosted.controller.deployment.InternalStepRunner; -import com.yahoo.vespa.hosted.controller.deployment.RunStatus; import org.json.JSONException; import org.json.JSONObject; import org.junit.Test; @@ -26,6 +23,7 @@ import java.nio.file.Path; import java.nio.file.Paths; import java.time.Duration; import java.time.Instant; +import java.util.Date; import java.util.Optional; import static com.yahoo.vespa.hosted.controller.api.integration.configserver.ConfigServerException.ErrorCode.INVALID_APPLICATION_PACKAGE; @@ -37,13 +35,10 @@ import static com.yahoo.vespa.hosted.controller.api.integration.deployment.JobTy import static com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType.stagingTest; import static com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType.systemTest; import static com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType.testUsCentral1; -import static com.yahoo.vespa.hosted.controller.api.integration.deployment.TesterCloud.Status.FAILURE; import static com.yahoo.vespa.hosted.controller.deployment.DeploymentContext.applicationPackage; import static com.yahoo.vespa.hosted.controller.deployment.RunStatus.deploymentFailed; -import static com.yahoo.vespa.hosted.controller.deployment.RunStatus.error; import static com.yahoo.vespa.hosted.controller.deployment.RunStatus.installationFailed; import static com.yahoo.vespa.hosted.controller.deployment.RunStatus.running; -import static com.yahoo.vespa.hosted.controller.deployment.RunStatus.testFailure; import static org.junit.Assert.assertEquals; /** @@ -56,6 +51,9 @@ public class JobControllerApiHandlerHelperTest { public void testResponses() { ApplicationPackage applicationPackage = new ApplicationPackageBuilder() .stagingTest() + .blockChange(true, true, "mon,tue", "7-13", "UTC") + .blockChange(false, true, "sun", "0-23", "CET") + .blockChange(true, false, "fri-sat", "8", "America/Los_Angeles") .region("us-central-1") .test("us-central-1") .parallel("us-west-1", "us-east-3") @@ -139,7 +137,6 @@ public class JobControllerApiHandlerHelperTest { userApp.runJob(devAwsUsEast2a, applicationPackage); assertResponse(JobControllerApiHandlerHelper.runResponse(tester.jobs().runs(userApp.instanceId(), devAwsUsEast2a), URI.create("https://some.url:43/root")), "dev-aws-us-east-2a-runs.json"); assertResponse(JobControllerApiHandlerHelper.jobTypeResponse(tester.controller(), userApp.instanceId(), URI.create("https://some.url:43/root/")), "overview-user-instance.json"); - assertResponse(JobControllerApiHandlerHelper.overviewResponse(tester.controller(), app.application().id(), URI.create("https://some.url:43/root/")), "deployment-overview-2.json"); } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/application-without-instances.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/application-without-instances.json new file mode 100644 index 00000000000..411f9074582 --- /dev/null +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/application-without-instances.json @@ -0,0 +1,13 @@ +{ + "tenant": "tenant1", + "application": "application1", + "deployments": "http://localhost:8080/application/v4/tenant/tenant1/application/application1/job/", + "compileVersion": "6.1.0", + "instances": [], + "pemDeployKeys": [], + "metrics": { + "queryServiceQuality": 0.0, + "writeServiceQuality": 0.0 + }, + "activity": {} +} diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/deployment-job-accepted.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/deployment-job-accepted.json deleted file mode 100644 index d37e9120837..00000000000 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/deployment-job-accepted.json +++ /dev/null @@ -1,4 +0,0 @@ -{ - "message": "Deployment started in run 1 of dev-us-east-1 for tenant1.application1.instance1. This may take about 15 minutes the first time.", - "run": 1 -}
\ No newline at end of file diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/deployment-overview-2.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/deployment-overview-2.json index a8be282deaf..de6a71c14de 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/deployment-overview-2.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/deployment-overview-2.json @@ -6,40 +6,144 @@ "type": "instance", "dependencies": [], "declared": true, - "instance": "default" + "instance": "default", + "readyAt": 0, + "deploying": { + "application": { + "build": 3, + "compileVersion": "6.1.0", + "sourceUrl": "repository1/tree/commit1", + "commit": "commit1" + } + }, + "latestVersions": { + "platform": { + "platform": "7.1.0", + "at": 0, + "upgrade": true, + "blockers": [ + { + "days": [ + "Mon", + "Tue" + ], + "hours": [ + 7, + 8, + 9, + 10, + 11, + 12, + 13 + ], + "zone": "UTC" + }, + { + "days": [ + "Sun" + ], + "hours": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23 + ], + "zone": "CET" + } + ] + }, + "application": { + "application": { + "build": 3, + "compileVersion": "6.1.0", + "sourceUrl": "repository1/tree/commit1", + "commit": "commit1" + }, + "at": 1000, + "upgrade": true, + "blockers": [ + { + "days": [ + "Mon", + "Tue" + ], + "hours": [ + 7, + 8, + 9, + 10, + 11, + 12, + 13 + ], + "zone": "UTC" + }, + { + "days": [ + "Fri", + "Sat" + ], + "hours": [ + 8 + ], + "zone": "America/Los_Angeles" + } + ] + } + } }, { "type": "test", "dependencies": [], "declared": false, "instance": "default", + "readyAt": 0, "jobName": "system-test", "url": "https://some.url:43/instance/default/job/system-test", "environment": "test", "region": "test.us-east-1", "toRun": [], - "readyAt": 0, "runs": [ { "id": 3, - "url": "https://some.url:43/instance/default/job/system-test/run/run 3 of system-test for tenant.application", + "url": "https://some.url:43/instance/default/job/system-test/run/3", "start": 3603000, "end": 3603000, "status": "success", "versions": { "targetPlatform": "6.1.0", "targetApplication": { - "id": 3, - "commit": "commit1", + "build": 3, "compileVersion": "6.1.0", - "sourceUrl": "repository1/tree/commit1" + "sourceUrl": "repository1/tree/commit1", + "commit": "commit1" }, "sourcePlatform": "6.1.0", "sourceApplication": { - "id": 2, - "commit": "commit1", + "build": 2, "compileVersion": "6.1.0", - "sourceUrl": "repository1/tree/commit1" + "sourceUrl": "repository1/tree/commit1", + "commit": "commit1" } }, "steps": [ @@ -87,24 +191,24 @@ }, { "id": 2, - "url": "https://some.url:43/instance/default/job/system-test/run/run 2 of system-test for tenant.application", + "url": "https://some.url:43/instance/default/job/system-test/run/2", "start": 1000, "end": 1000, "status": "success", "versions": { "targetPlatform": "6.1.0", "targetApplication": { - "id": 2, - "commit": "commit1", + "build": 2, "compileVersion": "6.1.0", - "sourceUrl": "repository1/tree/commit1" + "sourceUrl": "repository1/tree/commit1", + "commit": "commit1" }, "sourcePlatform": "6.1.0", "sourceApplication": { - "id": 1, - "commit": "commit1", + "build": 1, "compileVersion": "6.1.0", - "sourceUrl": "repository1/tree/commit1" + "sourceUrl": "repository1/tree/commit1", + "commit": "commit1" } }, "steps": [ @@ -152,17 +256,17 @@ }, { "id": 1, - "url": "https://some.url:43/instance/default/job/system-test/run/run 1 of system-test for tenant.application", + "url": "https://some.url:43/instance/default/job/system-test/run/1", "start": 0, "end": 0, "status": "success", "versions": { "targetPlatform": "6.1.0", "targetApplication": { - "id": 1, - "commit": "commit1", + "build": 1, "compileVersion": "6.1.0", - "sourceUrl": "repository1/tree/commit1" + "sourceUrl": "repository1/tree/commit1", + "commit": "commit1" } }, "steps": [ @@ -215,6 +319,9 @@ "dependencies": [], "declared": true, "instance": "default", + "readyAt": 4353000, + "delayedUntil": 4353000, + "coolingDownUntil": 4353000, "jobName": "staging-test", "url": "https://some.url:43/instance/default/job/staging-test", "environment": "staging", @@ -224,45 +331,42 @@ "versions": { "targetPlatform": "6.1.0", "targetApplication": { - "id": 3, - "commit": "commit1", + "build": 3, "compileVersion": "6.1.0", - "sourceUrl": "repository1/tree/commit1" + "sourceUrl": "repository1/tree/commit1", + "commit": "commit1" }, "sourcePlatform": "6.1.0", "sourceApplication": { - "id": 1, - "commit": "commit1", + "build": 1, "compileVersion": "6.1.0", - "sourceUrl": "repository1/tree/commit1" + "sourceUrl": "repository1/tree/commit1", + "commit": "commit1" } } } ], - "readyAt": 4353000, - "delayedUntil": 4353000, - "coolingDownUntil": 4353000, "runs": [ { "id": 5, - "url": "https://some.url:43/instance/default/job/staging-test/run/run 5 of staging-test for tenant.application", + "url": "https://some.url:43/instance/default/job/staging-test/run/5", "start": 3703000, "end": 3703000, "status": "installationFailed", "versions": { "targetPlatform": "6.1.0", "targetApplication": { - "id": 3, - "commit": "commit1", + "build": 3, "compileVersion": "6.1.0", - "sourceUrl": "repository1/tree/commit1" + "sourceUrl": "repository1/tree/commit1", + "commit": "commit1" }, "sourcePlatform": "6.1.0", "sourceApplication": { - "id": 1, - "commit": "commit1", + "build": 1, "compileVersion": "6.1.0", - "sourceUrl": "repository1/tree/commit1" + "sourceUrl": "repository1/tree/commit1", + "commit": "commit1" } }, "steps": [ @@ -326,24 +430,24 @@ }, { "id": 4, - "url": "https://some.url:43/instance/default/job/staging-test/run/run 4 of staging-test for tenant.application", + "url": "https://some.url:43/instance/default/job/staging-test/run/4", "start": 3603000, "end": 3603000, "status": "installationFailed", "versions": { "targetPlatform": "6.1.0", "targetApplication": { - "id": 3, - "commit": "commit1", + "build": 3, "compileVersion": "6.1.0", - "sourceUrl": "repository1/tree/commit1" + "sourceUrl": "repository1/tree/commit1", + "commit": "commit1" }, "sourcePlatform": "6.1.0", "sourceApplication": { - "id": 1, - "commit": "commit1", + "build": 1, "compileVersion": "6.1.0", - "sourceUrl": "repository1/tree/commit1" + "sourceUrl": "repository1/tree/commit1", + "commit": "commit1" } }, "steps": [ @@ -407,24 +511,24 @@ }, { "id": 3, - "url": "https://some.url:43/instance/default/job/staging-test/run/run 3 of staging-test for tenant.application", + "url": "https://some.url:43/instance/default/job/staging-test/run/3", "start": 3603000, "end": 3603000, "status": "success", "versions": { "targetPlatform": "6.1.0", "targetApplication": { - "id": 3, - "commit": "commit1", + "build": 3, "compileVersion": "6.1.0", - "sourceUrl": "repository1/tree/commit1" + "sourceUrl": "repository1/tree/commit1", + "commit": "commit1" }, "sourcePlatform": "6.1.0", "sourceApplication": { - "id": 2, - "commit": "commit1", + "build": 2, "compileVersion": "6.1.0", - "sourceUrl": "repository1/tree/commit1" + "sourceUrl": "repository1/tree/commit1", + "commit": "commit1" } }, "steps": [ @@ -488,24 +592,24 @@ }, { "id": 2, - "url": "https://some.url:43/instance/default/job/staging-test/run/run 2 of staging-test for tenant.application", + "url": "https://some.url:43/instance/default/job/staging-test/run/2", "start": 1000, "end": 1000, "status": "success", "versions": { "targetPlatform": "6.1.0", "targetApplication": { - "id": 2, - "commit": "commit1", + "build": 2, "compileVersion": "6.1.0", - "sourceUrl": "repository1/tree/commit1" + "sourceUrl": "repository1/tree/commit1", + "commit": "commit1" }, "sourcePlatform": "6.1.0", "sourceApplication": { - "id": 1, - "commit": "commit1", + "build": 1, "compileVersion": "6.1.0", - "sourceUrl": "repository1/tree/commit1" + "sourceUrl": "repository1/tree/commit1", + "commit": "commit1" } }, "steps": [ @@ -569,17 +673,17 @@ }, { "id": 1, - "url": "https://some.url:43/instance/default/job/staging-test/run/run 1 of staging-test for tenant.application", + "url": "https://some.url:43/instance/default/job/staging-test/run/1", "start": 0, "end": 0, "status": "success", "versions": { "targetPlatform": "6.1.0", "targetApplication": { - "id": 1, - "commit": "commit1", + "build": 1, "compileVersion": "6.1.0", - "sourceUrl": "repository1/tree/commit1" + "sourceUrl": "repository1/tree/commit1", + "commit": "commit1" } }, "steps": [ @@ -651,39 +755,39 @@ ], "declared": true, "instance": "default", + "readyAt": 3603000, "jobName": "production-us-central-1", "url": "https://some.url:43/instance/default/job/production-us-central-1", "environment": "prod", "region": "prod.us-central-1", "currentPlatform": "6.1.0", "currentApplication": { - "id": 3, - "commit": "commit1", + "build": 3, "compileVersion": "6.1.0", - "sourceUrl": "repository1/tree/commit1" + "sourceUrl": "repository1/tree/commit1", + "commit": "commit1" }, "toRun": [], - "readyAt": 3603000, "runs": [ { "id": 3, - "url": "https://some.url:43/instance/default/job/production-us-central-1/run/run 3 of production-us-central-1 for tenant.application", + "url": "https://some.url:43/instance/default/job/production-us-central-1/run/3", "start": 3603000, "status": "running", "versions": { "targetPlatform": "6.1.0", "targetApplication": { - "id": 3, - "commit": "commit1", + "build": 3, "compileVersion": "6.1.0", - "sourceUrl": "repository1/tree/commit1" + "sourceUrl": "repository1/tree/commit1", + "commit": "commit1" }, "sourcePlatform": "6.1.0", "sourceApplication": { - "id": 2, - "commit": "commit1", + "build": 2, "compileVersion": "6.1.0", - "sourceUrl": "repository1/tree/commit1" + "sourceUrl": "repository1/tree/commit1", + "commit": "commit1" } }, "steps": [ @@ -703,24 +807,24 @@ }, { "id": 2, - "url": "https://some.url:43/instance/default/job/production-us-central-1/run/run 2 of production-us-central-1 for tenant.application", + "url": "https://some.url:43/instance/default/job/production-us-central-1/run/2", "start": 1000, "end": 1000, "status": "success", "versions": { "targetPlatform": "6.1.0", "targetApplication": { - "id": 2, - "commit": "commit1", + "build": 2, "compileVersion": "6.1.0", - "sourceUrl": "repository1/tree/commit1" + "sourceUrl": "repository1/tree/commit1", + "commit": "commit1" }, "sourcePlatform": "6.1.0", "sourceApplication": { - "id": 1, - "commit": "commit1", + "build": 1, "compileVersion": "6.1.0", - "sourceUrl": "repository1/tree/commit1" + "sourceUrl": "repository1/tree/commit1", + "commit": "commit1" } }, "steps": [ @@ -740,17 +844,17 @@ }, { "id": 1, - "url": "https://some.url:43/instance/default/job/production-us-central-1/run/run 1 of production-us-central-1 for tenant.application", + "url": "https://some.url:43/instance/default/job/production-us-central-1/run/1", "start": 0, "end": 0, "status": "success", "versions": { "targetPlatform": "6.1.0", "targetApplication": { - "id": 1, - "commit": "commit1", + "build": 1, "compileVersion": "6.1.0", - "sourceUrl": "repository1/tree/commit1" + "sourceUrl": "repository1/tree/commit1", + "commit": "commit1" } }, "steps": [ @@ -786,17 +890,17 @@ "versions": { "targetPlatform": "6.1.0", "targetApplication": { - "id": 3, - "commit": "commit1", + "build": 3, "compileVersion": "6.1.0", - "sourceUrl": "repository1/tree/commit1" + "sourceUrl": "repository1/tree/commit1", + "commit": "commit1" }, "sourcePlatform": "6.1.0", "sourceApplication": { - "id": 3, - "commit": "commit1", + "build": 3, "compileVersion": "6.1.0", - "sourceUrl": "repository1/tree/commit1" + "sourceUrl": "repository1/tree/commit1", + "commit": "commit1" } } } @@ -804,24 +908,24 @@ "runs": [ { "id": 2, - "url": "https://some.url:43/instance/default/job/test-us-central-1/run/run 2 of test-us-central-1 for tenant.application", + "url": "https://some.url:43/instance/default/job/test-us-central-1/run/2", "start": 1000, "end": 1000, "status": "success", "versions": { "targetPlatform": "6.1.0", "targetApplication": { - "id": 2, - "commit": "commit1", + "build": 2, "compileVersion": "6.1.0", - "sourceUrl": "repository1/tree/commit1" + "sourceUrl": "repository1/tree/commit1", + "commit": "commit1" }, "sourcePlatform": "6.1.0", "sourceApplication": { - "id": 2, - "commit": "commit1", + "build": 2, "compileVersion": "6.1.0", - "sourceUrl": "repository1/tree/commit1" + "sourceUrl": "repository1/tree/commit1", + "commit": "commit1" } }, "steps": [ @@ -853,24 +957,24 @@ }, { "id": 1, - "url": "https://some.url:43/instance/default/job/test-us-central-1/run/run 1 of test-us-central-1 for tenant.application", + "url": "https://some.url:43/instance/default/job/test-us-central-1/run/1", "start": 0, "end": 0, "status": "success", "versions": { "targetPlatform": "6.1.0", "targetApplication": { - "id": 1, - "commit": "commit1", + "build": 1, "compileVersion": "6.1.0", - "sourceUrl": "repository1/tree/commit1" + "sourceUrl": "repository1/tree/commit1", + "commit": "commit1" }, "sourcePlatform": "6.1.0", "sourceApplication": { - "id": 1, - "commit": "commit1", + "build": 1, "compileVersion": "6.1.0", - "sourceUrl": "repository1/tree/commit1" + "sourceUrl": "repository1/tree/commit1", + "commit": "commit1" } }, "steps": [ @@ -915,27 +1019,27 @@ "region": "prod.us-west-1", "currentPlatform": "6.1.0", "currentApplication": { - "id": 2, - "commit": "commit1", + "build": 2, "compileVersion": "6.1.0", - "sourceUrl": "repository1/tree/commit1" + "sourceUrl": "repository1/tree/commit1", + "commit": "commit1" }, "toRun": [ { "versions": { "targetPlatform": "6.1.0", "targetApplication": { - "id": 3, - "commit": "commit1", + "build": 3, "compileVersion": "6.1.0", - "sourceUrl": "repository1/tree/commit1" + "sourceUrl": "repository1/tree/commit1", + "commit": "commit1" }, "sourcePlatform": "6.1.0", "sourceApplication": { - "id": 2, - "commit": "commit1", + "build": 2, "compileVersion": "6.1.0", - "sourceUrl": "repository1/tree/commit1" + "sourceUrl": "repository1/tree/commit1", + "commit": "commit1" } } } @@ -943,24 +1047,24 @@ "runs": [ { "id": 2, - "url": "https://some.url:43/instance/default/job/production-us-west-1/run/run 2 of production-us-west-1 for tenant.application", + "url": "https://some.url:43/instance/default/job/production-us-west-1/run/2", "start": 1000, "end": 3602000, "status": "installationFailed", "versions": { "targetPlatform": "6.1.0", "targetApplication": { - "id": 2, - "commit": "commit1", + "build": 2, "compileVersion": "6.1.0", - "sourceUrl": "repository1/tree/commit1" + "sourceUrl": "repository1/tree/commit1", + "commit": "commit1" }, "sourcePlatform": "6.1.0", "sourceApplication": { - "id": 1, - "commit": "commit1", + "build": 1, "compileVersion": "6.1.0", - "sourceUrl": "repository1/tree/commit1" + "sourceUrl": "repository1/tree/commit1", + "commit": "commit1" } }, "steps": [ @@ -980,17 +1084,17 @@ }, { "id": 1, - "url": "https://some.url:43/instance/default/job/production-us-west-1/run/run 1 of production-us-west-1 for tenant.application", + "url": "https://some.url:43/instance/default/job/production-us-west-1/run/1", "start": 0, "end": 0, "status": "success", "versions": { "targetPlatform": "6.1.0", "targetApplication": { - "id": 1, - "commit": "commit1", + "build": 1, "compileVersion": "6.1.0", - "sourceUrl": "repository1/tree/commit1" + "sourceUrl": "repository1/tree/commit1", + "commit": "commit1" } }, "steps": [ @@ -1023,27 +1127,27 @@ "region": "prod.us-east-3", "currentPlatform": "6.1.0", "currentApplication": { - "id": 1, - "commit": "commit1", + "build": 1, "compileVersion": "6.1.0", - "sourceUrl": "repository1/tree/commit1" + "sourceUrl": "repository1/tree/commit1", + "commit": "commit1" }, "toRun": [ { "versions": { "targetPlatform": "6.1.0", "targetApplication": { - "id": 3, - "commit": "commit1", + "build": 3, "compileVersion": "6.1.0", - "sourceUrl": "repository1/tree/commit1" + "sourceUrl": "repository1/tree/commit1", + "commit": "commit1" }, "sourcePlatform": "6.1.0", "sourceApplication": { - "id": 1, - "commit": "commit1", + "build": 1, "compileVersion": "6.1.0", - "sourceUrl": "repository1/tree/commit1" + "sourceUrl": "repository1/tree/commit1", + "commit": "commit1" } } } @@ -1051,24 +1155,24 @@ "runs": [ { "id": 2, - "url": "https://some.url:43/instance/default/job/production-us-east-3/run/run 2 of production-us-east-3 for tenant.application", + "url": "https://some.url:43/instance/default/job/production-us-east-3/run/2", "start": 1000, "end": 1000, "status": "deploymentFailed", "versions": { "targetPlatform": "6.1.0", "targetApplication": { - "id": 2, - "commit": "commit1", + "build": 2, "compileVersion": "6.1.0", - "sourceUrl": "repository1/tree/commit1" + "sourceUrl": "repository1/tree/commit1", + "commit": "commit1" }, "sourcePlatform": "6.1.0", "sourceApplication": { - "id": 1, - "commit": "commit1", + "build": 1, "compileVersion": "6.1.0", - "sourceUrl": "repository1/tree/commit1" + "sourceUrl": "repository1/tree/commit1", + "commit": "commit1" } }, "steps": [ @@ -1088,17 +1192,17 @@ }, { "id": 1, - "url": "https://some.url:43/instance/default/job/production-us-east-3/run/run 1 of production-us-east-3 for tenant.application", + "url": "https://some.url:43/instance/default/job/production-us-east-3/run/1", "start": 0, "end": 0, "status": "success", "versions": { "targetPlatform": "6.1.0", "targetApplication": { - "id": 1, - "commit": "commit1", + "build": 1, "compileVersion": "6.1.0", - "sourceUrl": "repository1/tree/commit1" + "sourceUrl": "repository1/tree/commit1", + "commit": "commit1" } }, "steps": [ diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/deployment-overview.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/deployment-overview.json index 027cca5dad2..282c18046d3 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/deployment-overview.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/deployment-overview.json @@ -6,32 +6,60 @@ "type": "instance", "dependencies": [], "declared": true, - "instance": "instance1" + "instance": "instance1", + "readyAt": 0, + "deploying": { + "application": { + "build": 4, + "compileVersion": "6.1.0", + "sourceUrl": "repository1/tree/commit1", + "commit": "commit1" + } + }, + "latestVersions": { + "platform": { + "platform": "6.1.0", + "at": "(ignore)", + "upgrade": false, + "blockers": [] + }, + "application": { + "application": { + "build": 4, + "compileVersion": "6.1.0", + "sourceUrl": "repository1/tree/commit1", + "commit": "commit1" + }, + "at": 1000, + "upgrade": false, + "blockers": [] + } + } }, { "type": "test", "dependencies": [], "declared": false, "instance": "instance1", + "readyAt": 0, "jobName": "system-test", "url": "http://localhost:8080/application/v4/tenant/tenant1/application/application1/instance/instance1/job/system-test", "environment": "test", "region": "test.us-east-1", "toRun": [], - "readyAt": 0, "runs": [ { "id": 2, - "url": "http://localhost:8080/application/v4/tenant/tenant1/application/application1/instance/instance1/job/system-test/run/run 2 of system-test for tenant1.application1.instance1", + "url": "http://localhost:8080/application/v4/tenant/tenant1/application/application1/instance/instance1/job/system-test/run/2", "start": "(ignore)", "status": "running", "versions": { "targetPlatform": "6.1.0", "targetApplication": { - "id": 4, - "commit": "commit1", + "build": 4, "compileVersion": "6.1.0", - "sourceUrl": "repository1/tree/commit1" + "sourceUrl": "repository1/tree/commit1", + "commit": "commit1" } }, "steps": [ @@ -79,17 +107,17 @@ }, { "id": 1, - "url": "http://localhost:8080/application/v4/tenant/tenant1/application/application1/instance/instance1/job/system-test/run/run 1 of system-test for tenant1.application1.instance1", + "url": "http://localhost:8080/application/v4/tenant/tenant1/application/application1/instance/instance1/job/system-test/run/1", "start": "(ignore)", "end": "(ignore)", "status": "success", "versions": { "targetPlatform": "6.1.0", "targetApplication": { - "id": 1, - "commit": "commit1", + "build": 1, "compileVersion": "6.1.0", - "sourceUrl": "repository1/tree/commit1" + "sourceUrl": "repository1/tree/commit1", + "commit": "commit1" } }, "steps": [ @@ -142,25 +170,25 @@ "dependencies": [], "declared": false, "instance": "instance1", + "readyAt": 0, "jobName": "staging-test", "url": "http://localhost:8080/application/v4/tenant/tenant1/application/application1/instance/instance1/job/staging-test", "environment": "staging", "region": "staging.us-east-3", "toRun": [], - "readyAt": 0, "runs": [ { "id": 2, - "url": "http://localhost:8080/application/v4/tenant/tenant1/application/application1/instance/instance1/job/staging-test/run/run 2 of staging-test for tenant1.application1.instance1", + "url": "http://localhost:8080/application/v4/tenant/tenant1/application/application1/instance/instance1/job/staging-test/run/2", "start": "(ignore)", "status": "running", "versions": { "targetPlatform": "6.1.0", "targetApplication": { - "id": 4, - "commit": "commit1", + "build": 4, "compileVersion": "6.1.0", - "sourceUrl": "repository1/tree/commit1" + "sourceUrl": "repository1/tree/commit1", + "commit": "commit1" } }, "steps": [ @@ -224,17 +252,17 @@ }, { "id": 1, - "url": "http://localhost:8080/application/v4/tenant/tenant1/application/application1/instance/instance1/job/staging-test/run/run 1 of staging-test for tenant1.application1.instance1", + "url": "http://localhost:8080/application/v4/tenant/tenant1/application/application1/instance/instance1/job/staging-test/run/1", "start": "(ignore)", "end": "(ignore)", "status": "success", "versions": { "targetPlatform": "6.1.0", "targetApplication": { - "id": 1, - "commit": "commit1", + "build": 1, "compileVersion": "6.1.0", - "sourceUrl": "repository1/tree/commit1" + "sourceUrl": "repository1/tree/commit1", + "commit": "commit1" } }, "steps": [ @@ -314,10 +342,10 @@ "versions": { "targetPlatform": "6.1.0", "targetApplication": { - "id": 4, - "commit": "commit1", + "build": 4, "compileVersion": "6.1.0", - "sourceUrl": "repository1/tree/commit1" + "sourceUrl": "repository1/tree/commit1", + "commit": "commit1" } } } @@ -325,17 +353,17 @@ "runs": [ { "id": 1, - "url": "http://localhost:8080/application/v4/tenant/tenant1/application/application1/instance/instance1/job/production-us-central-1/run/run 1 of production-us-central-1 for tenant1.application1.instance1", + "url": "http://localhost:8080/application/v4/tenant/tenant1/application/application1/instance/instance1/job/production-us-central-1/run/1", "start": "(ignore)", "end": "(ignore)", "status": "success", "versions": { "targetPlatform": "6.1.0", "targetApplication": { - "id": 1, - "commit": "commit1", + "build": 1, "compileVersion": "6.1.0", - "sourceUrl": "repository1/tree/commit1" + "sourceUrl": "repository1/tree/commit1", + "commit": "commit1" } }, "steps": [ @@ -371,10 +399,10 @@ "versions": { "targetPlatform": "6.1.0", "targetApplication": { - "id": 4, - "commit": "commit1", + "build": 4, "compileVersion": "6.1.0", - "sourceUrl": "repository1/tree/commit1" + "sourceUrl": "repository1/tree/commit1", + "commit": "commit1" } } } @@ -382,15 +410,15 @@ "runs": [ { "id": 1, - "url": "http://localhost:8080/application/v4/tenant/tenant1/application/application1/instance/instance1/job/production-us-west-1/run/run 1 of production-us-west-1 for tenant1.application1.instance1", + "url": "http://localhost:8080/application/v4/tenant/tenant1/application/application1/instance/instance1/job/production-us-west-1/run/1", "start": "(ignore)", "status": "aborted", "versions": { "targetPlatform": "6.1.0", "targetApplication": { - "id": 1, - "commit": "commit1", - "sourceUrl": "repository1/tree/commit1" + "build": 1, + "sourceUrl": "repository1/tree/commit1", + "commit": "commit1" } }, "steps": [ @@ -426,10 +454,10 @@ "versions": { "targetPlatform": "6.1.0", "targetApplication": { - "id": 4, - "commit": "commit1", + "build": 4, "compileVersion": "6.1.0", - "sourceUrl": "repository1/tree/commit1" + "sourceUrl": "repository1/tree/commit1", + "commit": "commit1" } } } @@ -437,16 +465,16 @@ "runs": [ { "id": 1, - "url": "http://localhost:8080/application/v4/tenant/tenant1/application/application1/instance/instance1/job/production-us-east-3/run/run 1 of production-us-east-3 for tenant1.application1.instance1", + "url": "http://localhost:8080/application/v4/tenant/tenant1/application/application1/instance/instance1/job/production-us-east-3/run/1", "start": "(ignore)", "status": "aborted", "versions": { "targetPlatform": "6.1.0", "targetApplication": { - "id": 1, - "commit": "commit1", + "build": 1, "compileVersion": "6.1.0", - "sourceUrl": "repository1/tree/commit1" + "sourceUrl": "repository1/tree/commit1", + "commit": "commit1" } }, "steps": [ @@ -473,19 +501,39 @@ 5 ], "declared": true, - "instance": "instance2" + "instance": "instance2", + "deploying": {}, + "latestVersions": { + "platform": { + "platform": "6.1.0", + "at": "(ignore)", + "upgrade": false, + "blockers": [] + }, + "application": { + "application": { + "build": 4, + "compileVersion": "6.1.0", + "sourceUrl": "repository1/tree/commit1", + "commit": "commit1" + }, + "at": 1000, + "upgrade": false, + "blockers": [] + } + } }, { "type": "test", "dependencies": [], "declared": false, "instance": "instance2", + "readyAt": 0, "jobName": "system-test", "url": "http://localhost:8080/application/v4/tenant/tenant1/application/application1/instance/instance2/job/system-test", "environment": "test", "region": "test.us-east-1", "toRun": [], - "readyAt": 0, "runs": [] }, { @@ -493,12 +541,12 @@ "dependencies": [], "declared": false, "instance": "instance2", + "readyAt": 0, "jobName": "staging-test", "url": "http://localhost:8080/application/v4/tenant/tenant1/application/application1/instance/instance2/job/staging-test", "environment": "staging", "region": "staging.us-east-3", "toRun": [], - "readyAt": 0, "runs": [] }, { @@ -517,10 +565,10 @@ "versions": { "targetPlatform": "6.1.0", "targetApplication": { - "id": 4, - "commit": "commit1", + "build": 4, "compileVersion": "6.1.0", - "sourceUrl": "repository1/tree/commit1" + "sourceUrl": "repository1/tree/commit1", + "commit": "commit1" } } } @@ -543,10 +591,10 @@ "versions": { "targetPlatform": "6.1.0", "targetApplication": { - "id": 4, - "commit": "commit1", + "build": 4, "compileVersion": "6.1.0", - "sourceUrl": "repository1/tree/commit1" + "sourceUrl": "repository1/tree/commit1", + "commit": "commit1" } } } @@ -569,10 +617,10 @@ "versions": { "targetPlatform": "6.1.0", "targetApplication": { - "id": 4, - "commit": "commit1", + "build": 4, "compileVersion": "6.1.0", - "sourceUrl": "repository1/tree/commit1" + "sourceUrl": "repository1/tree/commit1", + "commit": "commit1" } } } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/deployment-with-routing-policy.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/deployment-with-routing-policy.json index 10201af0272..eb8bf523474 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/deployment-with-routing-policy.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/deployment-with-routing-policy.json @@ -9,19 +9,22 @@ "cluster": "default", "tls": true, "url": "https://instance1.application1.tenant1.us-west-1.vespa.oath.cloud/", - "scope": "zone" + "scope": "zone", + "routingMethod": "exclusive" }, { "cluster": "default", "tls": true, "url": "https://c0.instance1.application1.tenant1.global.vespa.oath.cloud/", - "scope": "global" + "scope": "global", + "routingMethod": "exclusive" }, { "cluster": "default", "tls": true, "url": "https://instance1--application1--tenant1.us-west-1.prod.vespa:43", - "scope": "zone" + "scope": "zone", + "routingMethod": "shared" } ], "serviceUrls": [ diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/deployment.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/deployment.json index d96b291234d..ef8899c0860 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/deployment.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/deployment.json @@ -9,13 +9,15 @@ "cluster": "default", "tls": true, "url": "https://instance1--application1--tenant1.us-central-1.prod.vespa:43", - "scope": "zone" + "scope": "zone", + "routingMethod": "shared" }, { "cluster": "foo", "tls": true, "url": "https://instance1--application1--tenant1.global.vespa.oath.cloud:4443/", - "scope": "global" + "scope": "global", + "routingMethod": "shared" } ], "serviceUrls": [ diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/dev-overview.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/dev-overview.json index bfaa62a602d..b37d0d41ae4 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/dev-overview.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/dev-overview.json @@ -29,5 +29,37 @@ ], "url": "https://some.url:43/root/dev-us-east-1" } - } + }, + "deployment": [ + { + "jobName": "dev-us-east-1", + "runs": [ + { + "id": 1, + "url": "https://some.url:43/root/run/1", + "start": 0, + "end": 0, + "status": "success", + "versions": { + "targetPlatform": "6.1.0", + "targetApplication": {} + }, + "steps": [ + { + "name": "deployReal", + "status": "succeeded" + }, + { + "name": "installReal", + "status": "succeeded" + }, + { + "name": "copyVespaLogs", + "status": "succeeded" + } + ] + } + ] + } + ] } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/dev-us-east-1.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/dev-us-east-1.json index 3ff8533fdb3..cc930c94051 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/dev-us-east-1.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/dev-us-east-1.json @@ -9,7 +9,8 @@ "cluster": "default", "tls": true, "url": "https://instance1--application1--tenant1.us-east-1.dev.vespa:43", - "scope": "zone" + "scope": "zone", + "routingMethod": "shared" } ], "serviceUrls": [ diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/instance-without-change-multiple-deployments.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/instance-without-change-multiple-deployments.json deleted file mode 100644 index 6338306897c..00000000000 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/instance-without-change-multiple-deployments.json +++ /dev/null @@ -1,315 +0,0 @@ -{ - "tenant": "tenant1", - "application": "application1", - "instance": "instance1", - "deployments": "http://localhost:8080/application/v4/tenant/tenant1/application/application1/instance/instance1/job/", - "source": { - "gitRepository": "repository1", - "gitBranch": "master", - "gitCommit": "commit1" - }, - "sourceUrl": "repository1/tree/commit1", - "commit": "commit1", - "projectId": 1000, - "deploymentJobs": [ - { - "type": "system-test", - "success": true, - "lastTriggered": { - "id": 1, - "version": "(ignore)", - "revision": { - "buildNumber": "(ignore)", - "hash": "(ignore)", - "source": { - "gitRepository": "repository1", - "gitBranch": "master", - "gitCommit": "commit1" - }, - "sourceUrl": "repository1/tree/commit1", - "commit": "commit1" - }, - "reason": "unknown reason", - "at": "(ignore)" - }, - "lastCompleted": { - "id": 2, - "version": "(ignore)", - "revision": { - "buildNumber": "(ignore)", - "hash": "(ignore)", - "source": { - "gitRepository": "repository1", - "gitBranch": "master", - "gitCommit": "commit1" - }, - "sourceUrl": "repository1/tree/commit1", - "commit": "commit1" - }, - "reason": "unknown reason", - "at": "(ignore)" - }, - "lastSuccess": { - "id": 2, - "version": "(ignore)", - "revision": { - "buildNumber": "(ignore)", - "hash": "(ignore)", - "source": { - "gitRepository": "repository1", - "gitBranch": "master", - "gitCommit": "commit1" - }, - "sourceUrl": "repository1/tree/commit1", - "commit": "commit1" - }, - "reason": "unknown reason", - "at": "(ignore)" - } - }, - { - "type": "staging-test", - "success": true, - "lastTriggered": { - "id": 1, - "version": "(ignore)", - "revision": { - "buildNumber": "(ignore)", - "hash": "(ignore)", - "source": { - "gitRepository": "repository1", - "gitBranch": "master", - "gitCommit": "commit1" - }, - "sourceUrl": "repository1/tree/commit1", - "commit": "commit1" - }, - "reason": "unknown reason", - "at": "(ignore)" - }, - "lastCompleted": { - "id": 3, - "version": "(ignore)", - "revision": { - "buildNumber": "(ignore)", - "hash": "(ignore)", - "source": { - "gitRepository": "repository1", - "gitBranch": "master", - "gitCommit": "commit1" - }, - "sourceUrl": "repository1/tree/commit1", - "commit": "commit1" - }, - "reason": "unknown reason", - "at": "(ignore)" - }, - "lastSuccess": { - "id": 3, - "version": "(ignore)", - "revision": { - "buildNumber": "(ignore)", - "hash": "(ignore)", - "source": { - "gitRepository": "repository1", - "gitBranch": "master", - "gitCommit": "commit1" - }, - "sourceUrl": "repository1/tree/commit1", - "commit": "commit1" - }, - "reason": "unknown reason", - "at": "(ignore)" - } - }, - { - "type": "production-us-west-1", - "success": true, - "lastTriggered": { - "id": 1, - "version": "(ignore)", - "revision": { - "buildNumber": "(ignore)", - "hash": "(ignore)", - "source": { - "gitRepository": "repository1", - "gitBranch": "master", - "gitCommit": "commit1" - }, - "sourceUrl": "repository1/tree/commit1", - "commit": "commit1" - }, - "reason": "unknown reason", - "at": "(ignore)" - }, - "lastCompleted": { - "id": 1, - "version": "(ignore)", - "revision": { - "buildNumber": "(ignore)", - "hash": "(ignore)", - "source": { - "gitRepository": "repository1", - "gitBranch": "master", - "gitCommit": "commit1" - }, - "sourceUrl": "repository1/tree/commit1", - "commit": "commit1" - }, - "reason": "unknown reason", - "at": "(ignore)" - }, - "lastSuccess": { - "id": 1, - "version": "(ignore)", - "revision": { - "buildNumber": "(ignore)", - "hash": "(ignore)", - "source": { - "gitRepository": "repository1", - "gitBranch": "master", - "gitCommit": "commit1" - }, - "sourceUrl": "repository1/tree/commit1", - "commit": "commit1" - }, - "reason": "unknown reason", - "at": "(ignore)" - } - }, - { - "type": "production-us-east-3", - "success": true, - "lastTriggered": { - "id": 1, - "version": "(ignore)", - "revision": { - "buildNumber": "(ignore)", - "hash": "(ignore)", - "source": { - "gitRepository": "repository1", - "gitBranch": "master", - "gitCommit": "commit1" - }, - "sourceUrl": "repository1/tree/commit1", - "commit": "commit1" - }, - "reason": "unknown reason", - "at": "(ignore)" - }, - "lastCompleted": { - "id": 2, - "version": "(ignore)", - "revision": { - "buildNumber": "(ignore)", - "hash": "(ignore)", - "source": { - "gitRepository": "repository1", - "gitBranch": "master", - "gitCommit": "commit1" - }, - "sourceUrl": "repository1/tree/commit1", - "commit": "commit1" - }, - "reason": "unknown reason", - "at": "(ignore)" - }, - "lastSuccess": { - "id": 2, - "version": "(ignore)", - "revision": { - "buildNumber": "(ignore)", - "hash": "(ignore)", - "source": { - "gitRepository": "repository1", - "gitBranch": "master", - "gitCommit": "commit1" - }, - "sourceUrl": "repository1/tree/commit1", - "commit": "commit1" - }, - "reason": "unknown reason", - "at": "(ignore)" - } - } - ], - "changeBlockers": [], - "compileVersion": "(ignore)", - "globalRotations": [ - "https://instance1--application1--tenant1.global.vespa.oath.cloud:4443/" - ], - "rotationId": "rotation-id-1", - "instances": [ - { - "bcpStatus": { - "rotationStatus": "IN" - }, - "endpointStatus": [ - { - "endpointId": "default", - "rotationId": "rotation-id-1", - "clusterId": "foo", - "status": "IN", - "lastUpdated": "(ignore)" - } - ], - "applicationVersion": { - "hash": "1.0.1-commit1", - "build": 1, - "source": { - "gitRepository": "repository1", - "gitBranch": "master", - "gitCommit": "commit1" - }, - "sourceUrl": "repository1/tree/commit1", - "commit": "commit1" - }, - "status": "complete", - "environment": "prod", - "region": "us-west-1", - "instance": "instance1", - "url": "http://localhost:8080/application/v4/tenant/tenant1/application/application1/instance/instance1/environment/prod/region/us-west-1" - }, - { - "bcpStatus": { - "rotationStatus": "UNKNOWN" - }, - "endpointStatus": [ - { - "endpointId": "default", - "rotationId": "rotation-id-1", - "clusterId": "foo", - "status": "UNKNOWN", - "lastUpdated": "(ignore)" - } - ], - "applicationVersion": { - "hash": "1.0.1-commit1", - "build": 1, - "source": { - "gitRepository": "repository1", - "gitBranch": "master", - "gitCommit": "commit1" - }, - "sourceUrl": "repository1/tree/commit1", - "commit": "commit1" - }, - "status": "complete", - "environment": "prod", - "region": "us-east-3", - "instance": "instance1", - "url": "http://localhost:8080/application/v4/tenant/tenant1/application/application1/instance/instance1/environment/prod/region/us-east-3" - } - ], - "pemDeployKeys": [], - "metrics": { - "queryServiceQuality": 0.5, - "writeServiceQuality": 0.7 - }, - "activity": { - "lastQueried": 1527848130000, - "lastWritten": 1527848130000, - "lastQueriesPerSecond": 1.0, - "lastWritesPerSecond": 2.0 - } -} diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/jobs-direct-deployment.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/jobs-direct-deployment.json index 85a3245c308..e97c76668d3 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/jobs-direct-deployment.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/jobs-direct-deployment.json @@ -3,5 +3,6 @@ "deployments": [], "lastVersions": {}, "deploying": {}, - "jobs": {} + "jobs": {}, + "deployment": [] } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/jobs.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/jobs.json index 3ab7078ae6a..8cd102432d0 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/jobs.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/jobs.json @@ -328,5 +328,37 @@ ], "url": "http://localhost:8080/application/v4/tenant/tenant1/application/application1/instance/instance1/job/dev-us-east-1" } - } + }, + "deployment": [ + { + "jobName": "dev-us-east-1", + "runs": [ + { + "id": 1, + "url": "http://localhost:8080/application/v4/tenant/tenant1/application/application1/instance/instance1/job/run/1", + "start": "(ignore)", + "end": "(ignore)", + "status": "success", + "versions": { + "targetPlatform": "6.1.0", + "targetApplication": {} + }, + "steps": [ + { + "name": "deployReal", + "status": "succeeded" + }, + { + "name": "installReal", + "status": "succeeded" + }, + { + "name": "copyVespaLogs", + "status": "succeeded" + } + ] + } + ] + } + ] } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/overview-user-instance.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/overview-user-instance.json index 95afb12dcbf..aaa9127bdfd 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/overview-user-instance.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/overview-user-instance.json @@ -50,5 +50,37 @@ ], "url": "https://some.url:43/root/dev-aws-us-east-2a" } - } + }, + "deployment": [ + { + "jobName": "dev-aws-us-east-2a", + "runs": [ + { + "id": 1, + "url": "https://some.url:43/root//run/1", + "start": 3703000, + "end": 3703000, + "status": "success", + "versions": { + "targetPlatform": "7.1.0", + "targetApplication": {} + }, + "steps": [ + { + "name": "deployReal", + "status": "succeeded" + }, + { + "name": "installReal", + "status": "succeeded" + }, + { + "name": "copyVespaLogs", + "status": "succeeded" + } + ] + } + ] + } + ] } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/overview.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/overview.json index 9510b06fb42..1e1a4549006 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/overview.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/overview.json @@ -945,5 +945,6 @@ "url": "https://some.url:43/root/production-us-east-3" } }, - "devJobs": {} + "devJobs": {}, + "deployment": [] } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/prod-us-central-1.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/prod-us-central-1.json index bff8326fdeb..436c2767b3e 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/prod-us-central-1.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/prod-us-central-1.json @@ -12,13 +12,15 @@ "cluster": "default", "tls": true, "url": "https://instance1--application1--tenant1.us-central-1.prod.vespa:43", - "scope": "zone" + "scope": "zone", + "routingMethod": "shared" }, { "cluster": "foo", "tls": true, "url": "https://instance1--application1--tenant1.global.vespa.oath.cloud:4443/", - "scope": "global" + "scope": "global", + "routingMethod": "shared" } ], "serviceUrls": [ diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/filter/AthenzRoleFilterTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/filter/AthenzRoleFilterTest.java index 82b97a5b144..bef27f7a2f5 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/filter/AthenzRoleFilterTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/filter/AthenzRoleFilterTest.java @@ -73,13 +73,13 @@ public class AthenzRoleFilterTest { public void testTranslations() { // Hosted operators are always members of the hostedOperator role. - assertEquals(Set.of(Role.hostedOperator(), Role.systemFlagsDeployer(), Role.systemFlagsDryrunner()), + assertEquals(Set.of(Role.hostedOperator(), Role.systemFlagsDeployer(), Role.systemFlagsDryrunner(), Role.hostedSupporter()), filter.roles(HOSTED_OPERATOR, NO_CONTEXT_PATH)); - assertEquals(Set.of(Role.hostedOperator(), Role.systemFlagsDeployer(), Role.systemFlagsDryrunner()), + assertEquals(Set.of(Role.hostedOperator(), Role.systemFlagsDeployer(), Role.systemFlagsDryrunner(), Role.hostedSupporter()), filter.roles(HOSTED_OPERATOR, TENANT_CONTEXT_PATH)); - assertEquals(Set.of(Role.hostedOperator(), Role.systemFlagsDeployer(), Role.systemFlagsDryrunner()), + assertEquals(Set.of(Role.hostedOperator(), Role.systemFlagsDeployer(), Role.systemFlagsDryrunner(), Role.hostedSupporter()), filter.roles(HOSTED_OPERATOR, APPLICATION_CONTEXT_PATH)); // Tenant admins are members of the athenzTenantAdmin role within their tenant subtree. diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/filter/ControllerAuthorizationFilterTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/filter/ControllerAuthorizationFilterTest.java index a5520b42459..c95691fc120 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/filter/ControllerAuthorizationFilterTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/filter/ControllerAuthorizationFilterTest.java @@ -43,6 +43,16 @@ public class ControllerAuthorizationFilterTest { } @Test + public void supporter() { + ControllerTester tester = new ControllerTester(); + SecurityContext securityContext = new SecurityContext(() -> "operator", Set.of(Role.hostedSupporter())); + ControllerAuthorizationFilter filter = createFilter(tester); + + assertIsForbidden(invokeFilter(filter, createRequest(Method.POST, "/zone/v2/path", securityContext))); + assertIsAllowed(invokeFilter(filter, createRequest(Method.GET, "/zone/v1/path", securityContext))); + } + + @Test public void unprivileged() { ControllerTester tester = new ControllerTester(); SecurityContext securityContext = new SecurityContext(() -> "user", Set.of(Role.everyone())); diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/os/responses/versions-initial.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/os/responses/versions-initial.json deleted file mode 100644 index dbaa6623fae..00000000000 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/os/responses/versions-initial.json +++ /dev/null @@ -1,153 +0,0 @@ -{ - "versions": [ - { - "version": "0.0.0", - "targetVersion": false, - "cloud": "cloud1", - "nodes": [ - { - "hostname": "node-2-configserver-host-prod.us-east-3", - "environment": "prod", - "region": "us-east-3" - }, - { - "hostname": "node-1-configserver-host-prod.us-east-3", - "environment": "prod", - "region": "us-east-3" - }, - { - "hostname": "node-3-configserver-host-prod.us-east-3", - "environment": "prod", - "region": "us-east-3" - }, - { - "hostname": "node-1-configserver-host-prod.us-west-1", - "environment": "prod", - "region": "us-west-1" - }, - { - "hostname": "node-2-configserver-host-prod.us-west-1", - "environment": "prod", - "region": "us-west-1" - }, - { - "hostname": "node-3-configserver-host-prod.us-west-1", - "environment": "prod", - "region": "us-west-1" - }, - { - "hostname": "node-2-proxy-host-prod.us-east-3", - "environment": "prod", - "region": "us-east-3" - }, - { - "hostname": "node-3-proxy-host-prod.us-east-3", - "environment": "prod", - "region": "us-east-3" - }, - { - "hostname": "node-1-proxy-host-prod.us-east-3", - "environment": "prod", - "region": "us-east-3" - }, - { - "hostname": "node-2-proxy-host-prod.us-west-1", - "environment": "prod", - "region": "us-west-1" - }, - { - "hostname": "node-1-proxy-host-prod.us-west-1", - "environment": "prod", - "region": "us-west-1" - }, - { - "hostname": "node-3-proxy-host-prod.us-west-1", - "environment": "prod", - "region": "us-west-1" - }, - { - "hostname": "node-3-tenant-host-prod.us-east-3", - "environment": "prod", - "region": "us-east-3" - }, - { - "hostname": "node-2-tenant-host-prod.us-east-3", - "environment": "prod", - "region": "us-east-3" - }, - { - "hostname": "node-1-tenant-host-prod.us-east-3", - "environment": "prod", - "region": "us-east-3" - }, - { - "hostname": "node-3-tenant-host-prod.us-west-1", - "environment": "prod", - "region": "us-west-1" - }, - { - "hostname": "node-2-tenant-host-prod.us-west-1", - "environment": "prod", - "region": "us-west-1" - }, - { - "hostname": "node-1-tenant-host-prod.us-west-1", - "environment": "prod", - "region": "us-west-1" - } - ] - }, - { - "version": "0.0.0", - "targetVersion": false, - "cloud": "cloud2", - "nodes": [ - { - "hostname": "node-1-configserver-host-prod.eu-west-1", - "environment": "prod", - "region": "eu-west-1" - }, - { - "hostname": "node-2-configserver-host-prod.eu-west-1", - "environment": "prod", - "region": "eu-west-1" - }, - { - "hostname": "node-3-configserver-host-prod.eu-west-1", - "environment": "prod", - "region": "eu-west-1" - }, - { - "hostname": "node-1-proxy-host-prod.eu-west-1", - "environment": "prod", - "region": "eu-west-1" - }, - { - "hostname": "node-3-proxy-host-prod.eu-west-1", - "environment": "prod", - "region": "eu-west-1" - }, - { - "hostname": "node-2-proxy-host-prod.eu-west-1", - "environment": "prod", - "region": "eu-west-1" - }, - { - "hostname": "node-1-tenant-host-prod.eu-west-1", - "environment": "prod", - "region": "eu-west-1" - }, - { - "hostname": "node-3-tenant-host-prod.eu-west-1", - "environment": "prod", - "region": "eu-west-1" - }, - { - "hostname": "node-2-tenant-host-prod.eu-west-1", - "environment": "prod", - "region": "eu-west-1" - } - ] - } - ] -} diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/RoutingApiTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/RoutingApiTest.java index 635adc73d1d..d191b460697 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/RoutingApiTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/RoutingApiTest.java @@ -35,6 +35,88 @@ public class RoutingApiTest extends ControllerContainerTest { } @Test + public void discovery() { + // Deploy + var context = deploymentTester.newDeploymentContext("t1", "a1", "default"); + var westZone = ZoneId.from("prod", "us-west-1"); + var eastZone = ZoneId.from("prod", "us-east-3"); + var applicationPackage = new ApplicationPackageBuilder() + .region(westZone.region()) + .region(eastZone.region()) + .endpoint("default", "default", eastZone.region().value(), westZone.region().value()) + .build(); + context.submit(applicationPackage).deploy(); + + // GET root + tester.assertResponse(operatorRequest("http://localhost:8080/routing/v1/", "", + Request.Method.GET), + new File("discovery/root.json")); + + // GET tenant + tester.assertResponse(operatorRequest("http://localhost:8080/routing/v1/status/tenant/t1", "", + Request.Method.GET), + new File("discovery/tenant.json")); + + // GET application + tester.assertResponse(operatorRequest("http://localhost:8080/routing/v1/status/tenant/t1/application/a1/", + "", + Request.Method.GET), + new File("discovery/application.json")); + + // GET instance + tester.assertResponse(operatorRequest("http://localhost:8080/routing/v1/status/tenant/t1/application/a1/instance/default/", + "", + Request.Method.GET), + new File("discovery/instance.json")); + + // GET environment + tester.assertResponse(operatorRequest("http://localhost:8080/routing/v1/status/environment/", "", + Request.Method.GET), + new File("discovery/environment.json")); + } + + @Test + public void recursion() { + var context1 = deploymentTester.newDeploymentContext("t1", "a1", "default"); + var westZone = ZoneId.from("prod", "us-west-1"); + var eastZone = ZoneId.from("prod", "us-east-3"); + var package1 = new ApplicationPackageBuilder() + .region(westZone.region()) + .region(eastZone.region()) + .endpoint("default", "default", eastZone.region().value(), westZone.region().value()) + .build(); + context1.submit(package1).deploy(); + + var context2 = deploymentTester.newDeploymentContext("t1", "a2", "default"); + var package2 = new ApplicationPackageBuilder() + .region(westZone.region()) + .region(eastZone.region()) + .endpoint("default", "default", eastZone.region().value(), westZone.region().value()) + .build(); + context2.submit(package2).deploy(); + + // GET tenant recursively + tester.assertResponse(operatorRequest("http://localhost:8080/routing/v1/status/tenant/t1?recursive=true", "", + Request.Method.GET), + new File("recursion/tenant.json")); + + // GET application recursively + tester.assertResponse(operatorRequest("http://localhost:8080/routing/v1/status/tenant/t1/application/a1?recursive=true", "", + Request.Method.GET), + new File("recursion/application.json")); + + // GET instance recursively + tester.assertResponse(operatorRequest("http://localhost:8080/routing/v1/status/tenant/t1/application/a1/instance/default?recursive=true", "", + Request.Method.GET), + new File("recursion/application.json")); + + // GET environment recursively + tester.assertResponse(operatorRequest("http://localhost:8080/routing/v1/status/environment?recursive=true", "", + Request.Method.GET), + new File("recursion/environment.json")); + } + + @Test public void exclusive_routing() { var context = deploymentTester.newDeploymentContext(); // Zones support direct routing diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/discovery/application.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/discovery/application.json new file mode 100644 index 00000000000..deda734cbbf --- /dev/null +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/discovery/application.json @@ -0,0 +1,7 @@ +{ + "resources": [ + { + "url": "http://localhost:8080/routing/v1/status/tenant/t1/application/a1/instance/default/" + } + ] +} diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/discovery/environment.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/discovery/environment.json new file mode 100644 index 00000000000..1e06b279873 --- /dev/null +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/discovery/environment.json @@ -0,0 +1,43 @@ +{ + "resources": [ + { + "url": "http://localhost:8080/routing/v1/status/environment/dev/region/aws-us-east-2a/" + }, + { + "url": "http://localhost:8080/routing/v1/status/environment/dev/region/us-east-1/" + }, + { + "url": "http://localhost:8080/routing/v1/status/environment/perf/region/us-east-3/" + }, + { + "url": "http://localhost:8080/routing/v1/status/environment/prod/region/ap-northeast-1/" + }, + { + "url": "http://localhost:8080/routing/v1/status/environment/prod/region/ap-northeast-2/" + }, + { + "url": "http://localhost:8080/routing/v1/status/environment/prod/region/ap-southeast-1/" + }, + { + "url": "http://localhost:8080/routing/v1/status/environment/prod/region/aws-us-east-1a/" + }, + { + "url": "http://localhost:8080/routing/v1/status/environment/prod/region/eu-west-1/" + }, + { + "url": "http://localhost:8080/routing/v1/status/environment/prod/region/us-central-1/" + }, + { + "url": "http://localhost:8080/routing/v1/status/environment/prod/region/us-east-3/" + }, + { + "url": "http://localhost:8080/routing/v1/status/environment/prod/region/us-west-1/" + }, + { + "url": "http://localhost:8080/routing/v1/status/environment/staging/region/us-east-3/" + }, + { + "url": "http://localhost:8080/routing/v1/status/environment/test/region/us-east-1/" + } + ] +} diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/discovery/instance.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/discovery/instance.json new file mode 100644 index 00000000000..1a3ad823e14 --- /dev/null +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/discovery/instance.json @@ -0,0 +1,10 @@ +{ + "resources": [ + { + "url": "http://localhost:8080/routing/v1/status/tenant/t1/application/a1/instance/default/environment/prod/region/us-east-3/" + }, + { + "url": "http://localhost:8080/routing/v1/status/tenant/t1/application/a1/instance/default/environment/prod/region/us-west-1/" + } + ] +} diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/discovery/root.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/discovery/root.json new file mode 100644 index 00000000000..9b5630335aa --- /dev/null +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/discovery/root.json @@ -0,0 +1,10 @@ +{ + "resources": [ + { + "url": "http://localhost:8080/routing/v1/status/tenant/" + }, + { + "url": "http://localhost:8080/routing/v1/status/environment/" + } + ] +} diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/discovery/tenant.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/discovery/tenant.json new file mode 100644 index 00000000000..acd05d35c8d --- /dev/null +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/discovery/tenant.json @@ -0,0 +1,7 @@ +{ + "resources": [ + { + "url": "http://localhost:8080/routing/v1/status/tenant/t1/application/a1/" + } + ] +} diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/recursion/application.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/recursion/application.json new file mode 100644 index 00000000000..9a5d919e9b4 --- /dev/null +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/recursion/application.json @@ -0,0 +1,22 @@ +{ + "deployments": [ + { + "routingMethod": "shared", + "instance": "t1:a1:default", + "environment": "prod", + "region": "us-west-1", + "status": "in", + "agent": "unknown", + "changedAt": "(ignore)" + }, + { + "routingMethod": "shared", + "instance": "t1:a1:default", + "environment": "prod", + "region": "us-east-3", + "status": "in", + "agent": "unknown", + "changedAt": "(ignore)" + } + ] +} diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/recursion/environment.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/recursion/environment.json new file mode 100644 index 00000000000..f0dd0b7310d --- /dev/null +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/recursion/environment.json @@ -0,0 +1,108 @@ +{ + "zones": [ + { + "routingMethod": "shared", + "environment": "test", + "region": "us-east-1", + "status": "in", + "agent": "operator", + "changedAt": 0 + }, + { + "routingMethod": "shared", + "environment": "staging", + "region": "us-east-3", + "status": "in", + "agent": "operator", + "changedAt": 0 + }, + { + "routingMethod": "shared", + "environment": "dev", + "region": "us-east-1", + "status": "in", + "agent": "operator", + "changedAt": 0 + }, + { + "routingMethod": "shared", + "environment": "dev", + "region": "aws-us-east-2a", + "status": "in", + "agent": "operator", + "changedAt": 0 + }, + { + "routingMethod": "shared", + "environment": "perf", + "region": "us-east-3", + "status": "in", + "agent": "operator", + "changedAt": 0 + }, + { + "routingMethod": "shared", + "environment": "prod", + "region": "aws-us-east-1a", + "status": "in", + "agent": "operator", + "changedAt": 0 + }, + { + "routingMethod": "shared", + "environment": "prod", + "region": "ap-northeast-1", + "status": "in", + "agent": "operator", + "changedAt": 0 + }, + { + "routingMethod": "shared", + "environment": "prod", + "region": "ap-northeast-2", + "status": "in", + "agent": "operator", + "changedAt": 0 + }, + { + "routingMethod": "shared", + "environment": "prod", + "region": "ap-southeast-1", + "status": "in", + "agent": "operator", + "changedAt": 0 + }, + { + "routingMethod": "shared", + "environment": "prod", + "region": "us-east-3", + "status": "in", + "agent": "operator", + "changedAt": 0 + }, + { + "routingMethod": "shared", + "environment": "prod", + "region": "us-west-1", + "status": "in", + "agent": "operator", + "changedAt": 0 + }, + { + "routingMethod": "shared", + "environment": "prod", + "region": "us-central-1", + "status": "in", + "agent": "operator", + "changedAt": 0 + }, + { + "routingMethod": "shared", + "environment": "prod", + "region": "eu-west-1", + "status": "in", + "agent": "operator", + "changedAt": 0 + } + ] +} diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/recursion/tenant.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/recursion/tenant.json new file mode 100644 index 00000000000..85db7411c40 --- /dev/null +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/recursion/tenant.json @@ -0,0 +1,40 @@ +{ + "deployments": [ + { + "routingMethod": "shared", + "instance": "t1:a1:default", + "environment": "prod", + "region": "us-west-1", + "status": "in", + "agent": "unknown", + "changedAt": "(ignore)" + }, + { + "routingMethod": "shared", + "instance": "t1:a1:default", + "environment": "prod", + "region": "us-east-3", + "status": "in", + "agent": "unknown", + "changedAt": "(ignore)" + }, + { + "routingMethod": "shared", + "instance": "t1:a2:default", + "environment": "prod", + "region": "us-west-1", + "status": "in", + "agent": "unknown", + "changedAt": "(ignore)" + }, + { + "routingMethod": "shared", + "instance": "t1:a2:default", + "environment": "prod", + "region": "us-east-3", + "status": "in", + "agent": "unknown", + "changedAt": "(ignore)" + } + ] +} diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/user/UserApiTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/user/UserApiTest.java index d1dd50cfb4c..d70a09414bb 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/user/UserApiTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/user/UserApiTest.java @@ -203,7 +203,7 @@ public class UserApiTest extends ControllerContainerCloudTest { public void userMetadataTest() { ContainerTester tester = new ContainerTester(container, responseFiles); ControllerTester controller = new ControllerTester(tester); - Set<Role> operator = Set.of(Role.hostedOperator()); + Set<Role> operator = Set.of(Role.hostedOperator(), Role.hostedSupporter()); User user = new User("dev@domail", "Joe Developer", "dev", null); tester.assertResponse(request("/api/user/v1/user") diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/user/responses/user-without-applications.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/user/responses/user-without-applications.json index 17489bb15d8..400fe8d4d9b 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/user/responses/user-without-applications.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/user/responses/user-without-applications.json @@ -7,6 +7,7 @@ }, "tenants": {}, "operator": [ - "hostedOperator" + "hostedOperator", + "hostedSupporter" ] } diff --git a/default_build_settings.cmake b/default_build_settings.cmake index d6781f14e75..e29e4c32017 100644 --- a/default_build_settings.cmake +++ b/default_build_settings.cmake @@ -77,7 +77,7 @@ endfunction() function(setup_vespa_default_build_settings_fedora_32) message("-- Setting up default build settings for fedora 32") set(DEFAULT_EXTRA_INCLUDE_DIRECTORY "${VESPA_DEPS}/include" "/usr/include/openblas" PARENT_SCOPE) - set(DEFAULT_VESPA_LLVM_VERSION "9" PARENT_SCOPE) + set(DEFAULT_VESPA_LLVM_VERSION "10" PARENT_SCOPE) endfunction() function(setup_vespa_default_build_settings_ubuntu_18_10) diff --git a/dist/vespa.spec b/dist/vespa.spec index 3a336496f4c..c54e4442167 100644 --- a/dist/vespa.spec +++ b/dist/vespa.spec @@ -81,7 +81,7 @@ BuildRequires: gtest-devel BuildRequires: gmock-devel %endif %if 0%{?fc32} -BuildRequires: llvm-devel >= 9.0.0 +BuildRequires: llvm-devel >= 10.0.0 BuildRequires: boost-devel >= 1.69 BuildRequires: gtest-devel BuildRequires: gmock-devel @@ -174,8 +174,8 @@ Requires: llvm-libs >= 9.0.0 %define _vespa_llvm_version 9 %endif %if 0%{?fc32} -Requires: llvm-libs >= 9.0.0 -%define _vespa_llvm_version 9 +Requires: llvm-libs >= 10.0.0 +%define _vespa_llvm_version 10 %endif %define _extra_link_directory %{_vespa_deps_prefix}/lib64 %define _extra_include_directory %{_vespa_deps_prefix}/include;/usr/include/openblas diff --git a/eval/src/vespa/eval/eval/llvm/compile_cache.h b/eval/src/vespa/eval/eval/llvm/compile_cache.h index 65cec9c0d48..aaadec772a5 100644 --- a/eval/src/vespa/eval/eval/llvm/compile_cache.h +++ b/eval/src/vespa/eval/eval/llvm/compile_cache.h @@ -5,6 +5,7 @@ #include "compiled_function.h" #include <vespa/vespalib/util/executor.h> #include <condition_variable> +#include <atomic> #include <mutex> namespace vespalib { diff --git a/eval/src/vespa/eval/eval/llvm/llvm_wrapper.cpp b/eval/src/vespa/eval/eval/llvm/llvm_wrapper.cpp index 1d5515d7f4a..cce9838d967 100644 --- a/eval/src/vespa/eval/eval/llvm/llvm_wrapper.cpp +++ b/eval/src/vespa/eval/eval/llvm/llvm_wrapper.cpp @@ -12,15 +12,18 @@ #include <llvm/Analysis/Passes.h> #include <llvm/IR/DataLayout.h> #include <llvm/Transforms/Scalar.h> -#if LLVM_VERSION_MAJOR == 9 && defined(__clang__) +#if LLVM_VERSION_MAJOR >= 9 && defined(__clang__) // Avoid reference to undefined symbol llvm::cfg::Update<llvm::BasicBlock*>::dump() const #define NDEBUG #endif #include <llvm/LinkAllPasses.h> -#if LLVM_VERSION_MAJOR == 9 && defined(__clang__) +#if LLVM_VERSION_MAJOR >= 9 && defined(__clang__) #undef NDEBUG #endif #include <llvm/Transforms/IPO/PassManagerBuilder.h> +#if LLVM_VERSION_MAJOR > 9 +#include <llvm/Support/ManagedStatic.h> +#endif #include <vespa/eval/eval/check_type.h> #include <vespa/vespalib/stllike/hash_set.h> #include <vespa/vespalib/util/approx.h> diff --git a/fbench/src/httpclient/httpclient.cpp b/fbench/src/httpclient/httpclient.cpp index 002d2770dcd..99134a6e297 100644 --- a/fbench/src/httpclient/httpclient.cpp +++ b/fbench/src/httpclient/httpclient.cpp @@ -1,5 +1,6 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "httpclient.h" +#include <vespa/vespalib/net/socket_spec.h> #include <cassert> #include <cstring> @@ -69,7 +70,8 @@ HTTPClient::connect_socket() if (!handle.valid()) { return false; } - _socket = vespalib::SyncCryptoSocket::create(*_engine, std::move(handle), false); + _socket = vespalib::SyncCryptoSocket::create_client(*_engine, std::move(handle), + vespalib::SocketSpec::from_host_port(_hostname, _port)); return bool(_socket); } diff --git a/flags/src/main/java/com/yahoo/vespa/flags/Flags.java b/flags/src/main/java/com/yahoo/vespa/flags/Flags.java index 9981fa5b035..a1ee71b9a58 100644 --- a/flags/src/main/java/com/yahoo/vespa/flags/Flags.java +++ b/flags/src/main/java/com/yahoo/vespa/flags/Flags.java @@ -188,12 +188,6 @@ public class Flags { "Whether to disable CM3.", "Takes effect on next host admin tick", HOSTNAME); - public static final UnboundBooleanFlag USE_4443_UPSTREAM = defineFeatureFlag( - "use-4443-upstream", false, - "Use port 4443 for nginx upstream", - "Takes effect when routing container asks for new config", - APPLICATION_ID); - public static final UnboundBooleanFlag GENERATE_L4_ROUTING_CONFIG = defineFeatureFlag( "generate-l4-routing-config", false, "Whether routing nodes should generate L4 routing config", @@ -224,6 +218,11 @@ public class Flags { "Takes effect on restart of Docker container", ZONE_ID, APPLICATION_ID); + public static final UnboundStringFlag DOCKER_IMAGE_OVERRIDE = defineStringFlag( + "docker-image-override", "", + "Override the Docker image to use for deployments. This must containing the image name only, without tag", + "Takes effect on next host-admin tick", APPLICATION_ID); + /** WARNING: public for testing: All flags should be defined in {@link Flags}. */ public static UnboundBooleanFlag defineFeatureFlag(String flagId, boolean defaultValue, String description, String modificationEffect, FetchVector.Dimension... dimensions) { diff --git a/fnet/src/tests/connect/connect_test.cpp b/fnet/src/tests/connect/connect_test.cpp index b70b3fa8b01..d94b6759077 100644 --- a/fnet/src/tests/connect/connect_test.cpp +++ b/fnet/src/tests/connect/connect_test.cpp @@ -65,7 +65,11 @@ struct BlockingCryptoEngine : public CryptoEngine { Gate handshake_work_enter; Gate handshake_work_exit; Gate handshake_socket_deleted; - CryptoSocket::UP create_crypto_socket(SocketHandle socket, bool) override { + CryptoSocket::UP create_client_crypto_socket(SocketHandle socket, const SocketSpec &) override { + return std::make_unique<BlockingCryptoSocket>(std::move(socket), + handshake_work_enter, handshake_work_exit, handshake_socket_deleted); + } + CryptoSocket::UP create_server_crypto_socket(SocketHandle socket) override { return std::make_unique<BlockingCryptoSocket>(std::move(socket), handshake_work_enter, handshake_work_exit, handshake_socket_deleted); } diff --git a/fnet/src/vespa/fnet/connection.cpp b/fnet/src/vespa/fnet/connection.cpp index 5f7adb32af0..c5afd627a5a 100644 --- a/fnet/src/vespa/fnet/connection.cpp +++ b/fnet/src/vespa/fnet/connection.cpp @@ -9,6 +9,7 @@ #include "config.h" #include "transport_thread.h" #include "transport.h" +#include <vespa/vespalib/net/socket_spec.h> #include <vespa/log/log.h> LOG_SETUP(".fnet"); @@ -472,7 +473,7 @@ FNET_Connection::FNET_Connection(FNET_TransportThread *owner, _streamer(streamer), _serverAdapter(serverAdapter), _adminChannel(nullptr), - _socket(owner->owner().create_crypto_socket(std::move(socket), true)), + _socket(owner->owner().create_server_crypto_socket(std::move(socket))), _resolve_handler(nullptr), _context(), _state(FNET_CONNECTING), @@ -579,7 +580,7 @@ FNET_Connection::handle_add_event() { if (_resolve_handler) { auto tweak = [this](vespalib::SocketHandle &handle) { return Owner()->tune(handle); }; - _socket = Owner()->owner().create_crypto_socket(_resolve_handler->address.connect(tweak), false); + _socket = Owner()->owner().create_client_crypto_socket(_resolve_handler->address.connect(tweak), vespalib::SocketSpec(GetSpec())); _ioc_socket_fd = _socket->get_fd(); _resolve_handler.reset(); } diff --git a/fnet/src/vespa/fnet/frt/supervisor.cpp b/fnet/src/vespa/fnet/frt/supervisor.cpp index f9cbfe2a662..a8cb7884534 100644 --- a/fnet/src/vespa/fnet/frt/supervisor.cpp +++ b/fnet/src/vespa/fnet/frt/supervisor.cpp @@ -409,7 +409,7 @@ FRT_Supervisor::SchedulerPtr::SchedulerPtr(FNET_TransportThread *transport_threa namespace fnet::frt { StandaloneFRT::StandaloneFRT() - : _threadPool(std::make_unique<FastOS_ThreadPool>(1024*60)), + : _threadPool(std::make_unique<FastOS_ThreadPool>(1024*128)), _transport(std::make_unique<FNET_Transport>()), _supervisor(std::make_unique<FRT_Supervisor>(_transport.get())) { @@ -417,7 +417,7 @@ StandaloneFRT::StandaloneFRT() } StandaloneFRT::StandaloneFRT(vespalib::CryptoEngine::SP crypto) - : _threadPool(std::make_unique<FastOS_ThreadPool>(1024*60)), + : _threadPool(std::make_unique<FastOS_ThreadPool>(1024*128)), _transport(std::make_unique<FNET_Transport>(std::move(crypto), 1)), _supervisor(std::make_unique<FRT_Supervisor>(_transport.get())) { diff --git a/fnet/src/vespa/fnet/transport.cpp b/fnet/src/vespa/fnet/transport.cpp index 28e645d9e03..d3b52969c8c 100644 --- a/fnet/src/vespa/fnet/transport.cpp +++ b/fnet/src/vespa/fnet/transport.cpp @@ -54,9 +54,15 @@ FNET_Transport::resolve_async(const vespalib::string &spec, } vespalib::CryptoSocket::UP -FNET_Transport::create_crypto_socket(vespalib::SocketHandle socket, bool is_server) +FNET_Transport::create_client_crypto_socket(vespalib::SocketHandle socket, const vespalib::SocketSpec &spec) { - return _crypto_engine->create_crypto_socket(std::move(socket), is_server); + return _crypto_engine->create_client_crypto_socket(std::move(socket), spec); +} + +vespalib::CryptoSocket::UP +FNET_Transport::create_server_crypto_socket(vespalib::SocketHandle socket) +{ + return _crypto_engine->create_server_crypto_socket(std::move(socket)); } FNET_TransportThread * diff --git a/fnet/src/vespa/fnet/transport.h b/fnet/src/vespa/fnet/transport.h index 8d1ba48c1b0..02ef22c7fb6 100644 --- a/fnet/src/vespa/fnet/transport.h +++ b/fnet/src/vespa/fnet/transport.h @@ -79,17 +79,25 @@ public: vespalib::AsyncResolver::ResultHandler::WP result_handler); /** - * Wrap a plain socket endpoint in a CryptoSocket. The + * Wrap a plain socket endpoint (client side) in a CryptoSocket. The * implementation will be determined by the CryptoEngine used by * this Transport. * * @return socket abstraction able to perform encryption and decryption * @param socket low-level socket - * @param is_server which end of the connection the socket - * represents. This is needed to support - * asymmetrical handshaking. + * @param spec who we are connecting to **/ - vespalib::CryptoSocket::UP create_crypto_socket(vespalib::SocketHandle socket, bool is_server); + vespalib::CryptoSocket::UP create_client_crypto_socket(vespalib::SocketHandle socket, const vespalib::SocketSpec &spec); + + /** + * Wrap a plain socket endpoint (server side) in a CryptoSocket. The + * implementation will be determined by the CryptoEngine used by + * this Transport. + * + * @return socket abstraction able to perform encryption and decryption + * @param socket low-level socket + **/ + vespalib::CryptoSocket::UP create_server_crypto_socket(vespalib::SocketHandle socket); /** * Select one of the underlying transport threads. The selection diff --git a/jrt/src/com/yahoo/jrt/Connection.java b/jrt/src/com/yahoo/jrt/Connection.java index d4e1a15b957..c9c6d78ffba 100644 --- a/jrt/src/com/yahoo/jrt/Connection.java +++ b/jrt/src/com/yahoo/jrt/Connection.java @@ -93,7 +93,7 @@ class Connection extends Target { this.parent = parent; this.owner = owner; - this.socket = parent.transport().createCryptoSocket(channel, true); + this.socket = parent.transport().createServerCryptoSocket(channel); this.spec = null; server = true; owner.sessionInit(this); @@ -171,7 +171,7 @@ class Connection extends Target { return this; } try { - socket = parent.transport().createCryptoSocket(SocketChannel.open(spec.resolveAddress()), false); + socket = parent.transport().createClientCryptoSocket(SocketChannel.open(spec.resolveAddress()), spec); } catch (Exception e) { setLostReason(e); } diff --git a/jrt/src/com/yahoo/jrt/CryptoEngine.java b/jrt/src/com/yahoo/jrt/CryptoEngine.java index 8812264a3f1..6d1955d7f66 100644 --- a/jrt/src/com/yahoo/jrt/CryptoEngine.java +++ b/jrt/src/com/yahoo/jrt/CryptoEngine.java @@ -18,7 +18,8 @@ import java.nio.channels.SocketChannel; * encryption. **/ public interface CryptoEngine extends AutoCloseable { - CryptoSocket createCryptoSocket(SocketChannel channel, boolean isServer); + CryptoSocket createClientCryptoSocket(SocketChannel channel, Spec spec); + CryptoSocket createServerCryptoSocket(SocketChannel channel); static CryptoEngine createDefault() { if (!TransportSecurityUtils.isTransportSecurityEnabled()) { return new NullCryptoEngine(); diff --git a/jrt/src/com/yahoo/jrt/MaybeTlsCryptoEngine.java b/jrt/src/com/yahoo/jrt/MaybeTlsCryptoEngine.java index 801f2075c4e..18549df6f2c 100644 --- a/jrt/src/com/yahoo/jrt/MaybeTlsCryptoEngine.java +++ b/jrt/src/com/yahoo/jrt/MaybeTlsCryptoEngine.java @@ -21,17 +21,20 @@ public class MaybeTlsCryptoEngine implements CryptoEngine { } @Override - public CryptoSocket createCryptoSocket(SocketChannel channel, boolean isServer) { - if (isServer) { - return new MaybeTlsCryptoSocket(channel, tlsEngine, isServer); - } else if (useTlsWhenClient) { - return tlsEngine.createCryptoSocket(channel, false); + public CryptoSocket createClientCryptoSocket(SocketChannel channel, Spec spec) { + if (useTlsWhenClient) { + return tlsEngine.createClientCryptoSocket(channel, spec); } else { - return new NullCryptoSocket(channel, isServer); + return new NullCryptoSocket(channel, false); } } @Override + public CryptoSocket createServerCryptoSocket(SocketChannel channel) { + return new MaybeTlsCryptoSocket(channel, tlsEngine); + } + + @Override public String toString() { return "MaybeTlsCryptoEngine(useTlsWhenClient:" + useTlsWhenClient + ")"; } @Override diff --git a/jrt/src/com/yahoo/jrt/MaybeTlsCryptoSocket.java b/jrt/src/com/yahoo/jrt/MaybeTlsCryptoSocket.java index 5c4510665e7..60b7f342c9c 100644 --- a/jrt/src/com/yahoo/jrt/MaybeTlsCryptoSocket.java +++ b/jrt/src/com/yahoo/jrt/MaybeTlsCryptoSocket.java @@ -61,8 +61,8 @@ public class MaybeTlsCryptoSocket implements CryptoSocket { private TlsCryptoEngine factory; private Buffer buffer; - MyCryptoSocket(SocketChannel channel, TlsCryptoEngine factory, boolean isServer) { - super(channel, isServer); + MyCryptoSocket(SocketChannel channel, TlsCryptoEngine factory) { + super(channel, true); this.factory = factory; this.buffer = new Buffer(4096); } @@ -81,7 +81,7 @@ public class MaybeTlsCryptoSocket implements CryptoSocket { data[i] = src.get(i); } if (looksLikeTlsToMe(data)) { - TlsCryptoSocket tlsSocket = factory.createCryptoSocket(channel(), true); + TlsCryptoSocket tlsSocket = factory.createServerCryptoSocket(channel()); tlsSocket.injectReadData(buffer); socket = tlsSocket; return socket.handshake(); @@ -117,8 +117,8 @@ public class MaybeTlsCryptoSocket implements CryptoSocket { } } - public MaybeTlsCryptoSocket(SocketChannel channel, TlsCryptoEngine factory, boolean isServer) { - this.socket = new MyCryptoSocket(channel, factory, isServer); + public MaybeTlsCryptoSocket(SocketChannel channel, TlsCryptoEngine factory) { + this.socket = new MyCryptoSocket(channel, factory); } @Override public SocketChannel channel() { return socket.channel(); } diff --git a/jrt/src/com/yahoo/jrt/NullCryptoEngine.java b/jrt/src/com/yahoo/jrt/NullCryptoEngine.java index b5a53accf92..b97ec17a5dc 100644 --- a/jrt/src/com/yahoo/jrt/NullCryptoEngine.java +++ b/jrt/src/com/yahoo/jrt/NullCryptoEngine.java @@ -9,7 +9,10 @@ import java.nio.channels.SocketChannel; * CryptoEngine implementation that performs no encryption. **/ public class NullCryptoEngine implements CryptoEngine { - @Override public CryptoSocket createCryptoSocket(SocketChannel channel, boolean isServer) { - return new NullCryptoSocket(channel, isServer); + @Override public CryptoSocket createClientCryptoSocket(SocketChannel channel, Spec spec) { + return new NullCryptoSocket(channel, false); + } + @Override public CryptoSocket createServerCryptoSocket(SocketChannel channel) { + return new NullCryptoSocket(channel, true); } } diff --git a/jrt/src/com/yahoo/jrt/TlsCryptoEngine.java b/jrt/src/com/yahoo/jrt/TlsCryptoEngine.java index 84fbb7d4f01..7474220d4e7 100644 --- a/jrt/src/com/yahoo/jrt/TlsCryptoEngine.java +++ b/jrt/src/com/yahoo/jrt/TlsCryptoEngine.java @@ -20,9 +20,16 @@ public class TlsCryptoEngine implements CryptoEngine { } @Override - public TlsCryptoSocket createCryptoSocket(SocketChannel channel, boolean isServer) { + public TlsCryptoSocket createClientCryptoSocket(SocketChannel channel, Spec spec) { SSLEngine sslEngine = tlsContext.createSslEngine(); - sslEngine.setUseClientMode(!isServer); + sslEngine.setUseClientMode(true); + return new TlsCryptoSocket(channel, sslEngine); + } + + @Override + public TlsCryptoSocket createServerCryptoSocket(SocketChannel channel) { + SSLEngine sslEngine = tlsContext.createSslEngine(); + sslEngine.setUseClientMode(false); return new TlsCryptoSocket(channel, sslEngine); } diff --git a/jrt/src/com/yahoo/jrt/Transport.java b/jrt/src/com/yahoo/jrt/Transport.java index ad42409c48a..6f5a381fd6b 100644 --- a/jrt/src/com/yahoo/jrt/Transport.java +++ b/jrt/src/com/yahoo/jrt/Transport.java @@ -68,14 +68,26 @@ public class Transport { } /** - * Use the underlying CryptoEngine to create a CryptoSocket. + * Use the underlying CryptoEngine to create a CryptoSocket for + * the client side of a connection. * * @return CryptoSocket handling appropriate encryption * @param channel low-level socket channel to be wrapped by the CryptoSocket - * @param isServer flag indicating which end of the connection we are + * @param spec who we are connecting to, for hostname validation **/ - CryptoSocket createCryptoSocket(SocketChannel channel, boolean isServer) { - return cryptoEngine.createCryptoSocket(channel, isServer); + CryptoSocket createClientCryptoSocket(SocketChannel channel, Spec spec) { + return cryptoEngine.createClientCryptoSocket(channel, spec); + } + + /** + * Use the underlying CryptoEngine to create a CryptoSocket for + * the server side of a connection. + * + * @return CryptoSocket handling appropriate encryption + * @param channel low-level socket channel to be wrapped by the CryptoSocket + **/ + CryptoSocket createServerCryptoSocket(SocketChannel channel) { + return cryptoEngine.createServerCryptoSocket(channel); } /** diff --git a/jrt/src/com/yahoo/jrt/XorCryptoEngine.java b/jrt/src/com/yahoo/jrt/XorCryptoEngine.java index 4ba6d00faa4..d720ca4dc26 100644 --- a/jrt/src/com/yahoo/jrt/XorCryptoEngine.java +++ b/jrt/src/com/yahoo/jrt/XorCryptoEngine.java @@ -11,7 +11,10 @@ import java.nio.channels.SocketChannel; * from TLS. **/ public class XorCryptoEngine implements CryptoEngine { - @Override public CryptoSocket createCryptoSocket(SocketChannel channel, boolean isServer) { - return new XorCryptoSocket(channel, isServer); + @Override public CryptoSocket createClientCryptoSocket(SocketChannel channel, Spec spec) { + return new XorCryptoSocket(channel, false); + } + @Override public CryptoSocket createServerCryptoSocket(SocketChannel channel) { + return new XorCryptoSocket(channel, true); } } diff --git a/logforwarder/src/apps/vespa-logforwarder-start/cf-handler.cpp b/logforwarder/src/apps/vespa-logforwarder-start/cf-handler.cpp index addf9125508..22550a19383 100644 --- a/logforwarder/src/apps/vespa-logforwarder-start/cf-handler.cpp +++ b/logforwarder/src/apps/vespa-logforwarder-start/cf-handler.cpp @@ -46,10 +46,11 @@ bool fixDir(const vespalib::string &path) { } vespalib::string -cfFilePath(const vespalib::string &parent) { +cfFilePath(const vespalib::string &parent, const vespalib::string &filename) { vespalib::string path = parent + "/etc/system/local"; fixDir(path); - path += "/deploymentclient.conf"; + path += "/"; + path += filename; return path; } @@ -61,7 +62,7 @@ CfHandler::doConfigure() std::unique_ptr<LogforwarderConfig> cfg(_handle->getConfig()); const LogforwarderConfig& config(*cfg); - vespalib::string path = cfFilePath(config.splunkHome); + vespalib::string path = cfFilePath(config.splunkHome, "deploymentclient.conf"); vespalib::string tmpPath = path + ".new"; FILE *fp = fopen(tmpPath.c_str(), "w"); if (fp == NULL) return; @@ -76,6 +77,22 @@ CfHandler::doConfigure() fclose(fp); rename(tmpPath.c_str(), path.c_str()); + if (getenv("VESPA_HOSTNAME") != NULL && + getenv("VESPA_TENANT") != NULL && + getenv("VESPA_APPLICATION")!= NULL && + getenv("VESPA_INSTANCE") != NULL ) + { + path = cfFilePath(config.splunkHome, "inputs.conf"); + tmpPath = path + ".new"; + fp = fopen(tmpPath.c_str(), "w"); + if (fp != NULL) { + fprintf(fp, "[default]\n"); + fprintf(fp, "host = %s\n", getenv("VESPA_HOSTNAME")); + fprintf(fp, "_meta = vespa_tenant::%s vespa_application::%s vespa_instance::%s\n", getenv("VESPA_TENANT"), getenv("VESPA_APPLICATION"), getenv("VESPA_INSTANCE")); + fclose(fp); + rename(tmpPath.c_str(), path.c_str()); + } + } if (config.clientName.size() == 0 || config.deploymentServer.size() == 0) { diff --git a/logserver/bin/logserver-start.sh b/logserver/bin/logserver-start.sh index 9f55e218140..913cdb78327 100755 --- a/logserver/bin/logserver-start.sh +++ b/logserver/bin/logserver-start.sh @@ -78,7 +78,7 @@ ROOT=${VESPA_HOME%/} export ROOT cd $ROOT || { echo "Cannot cd to $ROOT" 1>&2; exit 1; } -addopts="-server -Xms32m -Xmx256m -XX:MaxDirectMemorySize=76m -XX:MaxJavaStackTraceDepth=1000000" +addopts="-server -Xms32m -Xmx256m -XX:CompressedClassSpaceSize=32m -XX:MaxDirectMemorySize=32m -XX:ThreadStackSize=256 -XX:MaxJavaStackTraceDepth=1000 -XX:ActiveProcessorCount=2" oomopt="-XX:+ExitOnOutOfMemoryError" diff --git a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/core/MetricsManager.java b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/core/MetricsManager.java index 4c4015220bc..53a05ef88f0 100644 --- a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/core/MetricsManager.java +++ b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/core/MetricsManager.java @@ -79,6 +79,16 @@ public class MetricsManager { * @return Metrics for all matching services. */ public List<MetricsPacket> getMetrics(List<VespaService> services, Instant startTime) { + return getMetricsAsBuilders(services, startTime).stream() + .map(MetricsPacket.Builder::build) + .collect(Collectors.toList()); + } + + /** + * Returns the metrics for the given services, in mutable state for further processing. + * NOTE: Use {@link #getMetrics(List, Instant)} instead, unless further processing of the metrics is necessary. + */ + public List<MetricsPacket.Builder> getMetricsAsBuilders(List<VespaService> services, Instant startTime) { if (services.isEmpty()) return Collections.emptyList(); log.log(DEBUG, () -> "Updating services prior to fetching metrics, number of services= " + services.size()); @@ -99,7 +109,6 @@ public class MetricsManager { .map(builder -> builder.putDimensionsIfAbsent(getGlobalDimensions())) .map(builder -> builder.putDimensionsIfAbsent(extraDimensions)) .map(builder -> adjustTimestamp(builder, startTime)) - .map(MetricsPacket.Builder::build) .collect(Collectors.toList()); } diff --git a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/http/ValuesFetcher.java b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/http/ValuesFetcher.java index 768c1beebef..ae0ef2fa57a 100644 --- a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/http/ValuesFetcher.java +++ b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/http/ValuesFetcher.java @@ -46,6 +46,16 @@ public class ValuesFetcher { .collect(Collectors.toList()); } + public List<MetricsPacket.Builder> fetchMetricsAsBuilders(String requestedConsumer) throws JsonRenderingException { + ConsumerId consumer = getConsumerOrDefault(requestedConsumer, metricsConsumers); + + return metricsManager.getMetricsAsBuilders(vespaServices.getVespaServices(), Instant.now()) + .stream() + .filter(builder -> builder.hasConsumer(consumer)) + .collect(Collectors.toList()); + } + + public List<MetricsPacket> fetchAllMetrics() throws JsonRenderingException { return metricsManager.getMetrics(vespaServices.getVespaServices(), Instant.now()); } diff --git a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/http/application/Node.java b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/http/application/Node.java index c8a8e65be5d..c439a037774 100644 --- a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/http/application/Node.java +++ b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/http/application/Node.java @@ -25,7 +25,7 @@ public class Node { } public Node(String role, String hostname, int port, String path) { - Objects.requireNonNull(role, "Null configId is not allowed"); + Objects.requireNonNull(role, "Null role is not allowed"); Objects.requireNonNull(hostname, "Null hostname is not allowed"); Objects.requireNonNull(path, "Null path is not allowed"); diff --git a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/http/application/PublicDimensionsProcessor.java b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/http/application/PublicDimensionsProcessor.java index 395ec0bea4f..4d1d57644b5 100644 --- a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/http/application/PublicDimensionsProcessor.java +++ b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/http/application/PublicDimensionsProcessor.java @@ -21,7 +21,7 @@ public class PublicDimensionsProcessor implements MetricsProcessor { private final int maxDimensions; private Set<DimensionId> publicDimensions = getPublicDimensions(); - PublicDimensionsProcessor(int maxDimensions) { + public PublicDimensionsProcessor(int maxDimensions) { int numCommonDimensions = PublicDimensions.commonDimensions.size(); if (numCommonDimensions > maxDimensions) { throw new IllegalArgumentException(String.format( diff --git a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/http/metrics/MetricsV2Handler.java b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/http/metrics/MetricsV2Handler.java new file mode 100644 index 00000000000..71d7857e48a --- /dev/null +++ b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/http/metrics/MetricsV2Handler.java @@ -0,0 +1,92 @@ +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package ai.vespa.metricsproxy.http.metrics; + +import ai.vespa.metricsproxy.core.MetricsConsumers; +import ai.vespa.metricsproxy.core.MetricsManager; +import ai.vespa.metricsproxy.http.ValuesFetcher; +import ai.vespa.metricsproxy.http.application.ClusterIdDimensionProcessor; +import ai.vespa.metricsproxy.http.application.Node; +import ai.vespa.metricsproxy.http.application.PublicDimensionsProcessor; +import ai.vespa.metricsproxy.http.application.ServiceIdDimensionProcessor; +import ai.vespa.metricsproxy.metric.model.MetricsPacket; +import ai.vespa.metricsproxy.metric.model.processing.MetricsProcessor; +import ai.vespa.metricsproxy.service.VespaServices; +import com.google.inject.Inject; +import com.yahoo.container.handler.metrics.ErrorResponse; +import com.yahoo.container.handler.metrics.HttpHandlerBase; +import com.yahoo.container.handler.metrics.JsonResponse; +import com.yahoo.container.jdisc.HttpResponse; +import com.yahoo.restapi.Path; + +import java.net.URI; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.concurrent.Executor; +import java.util.logging.Level; + +import static ai.vespa.metricsproxy.metric.model.json.GenericJsonUtil.toGenericApplicationModel; +import static ai.vespa.metricsproxy.metric.model.processing.MetricsProcessor.applyProcessors; +import static com.yahoo.jdisc.Response.Status.INTERNAL_SERVER_ERROR; +import static com.yahoo.jdisc.Response.Status.OK; +import static java.util.Collections.singletonMap; +import static java.util.stream.Collectors.toList; + +/** + * Http handler for the metrics/v2 rest api. + * + * @author gjoranv + */ +public class MetricsV2Handler extends HttpHandlerBase { + + public static final String V2_PATH = "/metrics/v2"; + public static final String VALUES_PATH = V2_PATH + "/values"; + private static final int MAX_DIMENSIONS = 10; + + private final ValuesFetcher valuesFetcher; + private final NodeInfoConfig nodeInfoConfig; + + @Inject + public MetricsV2Handler(Executor executor, + MetricsManager metricsManager, + VespaServices vespaServices, + MetricsConsumers metricsConsumers, + NodeInfoConfig nodeInfoConfig) { + super(executor); + this.nodeInfoConfig = nodeInfoConfig; + valuesFetcher = new ValuesFetcher(metricsManager, vespaServices, metricsConsumers); + } + + @Override + public Optional<HttpResponse> doHandle(URI requestUri, Path apiPath, String consumer) { + if (apiPath.matches(V2_PATH)) return Optional.of(resourceListResponse(requestUri, List.of(VALUES_PATH))); + if (apiPath.matches(VALUES_PATH)) return Optional.of(valuesResponse(consumer)); + return Optional.empty(); + } + + private JsonResponse valuesResponse(String consumer) { + try { + List<MetricsPacket.Builder> builders = valuesFetcher.fetchMetricsAsBuilders(consumer); + List<MetricsPacket> metrics = processAndBuild(builders, + new ServiceIdDimensionProcessor(), + new ClusterIdDimensionProcessor(), + new PublicDimensionsProcessor(MAX_DIMENSIONS)); + + Node localNode = new Node(nodeInfoConfig.role(), nodeInfoConfig.hostname(), 0, ""); + Map<Node, List<MetricsPacket>> metricsByNode = singletonMap(localNode, metrics); + return new JsonResponse(OK, toGenericApplicationModel(metricsByNode).serialize()); + } catch (Exception e) { + log.log(Level.WARNING, "Got exception when rendering metrics:", e); + return new ErrorResponse(INTERNAL_SERVER_ERROR, e.getMessage()); + } + } + + private static List<MetricsPacket> processAndBuild(List<MetricsPacket.Builder> builders, + MetricsProcessor... processors) { + return builders.stream() + .map(builder -> applyProcessors(builder, processors)) + .map(MetricsPacket.Builder::build) + .collect(toList()); + } + +} diff --git a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/model/MetricsPacket.java b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/model/MetricsPacket.java index 8ecf57237ef..8d5a1f50918 100644 --- a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/model/MetricsPacket.java +++ b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/model/MetricsPacket.java @@ -179,6 +179,10 @@ public class MetricsPacket { return this; } + public boolean hasConsumer(ConsumerId id) { + return consumers.contains(id); + } + public MetricsPacket build() { return new MetricsPacket(statusCode, statusMessage, timestamp, service, metrics, dimensions, consumers); } diff --git a/metrics-proxy/src/main/resources/configdefinitions/node-info.def b/metrics-proxy/src/main/resources/configdefinitions/node-info.def new file mode 100644 index 00000000000..e66433a96d0 --- /dev/null +++ b/metrics-proxy/src/main/resources/configdefinitions/node-info.def @@ -0,0 +1,5 @@ +# Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package=ai.vespa.metricsproxy.http.metrics + +role string +hostname string diff --git a/metrics-proxy/src/test/java/ai/vespa/metricsproxy/http/HttpHandlerTestBase.java b/metrics-proxy/src/test/java/ai/vespa/metricsproxy/http/HttpHandlerTestBase.java index 77c3a719cd9..d776368687d 100644 --- a/metrics-proxy/src/test/java/ai/vespa/metricsproxy/http/HttpHandlerTestBase.java +++ b/metrics-proxy/src/test/java/ai/vespa/metricsproxy/http/HttpHandlerTestBase.java @@ -24,6 +24,7 @@ import java.util.List; import static ai.vespa.metricsproxy.http.ValuesFetcher.DEFAULT_PUBLIC_CONSUMER_ID; import static ai.vespa.metricsproxy.metric.ExternalMetrics.VESPA_NODE_SERVICE_ID; +import static ai.vespa.metricsproxy.metric.dimensions.PublicDimensions.REASON; import static ai.vespa.metricsproxy.service.DummyService.METRIC_1; /** @@ -61,11 +62,12 @@ public class HttpHandlerTestBase { } protected static MetricsConsumers getMetricsConsumers() { + // Must use a whitelisted dimension to avoid it being removed for the MetricsV2Handler var defaultConsumerDimension = new ConsumersConfig.Consumer.Metric.Dimension.Builder() - .key("consumer-dim").value("default-val"); + .key(REASON).value("default-val"); var customConsumerDimension = new ConsumersConfig.Consumer.Metric.Dimension.Builder() - .key("consumer-dim").value("custom-val"); + .key(REASON).value("custom-val"); return new MetricsConsumers(new ConsumersConfig.Builder() .consumer(new ConsumersConfig.Consumer.Builder() diff --git a/metrics-proxy/src/test/java/ai/vespa/metricsproxy/http/metrics/MetricsHandlerTestBase.java b/metrics-proxy/src/test/java/ai/vespa/metricsproxy/http/metrics/MetricsHandlerTestBase.java new file mode 100644 index 00000000000..1c5ce695155 --- /dev/null +++ b/metrics-proxy/src/test/java/ai/vespa/metricsproxy/http/metrics/MetricsHandlerTestBase.java @@ -0,0 +1,196 @@ +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package ai.vespa.metricsproxy.http.metrics; + +import ai.vespa.metricsproxy.http.HttpHandlerTestBase; +import ai.vespa.metricsproxy.metric.model.json.GenericJsonModel; +import ai.vespa.metricsproxy.metric.model.json.GenericMetrics; +import ai.vespa.metricsproxy.metric.model.json.GenericService; +import ai.vespa.metricsproxy.service.DownService; +import com.fasterxml.jackson.databind.ObjectMapper; +import org.json.JSONArray; +import org.json.JSONObject; +import org.junit.Ignore; +import org.junit.Test; + +import java.io.IOException; + +import static ai.vespa.metricsproxy.metric.dimensions.PublicDimensions.INTERNAL_SERVICE_ID; +import static ai.vespa.metricsproxy.metric.dimensions.PublicDimensions.REASON; +import static ai.vespa.metricsproxy.metric.dimensions.PublicDimensions.SERVICE_ID; +import static ai.vespa.metricsproxy.metric.model.StatusCode.DOWN; +import static ai.vespa.metricsproxy.metric.model.json.JacksonUtil.createObjectMapper; +import static ai.vespa.metricsproxy.service.DummyService.METRIC_1; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotEquals; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; + +/** + * @author gjoranv + */ +public abstract class MetricsHandlerTestBase<MODEL> extends HttpHandlerTestBase { + + static String rootUri; + static String valuesUri; + + Class<MODEL> modelClass; + + abstract GenericJsonModel getGenericJsonModel(MODEL model); + + private MODEL getResponseAsJsonModel(String consumer) { + String response = testDriver.sendRequest(valuesUri + "?consumer=" + consumer).readAll(); + try { + return createObjectMapper().readValue(response, modelClass); + } catch (IOException e) { + fail("Failed to create json model: " + e.getMessage()); + throw new RuntimeException(e); + } + } + + private GenericJsonModel getResponseAsGenericJsonModel(String consumer) { + return getGenericJsonModel(getResponseAsJsonModel(consumer)); + } + + @Test + public void invalid_path_yields_error_response() throws Exception { + String response = testDriver.sendRequest(rootUri + "/invalid").readAll(); + JSONObject root = new JSONObject(response); + assertTrue(root.has("error")); + } + + @Test + public void root_response_contains_values_uri() throws Exception { + String response = testDriver.sendRequest(rootUri).readAll(); + JSONObject root = new JSONObject(response); + assertTrue(root.has("resources")); + + JSONArray resources = root.getJSONArray("resources"); + assertEquals(1, resources.length()); + + JSONObject valuesUrl = resources.getJSONObject(0); + assertEquals(valuesUri, valuesUrl.getString("url")); + } + + @Ignore + @Test + public void visually_inspect_values_response() throws Exception { + String response = testDriver.sendRequest(valuesUri).readAll(); + ObjectMapper mapper = createObjectMapper(); + var jsonModel = mapper.readValue(response, modelClass); + System.out.println(mapper.writerWithDefaultPrettyPrinter().writeValueAsString(jsonModel)); + } + + @Test + public void no_explicit_consumer_gives_the_default_consumer() { + String responseDefaultConsumer = testDriver.sendRequest(valuesUri + "?consumer=default").readAll(); + String responseNoConsumer = testDriver.sendRequest(valuesUri).readAll(); + assertEqualsExceptTimestamps(responseDefaultConsumer, responseNoConsumer); + } + + @Test + public void unknown_consumer_gives_the_default_consumer() { + String response = testDriver.sendRequest(valuesUri).readAll(); + String responseUnknownConsumer = testDriver.sendRequest(valuesUri + "?consumer=not_defined").readAll(); + assertEqualsExceptTimestamps(response, responseUnknownConsumer); + } + + private void assertEqualsExceptTimestamps(String s1, String s2) { + assertEquals(replaceTimestamps(s1), replaceTimestamps(s2)); + } + + private String replaceTimestamps(String s) { + return s.replaceAll("timestamp\":\\d+,", "timestamp\":1,"); + } + + @Test + public void response_contains_node_metrics() { + GenericJsonModel jsonModel = getResponseAsGenericJsonModel(DEFAULT_CONSUMER); + + assertNotNull(jsonModel.node); + assertEquals(1, jsonModel.node.metrics.size()); + assertEquals(12.345, jsonModel.node.metrics.get(0).values.get(CPU_METRIC), 0.0001d); + } + + @Test + public void response_contains_service_metrics() { + GenericJsonModel jsonModel = getResponseAsGenericJsonModel(DEFAULT_CONSUMER); + + assertEquals(2, jsonModel.services.size()); + GenericService dummyService = jsonModel.services.get(0); + assertEquals(2, dummyService.metrics.size()); + + GenericMetrics dummy0Metrics = getMetricsForService("dummy0", dummyService); + assertEquals(1L, dummy0Metrics.values.get(METRIC_1).longValue()); + assertEquals("default-val", dummy0Metrics.dimensions.get(REASON)); + + GenericMetrics dummy1Metrics = getMetricsForService("dummy1", dummyService); + assertEquals(6L, dummy1Metrics.values.get(METRIC_1).longValue()); + assertEquals("default-val", dummy1Metrics.dimensions.get(REASON)); + } + + @Test + public void custom_consumer_gets_only_its_whitelisted_metrics() { + GenericJsonModel jsonModel = getResponseAsGenericJsonModel(CUSTOM_CONSUMER); + + assertNotNull(jsonModel.node); + // TODO: see comment in ExternalMetrics.setExtraMetrics + // assertEquals(0, jsonModel.node.metrics.size()); + + assertEquals(2, jsonModel.services.size()); + GenericService dummyService = jsonModel.services.get(0); + assertEquals(2, dummyService.metrics.size()); + + GenericMetrics dummy0Metrics = getMetricsForService("dummy0", dummyService); + assertEquals("custom-val", dummy0Metrics.dimensions.get(REASON)); + + GenericMetrics dummy1Metrics = getMetricsForService("dummy1", dummyService); + assertEquals("custom-val", dummy1Metrics.dimensions.get(REASON)); + } + + private static GenericMetrics getMetricsForService(String serviceInstance, GenericService service) { + for (var metrics : service.metrics) { + if (getServiceIdDimension(metrics).equals(serviceInstance)) + return metrics; + } + fail("Could not find metrics for service instance " + serviceInstance); + throw new RuntimeException(); + } + + @Test + public void all_timestamps_are_equal_and_non_zero() { + GenericJsonModel jsonModel = getResponseAsGenericJsonModel(DEFAULT_CONSUMER); + + Long nodeTimestamp = jsonModel.node.timestamp; + assertNotEquals(0L, (long) nodeTimestamp); + for (var service : jsonModel.services) + assertEquals(nodeTimestamp, service.timestamp); + } + + @Test + public void all_consumers_get_health_from_service_that_is_down() { + assertDownServiceHealth(DEFAULT_CONSUMER); + assertDownServiceHealth(CUSTOM_CONSUMER); + } + + private void assertDownServiceHealth(String consumer) { + GenericJsonModel jsonModel = getResponseAsGenericJsonModel(consumer); + + GenericService downService = jsonModel.services.get(1); + assertEquals(DOWN.status, downService.status.code); + assertEquals("No response", downService.status.description); + + // Service should output metric dimensions, even without metrics, because they contain important info about the service. + assertEquals(1, downService.metrics.size()); + assertEquals(0, downService.metrics.get(0).values.size()); + assertFalse(downService.metrics.get(0).dimensions.isEmpty()); + assertEquals(DownService.NAME, getServiceIdDimension(downService.metrics.get(0))); + } + + private static String getServiceIdDimension(GenericMetrics metrics) { + var instanceDimension = metrics.dimensions.get(INTERNAL_SERVICE_ID); + return instanceDimension != null ? instanceDimension : metrics.dimensions.get(SERVICE_ID); + } + +} diff --git a/metrics-proxy/src/test/java/ai/vespa/metricsproxy/http/metrics/MetricsV1HandlerTest.java b/metrics-proxy/src/test/java/ai/vespa/metricsproxy/http/metrics/MetricsV1HandlerTest.java index 22f61114622..fe823466f7b 100644 --- a/metrics-proxy/src/test/java/ai/vespa/metricsproxy/http/metrics/MetricsV1HandlerTest.java +++ b/metrics-proxy/src/test/java/ai/vespa/metricsproxy/http/metrics/MetricsV1HandlerTest.java @@ -1,46 +1,30 @@ // Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package ai.vespa.metricsproxy.http.metrics; -import ai.vespa.metricsproxy.http.HttpHandlerTestBase; import ai.vespa.metricsproxy.metric.model.json.GenericJsonModel; -import ai.vespa.metricsproxy.metric.model.json.GenericMetrics; -import ai.vespa.metricsproxy.metric.model.json.GenericService; -import ai.vespa.metricsproxy.service.DownService; -import com.fasterxml.jackson.databind.ObjectMapper; import com.yahoo.container.jdisc.RequestHandlerTestDriver; -import org.json.JSONArray; -import org.json.JSONObject; +import org.junit.Before; import org.junit.BeforeClass; -import org.junit.Ignore; -import org.junit.Test; -import java.io.IOException; import java.util.concurrent.Executors; -import static ai.vespa.metricsproxy.core.VespaMetrics.INSTANCE_DIMENSION_ID; import static ai.vespa.metricsproxy.http.metrics.MetricsV1Handler.V1_PATH; import static ai.vespa.metricsproxy.http.metrics.MetricsV1Handler.VALUES_PATH; -import static ai.vespa.metricsproxy.metric.model.StatusCode.DOWN; -import static ai.vespa.metricsproxy.metric.model.json.JacksonUtil.createObjectMapper; -import static ai.vespa.metricsproxy.service.DummyService.METRIC_1; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertNotEquals; -import static org.junit.Assert.assertNotNull; -import static org.junit.Assert.assertTrue; -import static org.junit.Assert.fail; /** * @author gjoranv */ @SuppressWarnings("UnstableApiUsage") -public class MetricsV1HandlerTest extends HttpHandlerTestBase { +public class MetricsV1HandlerTest extends MetricsHandlerTestBase<GenericJsonModel> { private static final String V1_URI = URI_BASE + V1_PATH; private static final String VALUES_URI = URI_BASE + VALUES_PATH; + @BeforeClass public static void setup() { + rootUri = V1_URI; + valuesUri = VALUES_URI; var handler = new MetricsV1Handler(Executors.newSingleThreadExecutor(), getMetricsManager(), vespaServices, @@ -48,149 +32,14 @@ public class MetricsV1HandlerTest extends HttpHandlerTestBase { testDriver = new RequestHandlerTestDriver(handler); } - private GenericJsonModel getResponseAsJsonModel(String consumer) { - String response = testDriver.sendRequest(VALUES_URI + "?consumer=" + consumer).readAll(); - try { - return createObjectMapper().readValue(response, GenericJsonModel.class); - } catch (IOException e) { - fail("Failed to create json model: " + e.getMessage()); - throw new RuntimeException(e); - } - } - - @Test - public void v1_response_contains_values_uri() throws Exception { - String response = testDriver.sendRequest(V1_URI).readAll(); - JSONObject root = new JSONObject(response); - assertTrue(root.has("resources")); - - JSONArray resources = root.getJSONArray("resources"); - assertEquals(1, resources.length()); - - JSONObject valuesUrl = resources.getJSONObject(0); - assertEquals(VALUES_URI, valuesUrl.getString("url")); - } - - @Ignore - @Test - public void visually_inspect_values_response() throws Exception { - String response = testDriver.sendRequest(VALUES_URI).readAll(); - ObjectMapper mapper = createObjectMapper(); - var jsonModel = mapper.readValue(response, GenericJsonModel.class); - System.out.println(mapper.writerWithDefaultPrettyPrinter().writeValueAsString(jsonModel)); - } - - @Test - public void no_explicit_consumer_gives_the_default_consumer() { - String responseDefaultConsumer = testDriver.sendRequest(VALUES_URI + "?consumer=default").readAll(); - String responseNoConsumer = testDriver.sendRequest(VALUES_URI).readAll(); - assertEqualsExceptTimestamps(responseDefaultConsumer, responseNoConsumer); - } - - @Test - public void unknown_consumer_gives_the_default_consumer() { - String response = testDriver.sendRequest(VALUES_URI).readAll(); - String responseUnknownConsumer = testDriver.sendRequest(VALUES_URI + "?consumer=not_defined").readAll(); - assertEqualsExceptTimestamps(response, responseUnknownConsumer); - } - - private void assertEqualsExceptTimestamps(String s1, String s2) { - assertEquals(replaceTimestamps(s1), replaceTimestamps(s2)); - } - - @Test - public void response_contains_node_metrics() { - GenericJsonModel jsonModel = getResponseAsJsonModel(DEFAULT_CONSUMER); - - assertNotNull(jsonModel.node); - assertEquals(1, jsonModel.node.metrics.size()); - assertEquals(12.345, jsonModel.node.metrics.get(0).values.get(CPU_METRIC), 0.0001d); - } - - @Test - public void response_contains_service_metrics() { - GenericJsonModel jsonModel = getResponseAsJsonModel(DEFAULT_CONSUMER); - - assertEquals(2, jsonModel.services.size()); - GenericService dummyService = jsonModel.services.get(0); - assertEquals(2, dummyService.metrics.size()); - - GenericMetrics dummy0Metrics = getMetricsForInstance("dummy0", dummyService); - assertEquals(1L, dummy0Metrics.values.get(METRIC_1).longValue()); - assertEquals("default-val", dummy0Metrics.dimensions.get("consumer-dim")); - - GenericMetrics dummy1Metrics = getMetricsForInstance("dummy1", dummyService); - assertEquals(6L, dummy1Metrics.values.get(METRIC_1).longValue()); - assertEquals("default-val", dummy1Metrics.dimensions.get("consumer-dim")); - } - - @Test - public void all_consumers_get_health_from_service_that_is_down() { - assertDownServiceHealth(DEFAULT_CONSUMER); - assertDownServiceHealth(CUSTOM_CONSUMER); - } - - @Test - public void all_timestamps_are_equal_and_non_zero() { - GenericJsonModel jsonModel = getResponseAsJsonModel(DEFAULT_CONSUMER); - - Long nodeTimestamp = jsonModel.node.timestamp; - assertNotEquals(0L, (long) nodeTimestamp); - for (var service : jsonModel.services) - assertEquals(nodeTimestamp, service.timestamp); - } - - @Test - public void custom_consumer_gets_only_its_whitelisted_metrics() { - GenericJsonModel jsonModel = getResponseAsJsonModel(CUSTOM_CONSUMER); - - assertNotNull(jsonModel.node); - // TODO: see comment in ExternalMetrics.setExtraMetrics - // assertEquals(0, jsonModel.node.metrics.size()); - - assertEquals(2, jsonModel.services.size()); - GenericService dummyService = jsonModel.services.get(0); - assertEquals(2, dummyService.metrics.size()); - - GenericMetrics dummy0Metrics = getMetricsForInstance("dummy0", dummyService); - assertEquals("custom-val", dummy0Metrics.dimensions.get("consumer-dim")); - - GenericMetrics dummy1Metrics = getMetricsForInstance("dummy1", dummyService); - assertEquals("custom-val", dummy1Metrics.dimensions.get("consumer-dim")); - } - - @Test - public void invalid_path_yields_error_response() throws Exception { - String response = testDriver.sendRequest(V1_URI + "/invalid").readAll(); - JSONObject root = new JSONObject(response); - assertTrue(root.has("error")); - } - - private void assertDownServiceHealth(String consumer) { - GenericJsonModel jsonModel = getResponseAsJsonModel(consumer); - - GenericService downService = jsonModel.services.get(1); - assertEquals(DOWN.status, downService.status.code); - assertEquals("No response", downService.status.description); - - // Service should output metric dimensions, even without metrics, because they contain important info about the service. - assertEquals(1, downService.metrics.size()); - assertEquals(0, downService.metrics.get(0).values.size()); - assertFalse(downService.metrics.get(0).dimensions.isEmpty()); - assertEquals(DownService.NAME, downService.metrics.get(0).dimensions.get(INSTANCE_DIMENSION_ID.id)); - } - - private String replaceTimestamps(String s) { - return s.replaceAll("timestamp\":\\d+,", "timestamp\":1,"); + @Before + public void initModelClass() { + modelClass = GenericJsonModel.class; } - private static GenericMetrics getMetricsForInstance(String instance, GenericService service) { - for (var metrics : service.metrics) { - if (metrics.dimensions.get(INSTANCE_DIMENSION_ID.id).equals(instance)) - return metrics; - } - fail("Could not find metrics for service instance " + instance); - throw new RuntimeException(); + @Override + GenericJsonModel getGenericJsonModel(GenericJsonModel genericJsonModel) { + return genericJsonModel; } } diff --git a/metrics-proxy/src/test/java/ai/vespa/metricsproxy/http/metrics/MetricsV2HandlerTest.java b/metrics-proxy/src/test/java/ai/vespa/metricsproxy/http/metrics/MetricsV2HandlerTest.java new file mode 100644 index 00000000000..27ee6be4be3 --- /dev/null +++ b/metrics-proxy/src/test/java/ai/vespa/metricsproxy/http/metrics/MetricsV2HandlerTest.java @@ -0,0 +1,53 @@ +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package ai.vespa.metricsproxy.http.metrics; + +import ai.vespa.metricsproxy.metric.model.json.GenericApplicationModel; +import ai.vespa.metricsproxy.metric.model.json.GenericJsonModel; +import com.yahoo.container.jdisc.RequestHandlerTestDriver; +import org.junit.Before; +import org.junit.BeforeClass; + +import java.util.concurrent.Executors; + +import static ai.vespa.metricsproxy.http.metrics.MetricsV2Handler.V2_PATH; +import static ai.vespa.metricsproxy.http.metrics.MetricsV2Handler.VALUES_PATH; + +/** + * @author gjoranv + */ +@SuppressWarnings("UnstableApiUsage") +public class MetricsV2HandlerTest extends MetricsHandlerTestBase<GenericApplicationModel> { + + private static final String V2_URI = URI_BASE + V2_PATH; + private static final String VALUES_URI = URI_BASE + VALUES_PATH; + + + @BeforeClass + public static void setup() { + rootUri = V2_URI; + valuesUri = VALUES_URI; + var handler = new MetricsV2Handler(Executors.newSingleThreadExecutor(), + getMetricsManager(), + vespaServices, + getMetricsConsumers(), + nodeInfoConfig()); + testDriver = new RequestHandlerTestDriver(handler); + } + + @Before + public void initModelClass() { + modelClass = GenericApplicationModel.class; + } + + @Override + GenericJsonModel getGenericJsonModel(GenericApplicationModel genericApplicationModel) { + return genericApplicationModel.nodes.get(0); + } + + private static NodeInfoConfig nodeInfoConfig() { + return new NodeInfoConfig.Builder() + .role("my-role") + .hostname("my-hostname") + .build(); + } +} diff --git a/metrics-proxy/src/test/java/ai/vespa/metricsproxy/http/prometheus/PrometheusHandlerTest.java b/metrics-proxy/src/test/java/ai/vespa/metricsproxy/http/prometheus/PrometheusHandlerTest.java index a85f0425b4b..a224c4090b3 100644 --- a/metrics-proxy/src/test/java/ai/vespa/metricsproxy/http/prometheus/PrometheusHandlerTest.java +++ b/metrics-proxy/src/test/java/ai/vespa/metricsproxy/http/prometheus/PrometheusHandlerTest.java @@ -12,6 +12,7 @@ import org.junit.Test; import java.util.concurrent.Executors; +import static ai.vespa.metricsproxy.metric.dimensions.PublicDimensions.REASON; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; @@ -83,7 +84,7 @@ public class PrometheusHandlerTest extends HttpHandlerTestBase { @Test public void service_metrics_have_configured_dimensions() { String dummy0 = getLine(valuesResponse, DummyService.NAME + "0"); - assertTrue(dummy0.contains("consumer_dim=\"default-val\"")); + assertTrue(dummy0.contains(REASON + "=\"default-val\"")); } @Test diff --git a/metrics-proxy/src/test/java/ai/vespa/metricsproxy/metric/model/MetricsPacketTest.java b/metrics-proxy/src/test/java/ai/vespa/metricsproxy/metric/model/MetricsPacketTest.java index 9b37a805245..78c80689299 100644 --- a/metrics-proxy/src/test/java/ai/vespa/metricsproxy/metric/model/MetricsPacketTest.java +++ b/metrics-proxy/src/test/java/ai/vespa/metricsproxy/metric/model/MetricsPacketTest.java @@ -14,6 +14,7 @@ import java.util.Map; import static ai.vespa.metricsproxy.metric.model.ConsumerId.toConsumerId; import static ai.vespa.metricsproxy.metric.model.MetricId.toMetricId; import static ai.vespa.metricsproxy.metric.model.ServiceId.toServiceId; +import static java.util.Collections.singleton; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; @@ -45,13 +46,23 @@ public class MetricsPacketTest { MetricsPacket packet = new MetricsPacket.Builder(toServiceId("foo")) .statusCode(0) .statusMessage("") - .addConsumers(Collections.singleton(DUPLICATE_CONSUMER)) - .addConsumers(Collections.singleton(DUPLICATE_CONSUMER)) + .addConsumers(singleton(DUPLICATE_CONSUMER)) + .addConsumers(singleton(DUPLICATE_CONSUMER)) .build(); assertEquals(1, packet.consumers().size()); } @Test + public void builder_allows_inspecting_consumers() { + var consumer = toConsumerId("my-consumer"); + var builder = new MetricsPacket.Builder(toServiceId("foo")) + .statusCode(0) + .statusMessage("") + .addConsumers(singleton(consumer)); + assertTrue(builder.hasConsumer(consumer)); + } + + @Test public void builder_can_retain_subset_of_metrics() { MetricsPacket packet = new MetricsPacket.Builder(toServiceId("foo")) .putMetrics(ImmutableList.of( diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/NodeRepository.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/NodeRepository.java index 7d875434e1c..7d925b2a4aa 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/NodeRepository.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/NodeRepository.java @@ -1,4 +1,4 @@ -// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.provision; import com.google.inject.Inject; @@ -16,7 +16,10 @@ import com.yahoo.config.provisioning.NodeRepositoryConfig; import com.yahoo.transaction.Mutex; import com.yahoo.transaction.NestedTransaction; import com.yahoo.vespa.curator.Curator; +import com.yahoo.vespa.flags.FlagSource; +import com.yahoo.vespa.flags.Flags; import com.yahoo.vespa.hosted.provision.lb.LoadBalancer; +import com.yahoo.vespa.hosted.provision.lb.LoadBalancerId; import com.yahoo.vespa.hosted.provision.lb.LoadBalancerInstance; import com.yahoo.vespa.hosted.provision.lb.LoadBalancerList; import com.yahoo.vespa.hosted.provision.maintenance.InfrastructureVersions; @@ -50,6 +53,7 @@ import java.util.Optional; import java.util.Set; import java.util.TreeSet; import java.util.function.BiFunction; +import java.util.function.Predicate; import java.util.stream.Collectors; import java.util.stream.Stream; @@ -99,8 +103,8 @@ public class NodeRepository extends AbstractComponent { * This will use the system time to make time-sensitive decisions */ @Inject - public NodeRepository(NodeRepositoryConfig config, NodeFlavors flavors, Curator curator, Zone zone) { - this(flavors, curator, Clock.systemUTC(), zone, new DnsNameResolver(), DockerImage.fromString(config.dockerImage()), config.useCuratorClientCache()); + public NodeRepository(NodeRepositoryConfig config, NodeFlavors flavors, Curator curator, Zone zone, FlagSource flagSource) { + this(flavors, curator, Clock.systemUTC(), zone, new DnsNameResolver(), DockerImage.fromString(config.dockerImage()), config.useCuratorClientCache(), flagSource); } /** @@ -108,7 +112,7 @@ public class NodeRepository extends AbstractComponent { * which will be used for time-sensitive decisions. */ public NodeRepository(NodeFlavors flavors, Curator curator, Clock clock, Zone zone, NameResolver nameResolver, - DockerImage dockerImage, boolean useCuratorClientCache) { + DockerImage dockerImage, boolean useCuratorClientCache, FlagSource flagSource) { this.db = new CuratorDatabaseClient(flavors, curator, clock, zone, useCuratorClientCache); this.zone = zone; this.clock = clock; @@ -117,7 +121,7 @@ public class NodeRepository extends AbstractComponent { this.osVersions = new OsVersions(this); this.infrastructureVersions = new InfrastructureVersions(db); this.firmwareChecks = new FirmwareChecks(db, clock); - this.dockerImages = new DockerImages(db, dockerImage); + this.dockerImages = new DockerImages(db, dockerImage, Flags.DOCKER_IMAGE_OVERRIDE.bindTo(flagSource)); this.jobControl = new JobControl(db); // read and write all nodes to make sure they are stored in the latest version of the serialized format @@ -128,8 +132,8 @@ public class NodeRepository extends AbstractComponent { /** Returns the curator database client used by this */ public CuratorDatabaseClient database() { return db; } - /** Returns the Docker image to use for nodes in this */ - public DockerImage dockerImage(NodeType nodeType) { return dockerImages.dockerImageFor(nodeType); } + /** Returns the Docker image to use for given node */ + public DockerImage dockerImage(Node node) { return dockerImages.dockerImageFor(node); } /** @return The name resolver used to resolve hostname and ip addresses */ public NameResolver nameResolver() { return nameResolver; } @@ -194,7 +198,16 @@ public class NodeRepository extends AbstractComponent { /** Returns a filterable list of all load balancers in this repository */ public LoadBalancerList loadBalancers() { - return LoadBalancerList.copyOf(database().readLoadBalancers().values()); + return loadBalancers((ignored) -> true); + } + + /** Returns a filterable list of load balancers belonging to given application */ + public LoadBalancerList loadBalancers(ApplicationId application) { + return loadBalancers((id) -> id.application().equals(application)); + } + + private LoadBalancerList loadBalancers(Predicate<LoadBalancerId> predicate) { + return LoadBalancerList.copyOf(db.readLoadBalancers(predicate).values()); } public List<Node> getNodes(ApplicationId id, Node.State ... inState) { return db.getNodes(id, inState); } @@ -204,7 +217,7 @@ public class NodeRepository extends AbstractComponent { /** * Returns the ACL for the node (trusted nodes, networks and ports) */ - private NodeAcl getNodeAcl(Node node, NodeList candidates, LoadBalancerList loadBalancers) { + private NodeAcl getNodeAcl(Node node, NodeList candidates) { Set<Node> trustedNodes = new TreeSet<>(Comparator.comparing(Node::hostname)); Set<Integer> trustedPorts = new LinkedHashSet<>(); Set<String> trustedNetworks = new LinkedHashSet<>(); @@ -221,10 +234,10 @@ public class NodeRepository extends AbstractComponent { candidates.parentOf(node).ifPresent(trustedNodes::add); node.allocation().ifPresent(allocation -> { trustedNodes.addAll(candidates.owner(allocation.owner()).asList()); - loadBalancers.owner(allocation.owner()).asList().stream() - .map(LoadBalancer::instance) - .map(LoadBalancerInstance::networks) - .forEach(trustedNetworks::addAll); + loadBalancers(allocation.owner()).asList().stream() + .map(LoadBalancer::instance) + .map(LoadBalancerInstance::networks) + .forEach(trustedNetworks::addAll); }); switch (node.type()) { @@ -293,13 +306,12 @@ public class NodeRepository extends AbstractComponent { */ public List<NodeAcl> getNodeAcls(Node node, boolean children) { NodeList candidates = list(); - LoadBalancerList loadBalancers = loadBalancers(); if (children) { return candidates.childrenOf(node).asList().stream() - .map(childNode -> getNodeAcl(childNode, candidates, loadBalancers)) + .map(childNode -> getNodeAcl(childNode, candidates)) .collect(Collectors.collectingAndThen(Collectors.toList(), Collections::unmodifiableList)); } - return Collections.singletonList(getNodeAcl(node, candidates, loadBalancers)); + return Collections.singletonList(getNodeAcl(node, candidates)); } public NodeFlavors getAvailableFlavors() { diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/lb/LoadBalancerList.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/lb/LoadBalancerList.java index 014d3df8d9a..bad16bf7d12 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/lb/LoadBalancerList.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/lb/LoadBalancerList.java @@ -1,9 +1,6 @@ -// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.provision.lb; -import com.yahoo.config.provision.ApplicationId; - -import java.time.Instant; import java.util.Collection; import java.util.Iterator; import java.util.List; @@ -24,21 +21,11 @@ public class LoadBalancerList implements Iterable<LoadBalancer> { this.loadBalancers = List.copyOf(Objects.requireNonNull(loadBalancers, "loadBalancers must be non-null")); } - /** Returns the subset of load balancers owned by given application */ - public LoadBalancerList owner(ApplicationId application) { - return of(loadBalancers.stream().filter(lb -> lb.id().application().equals(application))); - } - /** Returns the subset of load balancers that are in given state */ public LoadBalancerList in(LoadBalancer.State state) { return of(loadBalancers.stream().filter(lb -> lb.state() == state)); } - /** Returns the subset of load balancers that last changed before given instant */ - public LoadBalancerList changedBefore(Instant instant) { - return of(loadBalancers.stream().filter(lb -> lb.changedAt().isBefore(instant))); - } - public List<LoadBalancer> asList() { return loadBalancers; } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/LoadBalancerExpirer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/LoadBalancerExpirer.java index bcb0c901f14..0f363991310 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/LoadBalancerExpirer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/LoadBalancerExpirer.java @@ -15,6 +15,8 @@ import java.util.ArrayList; import java.util.LinkedHashSet; import java.util.List; import java.util.Objects; +import java.util.concurrent.atomic.AtomicReference; +import java.util.function.Consumer; import java.util.stream.Collectors; /** @@ -51,37 +53,31 @@ public class LoadBalancerExpirer extends Maintainer { /** Move reserved load balancer that have expired to inactive */ private void expireReserved() { - try (var lock = db.lockLoadBalancers()) { - var now = nodeRepository().clock().instant(); - var expirationTime = now.minus(reservedExpiry); - var expired = nodeRepository().loadBalancers() - .in(State.reserved) - .changedBefore(expirationTime); - expired.forEach(lb -> db.writeLoadBalancer(lb.with(State.inactive, now))); - } + var now = nodeRepository().clock().instant(); + withLoadBalancersIn(State.reserved, lb -> { + var gracePeriod = now.minus(reservedExpiry); + if (!lb.changedAt().isBefore(gracePeriod)) return; // Should not move to inactive yet + db.writeLoadBalancer(lb.with(State.inactive, now)); + }); } /** Deprovision inactive load balancers that have expired */ private void removeInactive() { var failed = new ArrayList<LoadBalancerId>(); - Exception lastException = null; - try (var lock = db.lockLoadBalancers()) { - var now = nodeRepository().clock().instant(); - var expirationTime = now.minus(inactiveExpiry); - var expired = nodeRepository().loadBalancers() - .in(State.inactive) - .changedBefore(expirationTime); - for (var lb : expired) { - if (!allocatedNodes(lb.id()).isEmpty()) continue; // Defer removal if there are still nodes allocated to application - try { - service.remove(lb.id().application(), lb.id().cluster()); - db.removeLoadBalancer(lb.id()); - } catch (Exception e) { - failed.add(lb.id()); - lastException = e; - } + var lastException = new AtomicReference<Exception>(); + var now = nodeRepository().clock().instant(); + withLoadBalancersIn(State.inactive, lb -> { + var gracePeriod = now.minus(inactiveExpiry); + if (!lb.changedAt().isBefore(gracePeriod)) return; // Should not be removed yet + if (!allocatedNodes(lb.id()).isEmpty()) return; // Still has nodes, do not remove + try { + service.remove(lb.id().application(), lb.id().cluster()); + db.removeLoadBalancer(lb.id()); + } catch (Exception e){ + failed.add(lb.id()); + lastException.set(e); } - } + }); if (!failed.isEmpty()) { log.log(LogLevel.WARNING, String.format("Failed to remove %d load balancers: %s, retrying in %s", failed.size(), @@ -89,30 +85,27 @@ public class LoadBalancerExpirer extends Maintainer { .map(LoadBalancerId::serializedForm) .collect(Collectors.joining(", ")), interval()), - lastException); + lastException.get()); } } /** Remove reals from inactive load balancers */ private void pruneReals() { var failed = new ArrayList<LoadBalancerId>(); - Exception lastException = null; - try (var lock = db.lockLoadBalancers()) { - var deactivated = nodeRepository().loadBalancers().in(State.inactive); - for (var lb : deactivated) { - var allocatedNodes = allocatedNodes(lb.id()).stream().map(Node::hostname).collect(Collectors.toSet()); - var reals = new LinkedHashSet<>(lb.instance().reals()); - // Remove any real no longer allocated to this application - reals.removeIf(real -> !allocatedNodes.contains(real.hostname().value())); - try { - service.create(lb.id().application(), lb.id().cluster(), reals, true); - db.writeLoadBalancer(lb.with(lb.instance().withReals(reals))); - } catch (Exception e) { - failed.add(lb.id()); - lastException = e; - } + var lastException = new AtomicReference<Exception>(); + withLoadBalancersIn(State.inactive, lb -> { + var allocatedNodes = allocatedNodes(lb.id()).stream().map(Node::hostname).collect(Collectors.toSet()); + var reals = new LinkedHashSet<>(lb.instance().reals()); + // Remove any real no longer allocated to this application + reals.removeIf(real -> !allocatedNodes.contains(real.hostname().value())); + try { + service.create(lb.id().application(), lb.id().cluster(), reals, true); + db.writeLoadBalancer(lb.with(lb.instance().withReals(reals))); + } catch (Exception e) { + failed.add(lb.id()); + lastException.set(e); } - } + }); if (!failed.isEmpty()) { log.log(LogLevel.WARNING, String.format("Failed to remove reals from %d load balancers: %s, retrying in %s", failed.size(), @@ -120,7 +113,21 @@ public class LoadBalancerExpirer extends Maintainer { .map(LoadBalancerId::serializedForm) .collect(Collectors.joining(", ")), interval()), - lastException); + lastException.get()); + } + } + + /** Apply operation to all load balancers that exist in given state, while holding lock */ + private void withLoadBalancersIn(LoadBalancer.State state, Consumer<LoadBalancer> operation) { + try (var legacyLock = db.lockLoadBalancers()) { + for (var id : db.readLoadBalancerIds()) { + try (var lock = db.lockLoadBalancers(id.application())) { + var loadBalancer = db.readLoadBalancer(id); + if (loadBalancer.isEmpty()) continue; // Load balancer was removed during loop + if (loadBalancer.get().state() != state) continue; // Wrong state + operation.accept(loadBalancer.get()); + } + } } } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/CuratorDatabaseClient.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/CuratorDatabaseClient.java index ce0bcc2e337..0a8575578ce 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/CuratorDatabaseClient.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/CuratorDatabaseClient.java @@ -1,4 +1,4 @@ -// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.provision.persistence; import com.google.common.util.concurrent.UncheckedTimeoutException; @@ -38,6 +38,7 @@ import java.util.Optional; import java.util.Set; import java.util.TreeMap; import java.util.function.Function; +import java.util.function.Predicate; import java.util.logging.Level; import java.util.logging.Logger; import java.util.stream.Collectors; @@ -483,18 +484,16 @@ public class CuratorDatabaseClient { // Load balancers public List<LoadBalancerId> readLoadBalancerIds() { - return curatorDatabase.getChildren(loadBalancersRoot).stream() - .map(LoadBalancerId::fromSerializedForm) - .collect(Collectors.toUnmodifiableList()); + return readLoadBalancerIds((ignored) -> true); } - public Map<LoadBalancerId, LoadBalancer> readLoadBalancers() { - return readLoadBalancerIds().stream() - .map(this::readLoadBalancer) - .filter(Optional::isPresent) - .map(Optional::get) - .collect(collectingAndThen(toMap(LoadBalancer::id, Function.identity()), - Collections::unmodifiableMap)); + public Map<LoadBalancerId, LoadBalancer> readLoadBalancers(Predicate<LoadBalancerId> filter) { + return readLoadBalancerIds(filter).stream() + .map(this::readLoadBalancer) + .filter(Optional::isPresent) + .map(Optional::get) + .collect(collectingAndThen(toMap(LoadBalancer::id, Function.identity()), + Collections::unmodifiableMap)); } public Optional<LoadBalancer> readLoadBalancer(LoadBalancerId id) { @@ -522,14 +521,26 @@ public class CuratorDatabaseClient { transaction.commit(); } + // TODO(mpolden): Remove this and all usages once migration to per-application lock is complete public Lock lockLoadBalancers() { return lock(lockRoot.append("loadBalancersLock"), defaultLockTimeout); } + public Lock lockLoadBalancers(ApplicationId application) { + return lock(lockRoot.append("loadBalancersLock2").append(application.serializedForm()), defaultLockTimeout); + } + private Path loadBalancerPath(LoadBalancerId id) { return loadBalancersRoot.append(id.serializedForm()); } + private List<LoadBalancerId> readLoadBalancerIds(Predicate<LoadBalancerId> predicate) { + return curatorDatabase.getChildren(loadBalancersRoot).stream() + .map(LoadBalancerId::fromSerializedForm) + .filter(predicate) + .collect(Collectors.toUnmodifiableList()); + } + private Transaction.Operation createOrSet(Path path, byte[] data) { if (curatorDatabase.exists(path)) { return CuratorOperations.setData(path.getAbsolute(), data); diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/DockerImages.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/DockerImages.java index 9a06f2a980a..4416106f23e 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/DockerImages.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/DockerImages.java @@ -3,9 +3,14 @@ package com.yahoo.vespa.hosted.provision.provisioning; import com.google.common.base.Supplier; import com.google.common.base.Suppliers; +import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.DockerImage; import com.yahoo.config.provision.NodeType; import com.yahoo.vespa.curator.Lock; +import com.yahoo.vespa.flags.FetchVector; +import com.yahoo.vespa.flags.StringFlag; +import com.yahoo.vespa.hosted.provision.Node; +import com.yahoo.vespa.hosted.provision.node.Allocation; import com.yahoo.vespa.hosted.provision.persistence.CuratorDatabaseClient; import java.time.Duration; @@ -13,6 +18,7 @@ import java.util.Collections; import java.util.Map; import java.util.Optional; import java.util.concurrent.TimeUnit; +import java.util.function.Predicate; import java.util.logging.Logger; /** @@ -28,6 +34,7 @@ public class DockerImages { private final CuratorDatabaseClient db; private final DockerImage defaultImage; private final Duration cacheTtl; + private final StringFlag imageOverride; /** * Docker image is read on every request to /nodes/v2/node/[fqdn]. Cache current getDockerImages to avoid @@ -36,20 +43,41 @@ public class DockerImages { */ private volatile Supplier<Map<NodeType, DockerImage>> dockerImages; - public DockerImages(CuratorDatabaseClient db, DockerImage defaultImage) { - this(db, defaultImage, defaultCacheTtl); + public DockerImages(CuratorDatabaseClient db, DockerImage defaultImage, StringFlag imageOverride) { + this(db, defaultImage, defaultCacheTtl, imageOverride); } - DockerImages(CuratorDatabaseClient db, DockerImage defaultImage, Duration cacheTtl) { + DockerImages(CuratorDatabaseClient db, DockerImage defaultImage, Duration cacheTtl, StringFlag imageOverride) { this.db = db; this.defaultImage = defaultImage; this.cacheTtl = cacheTtl; + this.imageOverride = imageOverride; createCache(); } private void createCache() { this.dockerImages = Suppliers.memoizeWithExpiration(() -> Collections.unmodifiableMap(db.readDockerImages()), - cacheTtl.toMillis(), TimeUnit.MILLISECONDS); + cacheTtl.toMillis(), TimeUnit.MILLISECONDS); + } + + /** Returns the image to use for given node and zone */ + public DockerImage dockerImageFor(Node node) { + if (node.type().isDockerHost()) { + // Docker hosts do not run in containers, and thus has no image. Return the image of the child node type + // instead as this allows the host to pre-download the (likely) image its node will run. + // + // Note that if the Docker image has been overridden through feature flag, the preloaded image won't match. + return dockerImageFor(node.type().childNodeType()); + } + return node.allocation() + .map(Allocation::owner) + .map(ApplicationId::serializedForm) + // Return overridden image for this application + .map(application -> imageOverride.with(FetchVector.Dimension.APPLICATION_ID, application).value()) + .filter(Predicate.not(String::isEmpty)) + .map(DockerImage::fromString) + // ... or default Docker image for this node type + .orElseGet(() -> dockerImageFor(node.type())); } /** Returns the current docker images for each node type */ @@ -58,7 +86,7 @@ public class DockerImages { } /** Returns the current docker image for given node type, or default */ - public DockerImage dockerImageFor(NodeType type) { + private DockerImage dockerImageFor(NodeType type) { return getDockerImages().getOrDefault(type, defaultImage); } @@ -69,8 +97,8 @@ public class DockerImages { } try (Lock lock = db.lockDockerImages()) { Map<NodeType, DockerImage> dockerImages = db.readDockerImages(); - - dockerImage.ifPresentOrElse(image -> dockerImages.put(nodeType, image), () -> dockerImages.remove(nodeType)); + dockerImage.ifPresentOrElse(image -> dockerImages.put(nodeType, image), + () -> dockerImages.remove(nodeType)); db.writeDockerImages(dockerImages); createCache(); // Throw away current cache log.info("Set docker image for " + nodeType + " nodes to " + dockerImage.map(DockerImage::asString).orElse(null)); diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/LoadBalancerProvisioner.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/LoadBalancerProvisioner.java index 1500154aa07..d4d5b46dfdf 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/LoadBalancerProvisioner.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/LoadBalancerProvisioner.java @@ -1,4 +1,4 @@ -// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.provision.provisioning; import com.yahoo.config.provision.ApplicationId; @@ -51,10 +51,12 @@ public class LoadBalancerProvisioner { this.db = nodeRepository.database(); this.service = service; // Read and write all load balancers to make sure they are stored in the latest version of the serialization format - try (var lock = db.lockLoadBalancers()) { + try (var legacyLock = db.lockLoadBalancers()) { for (var id : db.readLoadBalancerIds()) { - var loadBalancer = db.readLoadBalancer(id); - loadBalancer.ifPresent(db::writeLoadBalancer); + try (var lock = db.lockLoadBalancers(id.application())) { + var loadBalancer = db.readLoadBalancer(id); + loadBalancer.ifPresent(db::writeLoadBalancer); + } } } } @@ -73,8 +75,10 @@ public class LoadBalancerProvisioner { if (requestedNodes.type() != NodeType.tenant) return; // Nothing to provision for this node type if (!cluster.type().isContainer()) return; // Nothing to provision for this cluster type if (application.instance().isTester()) return; // Do not provision for tester instances - try (var loadBalancersLock = db.lockLoadBalancers()) { - provision(application, cluster.id(), false, loadBalancersLock); + try (var legacyLock = db.lockLoadBalancers()) { + try (var lock = db.lockLoadBalancers(application)) { + provision(application, cluster.id(), false, lock); + } } } @@ -90,15 +94,17 @@ public class LoadBalancerProvisioner { */ public void activate(ApplicationId application, Set<ClusterSpec> clusters, @SuppressWarnings("unused") Mutex applicationLock, NestedTransaction transaction) { - try (var loadBalancersLock = db.lockLoadBalancers()) { - var containerClusters = containerClusterOf(clusters); - for (var clusterId : containerClusters) { - // Provision again to ensure that load balancer instance is re-configured with correct nodes - provision(application, clusterId, true, loadBalancersLock); + try (var legacyLock = db.lockLoadBalancers()) { + try (var lock = db.lockLoadBalancers(application)) { + var containerClusters = containerClusterOf(clusters); + for (var clusterId : containerClusters) { + // Provision again to ensure that load balancer instance is re-configured with correct nodes + provision(application, clusterId, true, lock); + } + // Deactivate any surplus load balancers, i.e. load balancers for clusters that have been removed + var surplusLoadBalancers = surplusLoadBalancersOf(application, containerClusters); + deactivate(surplusLoadBalancers, transaction); } - // Deactivate any surplus load balancers, i.e. load balancers for clusters that have been removed - var surplusLoadBalancers = surplusLoadBalancersOf(application, containerClusters); - deactivate(surplusLoadBalancers, transaction); } } @@ -108,16 +114,17 @@ public class LoadBalancerProvisioner { */ public void deactivate(ApplicationId application, NestedTransaction transaction) { try (var applicationLock = nodeRepository.lock(application)) { - try (Mutex loadBalancersLock = db.lockLoadBalancers()) { - deactivate(nodeRepository.loadBalancers().owner(application).asList(), transaction); + try (var legacyLock = db.lockLoadBalancers()) { + try (var lock = db.lockLoadBalancers(application)) { + deactivate(nodeRepository.loadBalancers(application).asList(), transaction); + } } } } /** Returns load balancers of given application that are no longer referenced by given clusters */ private List<LoadBalancer> surplusLoadBalancersOf(ApplicationId application, Set<ClusterSpec.Id> activeClusters) { - var activeLoadBalancersByCluster = nodeRepository.loadBalancers() - .owner(application) + var activeLoadBalancersByCluster = nodeRepository.loadBalancers(application) .in(LoadBalancer.State.active) .asList() .stream() diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/LoadBalancersResponse.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/LoadBalancersResponse.java index 9f8f4a804d1..3147d4caded 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/LoadBalancersResponse.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/LoadBalancersResponse.java @@ -1,4 +1,4 @@ -// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.provision.restapi.v2; import com.yahoo.config.provision.ApplicationId; @@ -37,10 +37,14 @@ public class LoadBalancersResponse extends HttpResponse { } private List<LoadBalancer> loadBalancers() { - LoadBalancerList loadBalancers = nodeRepository.loadBalancers(); - return application().map(loadBalancers::owner) - .map(LoadBalancerList::asList) - .orElseGet(loadBalancers::asList); + LoadBalancerList loadBalancers; + var application = application(); + if (application.isPresent()) { + loadBalancers = nodeRepository.loadBalancers(application.get()); + } else { + loadBalancers = nodeRepository.loadBalancers(); + } + return loadBalancers.asList(); } @Override diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/NodesResponse.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/NodesResponse.java index 8ca8dfc26f6..7f283452538 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/NodesResponse.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/NodesResponse.java @@ -157,7 +157,7 @@ class NodesResponse extends HttpResponse { toSlime(allocation.membership(), object.setObject("membership")); object.setLong("restartGeneration", allocation.restartGeneration().wanted()); object.setLong("currentRestartGeneration", allocation.restartGeneration().current()); - object.setString("wantedDockerImage", dockerImageFor(node.type()).withTag(allocation.membership().cluster().vespaVersion()).asString()); + object.setString("wantedDockerImage", nodeRepository.dockerImage(node).withTag(allocation.membership().cluster().vespaVersion()).asString()); object.setString("wantedVespaVersion", allocation.membership().cluster().vespaVersion().toFullString()); toSlime(allocation.requestedResources(), object.setObject("requestedResources")); allocation.networkPorts().ifPresent(ports -> NetworkPortsSerializer.toSlime(ports, object.setArray("networkPorts"))); @@ -222,16 +222,10 @@ class NodesResponse extends HttpResponse { // TODO: Remove current + wanted docker image from response for non-docker types private Optional<DockerImage> currentDockerImage(Node node) { return node.status().dockerImage() - .or(() -> Optional.of(node) - .filter(n -> n.flavor().getType() != Flavor.Type.DOCKER_CONTAINER) - .flatMap(n -> n.status().vespaVersion() - .map(version -> dockerImageFor(n.type()).withTag(version)))); - } - - // Docker hosts are not running in an image, but return the image of the node type running on it anyway, - // this allows the docker host to pre-download the (likely) image its node will run - private DockerImage dockerImageFor(NodeType nodeType) { - return nodeRepository.dockerImage(nodeType.isDockerHost() ? nodeType.childNodeType() : nodeType); + .or(() -> Optional.of(node) + .filter(n -> n.flavor().getType() != Flavor.Type.DOCKER_CONTAINER) + .flatMap(n -> n.status().vespaVersion() + .map(version -> nodeRepository.dockerImage(n).withTag(version)))); } private void ipAddressesToSlime(Set<String> ipAddresses, Cursor array) { diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockNodeRepository.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockNodeRepository.java index 1817470a63b..a2579bee0a1 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockNodeRepository.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockNodeRepository.java @@ -54,7 +54,7 @@ public class MockNodeRepository extends NodeRepository { super(flavors, curator, Clock.fixed(Instant.ofEpochMilli(123), ZoneId.of("Z")), Zone.defaultZone(), new MockNameResolver().mockAnyLookup(), DockerImage.fromString("docker-registry.domain.tld:8080/dist/vespa"), - true); + true, new InMemoryFlagSource()); this.flavors = flavors; curator.setZooKeeperEnsembleConnectionSpec("cfg1:1234,cfg2:1234,cfg3:1234"); diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/NodeRepositoryTester.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/NodeRepositoryTester.java index f0f523b9b9b..ab813ddeb5a 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/NodeRepositoryTester.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/NodeRepositoryTester.java @@ -9,6 +9,7 @@ import com.yahoo.config.provision.Zone; import com.yahoo.config.provisioning.FlavorsConfig; import com.yahoo.test.ManualClock; import com.yahoo.vespa.curator.mock.MockCurator; +import com.yahoo.vespa.flags.InMemoryFlagSource; import com.yahoo.vespa.hosted.provision.node.Agent; import com.yahoo.vespa.hosted.provision.provisioning.FlavorConfigBuilder; import com.yahoo.vespa.hosted.provision.testutils.MockNameResolver; @@ -27,9 +28,12 @@ public class NodeRepositoryTester { private final NodeRepository nodeRepository; private final Clock clock; private final MockCurator curator; - - + public NodeRepositoryTester() { + this(new InMemoryFlagSource()); + } + + public NodeRepositoryTester(InMemoryFlagSource flagSource) { nodeFlavors = new NodeFlavors(createConfig()); clock = new ManualClock(); curator = new MockCurator(); @@ -37,7 +41,7 @@ public class NodeRepositoryTester { nodeRepository = new NodeRepository(nodeFlavors, curator, clock, Zone.defaultZone(), new MockNameResolver().mockAnyLookup(), DockerImage.fromString("docker-registry.domain.tld:8080/dist/vespa"), - true); + true, flagSource); } public NodeRepository nodeRepository() { return nodeRepository; } diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/CapacityCheckerTester.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/CapacityCheckerTester.java index afac44856c9..96236b5fb84 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/CapacityCheckerTester.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/CapacityCheckerTester.java @@ -22,6 +22,7 @@ import com.yahoo.config.provision.Zone; import com.yahoo.test.ManualClock; import com.yahoo.vespa.curator.Curator; import com.yahoo.vespa.curator.mock.MockCurator; +import com.yahoo.vespa.flags.InMemoryFlagSource; import com.yahoo.vespa.hosted.provision.Node; import com.yahoo.vespa.hosted.provision.NodeRepository; import com.yahoo.vespa.hosted.provision.node.IP; @@ -54,7 +55,8 @@ public class CapacityCheckerTester { Curator curator = new MockCurator(); NodeFlavors f = new NodeFlavors(new FlavorConfigBuilder().build()); nodeRepository = new NodeRepository(f, curator, clock, zone, new MockNameResolver().mockAnyLookup(), - DockerImage.fromString("docker-registry.domain.tld:8080/dist/vespa"), true); + DockerImage.fromString("docker-registry.domain.tld:8080/dist/vespa"), true, + new InMemoryFlagSource()); } private void updateCapacityChecker() { diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/DynamicProvisioningMaintainerTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/DynamicProvisioningMaintainerTest.java index c7a1486a1a4..9dd8de6d306 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/DynamicProvisioningMaintainerTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/DynamicProvisioningMaintainerTest.java @@ -186,7 +186,8 @@ public class DynamicProvisioningMaintainerTest { private final ManualClock clock = new ManualClock(); private final NodeRepository nodeRepository = new NodeRepository( - nodeFlavors, new MockCurator(), clock, Zone.defaultZone(), new MockNameResolver().mockAnyLookup(), DockerImage.fromString("docker-image"), true); + nodeFlavors, new MockCurator(), clock, Zone.defaultZone(), new MockNameResolver().mockAnyLookup(), + DockerImage.fromString("docker-image"), true, new InMemoryFlagSource()); Node addNode(String hostname, Optional<String> parentHostname, NodeType nodeType, Node.State state, Optional<ApplicationId> application) { Node node = createNode(hostname, parentHostname, nodeType, state, application); @@ -207,4 +208,4 @@ public class DynamicProvisioningMaintainerTest { state, allocation, History.empty(), nodeType, new Reports(), Optional.empty(), Optional.empty()); } } -}
\ No newline at end of file +} diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/FailedExpirerTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/FailedExpirerTest.java index 8509722b016..c293a3436b8 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/FailedExpirerTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/FailedExpirerTest.java @@ -256,7 +256,7 @@ public class FailedExpirerTest { this.nodeRepository = new NodeRepository(nodeFlavors, curator, clock, zone, new MockNameResolver().mockAnyLookup(), DockerImage.fromString("docker-image"), - true); + true, new InMemoryFlagSource()); this.provisioner = new NodeRepositoryProvisioner(nodeRepository, Zone.defaultZone(), new MockProvisionServiceProvider(), new InMemoryFlagSource()); this.expirer = new FailedExpirer(nodeRepository, zone, clock, Duration.ofMinutes(30)); } diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/LoadBalancerExpirerTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/LoadBalancerExpirerTest.java index 12b48fd7a35..7e8fcddb1ae 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/LoadBalancerExpirerTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/LoadBalancerExpirerTest.java @@ -1,4 +1,4 @@ -// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.provision.maintenance; import com.yahoo.component.Vtag; @@ -32,14 +32,14 @@ import static org.junit.Assert.assertTrue; */ public class LoadBalancerExpirerTest { - private ProvisioningTester tester = new ProvisioningTester.Builder().build(); + private final ProvisioningTester tester = new ProvisioningTester.Builder().build(); @Test public void expire_inactive() { LoadBalancerExpirer expirer = new LoadBalancerExpirer(tester.nodeRepository(), Duration.ofDays(1), tester.loadBalancerService()); - Supplier<Map<LoadBalancerId, LoadBalancer>> loadBalancers = () -> tester.nodeRepository().database().readLoadBalancers(); + Supplier<Map<LoadBalancerId, LoadBalancer>> loadBalancers = () -> tester.nodeRepository().database().readLoadBalancers((ignored) -> true); // Deploy two applications with a total of three load balancers ClusterSpec.Id cluster1 = ClusterSpec.Id.from("qrs"); @@ -67,14 +67,15 @@ public class LoadBalancerExpirerTest { // Expirer prunes reals before expiration time of load balancer itself expirer.maintain(); assertEquals(Set.of(), tester.loadBalancerService().instances().get(lb1).reals()); - assertEquals(Set.of(), tester.nodeRepository().loadBalancers().owner(lb1.application()).asList().get(0).instance().reals()); + assertEquals(Set.of(), loadBalancers.get().get(lb1).instance().reals()); // Expirer defers removal of load balancer until expiration time passes expirer.maintain(); + assertSame(LoadBalancer.State.inactive, loadBalancers.get().get(lb1).state()); assertTrue("Inactive load balancer not removed", tester.loadBalancerService().instances().containsKey(lb1)); // Expirer removes load balancers once expiration time passes - tester.clock().advance(Duration.ofHours(1)); + tester.clock().advance(Duration.ofHours(1).plus(Duration.ofSeconds(1))); expirer.maintain(); assertFalse("Inactive load balancer removed", tester.loadBalancerService().instances().containsKey(lb1)); @@ -85,7 +86,7 @@ public class LoadBalancerExpirerTest { // A single cluster is removed deployApplication(app2, cluster1); expirer.maintain(); - assertEquals(LoadBalancer.State.inactive, loadBalancers.get().get(lb3).state()); + assertSame(LoadBalancer.State.inactive, loadBalancers.get().get(lb3).state()); // Expirer defers removal while nodes are still allocated to cluster expirer.maintain(); @@ -93,7 +94,7 @@ public class LoadBalancerExpirerTest { dirtyNodesOf(app2, cluster2); // Expirer removes load balancer for removed cluster - tester.clock().advance(Duration.ofHours(1)); + tester.clock().advance(Duration.ofHours(1).plus(Duration.ofSeconds(1))); expirer.maintain(); assertFalse("Inactive load balancer removed", tester.loadBalancerService().instances().containsKey(lb3)); } @@ -103,7 +104,7 @@ public class LoadBalancerExpirerTest { LoadBalancerExpirer expirer = new LoadBalancerExpirer(tester.nodeRepository(), Duration.ofDays(1), tester.loadBalancerService()); - Supplier<Map<LoadBalancerId, LoadBalancer>> loadBalancers = () -> tester.nodeRepository().database().readLoadBalancers(); + Supplier<Map<LoadBalancerId, LoadBalancer>> loadBalancers = () -> tester.nodeRepository().database().readLoadBalancers((ignored) -> true); // Prepare application @@ -121,7 +122,7 @@ public class LoadBalancerExpirerTest { // Application never activates and nodes are dirtied. Expirer moves load balancer to inactive after timeout dirtyNodesOf(app, cluster); - tester.clock().advance(Duration.ofHours(1)); + tester.clock().advance(Duration.ofHours(1).plus(Duration.ofSeconds(1))); expirer.maintain(); assertSame(LoadBalancer.State.inactive, loadBalancers.get().get(lb).state()); @@ -130,7 +131,7 @@ public class LoadBalancerExpirerTest { assertSame(LoadBalancer.State.inactive, loadBalancers.get().get(lb).state()); // Expirer removes inactive load balancer - tester.clock().advance(Duration.ofHours(1)); + tester.clock().advance(Duration.ofHours(1).plus(Duration.ofSeconds(1))); expirer.maintain(); assertFalse("Inactive load balancer removed", loadBalancers.get().containsKey(lb)); } diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/MaintenanceTester.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/MaintenanceTester.java index a4b66d3cf9e..246f2509397 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/MaintenanceTester.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/MaintenanceTester.java @@ -9,6 +9,7 @@ import com.yahoo.config.provision.RegionName; import com.yahoo.config.provision.Zone; import com.yahoo.test.ManualClock; import com.yahoo.vespa.curator.mock.MockCurator; +import com.yahoo.vespa.flags.InMemoryFlagSource; import com.yahoo.vespa.hosted.provision.Node; import com.yahoo.vespa.hosted.provision.NodeRepository; import com.yahoo.vespa.hosted.provision.node.Agent; @@ -35,7 +36,7 @@ public class MaintenanceTester { public final NodeRepository nodeRepository = new NodeRepository(nodeFlavors, curator, clock, zone, new MockNameResolver().mockAnyLookup(), DockerImage.fromString("docker-registry.domain.tld:8080/dist/vespa"), - true); + true, new InMemoryFlagSource()); public MaintenanceTester() { curator.setZooKeeperEnsembleConnectionSpec("zk1.host:1,zk2.host:2,zk3.host:3"); diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/MetricsReporterTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/MetricsReporterTest.java index 672709a2f8f..321812497bd 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/MetricsReporterTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/MetricsReporterTest.java @@ -13,6 +13,7 @@ import com.yahoo.jdisc.Metric; import com.yahoo.test.ManualClock; import com.yahoo.vespa.curator.Curator; import com.yahoo.vespa.curator.mock.MockCurator; +import com.yahoo.vespa.flags.InMemoryFlagSource; import com.yahoo.vespa.hosted.provision.LockedNodeList; import com.yahoo.vespa.hosted.provision.Node; import com.yahoo.vespa.hosted.provision.NodeRepository; @@ -56,7 +57,7 @@ public class MetricsReporterTest { NodeRepository nodeRepository = new NodeRepository(nodeFlavors, curator, Clock.systemUTC(), Zone.defaultZone(), new MockNameResolver().mockAnyLookup(), DockerImage.fromString("docker-registry.domain.tld:8080/dist/vespa"), - true); + true, new InMemoryFlagSource()); Node node = nodeRepository.createNode("openStackId", "hostname", Optional.empty(), nodeFlavors.getFlavorOrThrow("default"), NodeType.tenant); nodeRepository.addNodes(List.of(node)); Node hostNode = nodeRepository.createNode("openStackId2", "parent", Optional.empty(), nodeFlavors.getFlavorOrThrow("default"), NodeType.proxy); @@ -121,7 +122,7 @@ public class MetricsReporterTest { NodeRepository nodeRepository = new NodeRepository(nodeFlavors, curator, Clock.systemUTC(), Zone.defaultZone(), new MockNameResolver().mockAnyLookup(), DockerImage.fromString("docker-registry.domain.tld:8080/dist/vespa"), - true); + true, new InMemoryFlagSource()); // Allow 4 containers Set<String> ipAddressPool = Set.of("::2", "::3", "::4", "::5"); diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/NodeFailTester.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/NodeFailTester.java index 5872a78e1e2..033ddcd827e 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/NodeFailTester.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/NodeFailTester.java @@ -75,7 +75,8 @@ public class NodeFailTester { clock = new ManualClock(); curator = new MockCurator(); nodeRepository = new NodeRepository(nodeFlavors, curator, clock, zone, new MockNameResolver().mockAnyLookup(), - DockerImage.fromString("docker-registry.domain.tld:8080/dist/vespa"), true); + DockerImage.fromString("docker-registry.domain.tld:8080/dist/vespa"), true, + new InMemoryFlagSource()); provisioner = new NodeRepositoryProvisioner(nodeRepository, zone, new MockProvisionServiceProvider(), new InMemoryFlagSource()); hostLivenessTracker = new TestHostLivenessTracker(clock); orchestrator = new OrchestratorMock(); diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/OperatorChangeApplicationMaintainerTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/OperatorChangeApplicationMaintainerTest.java index 50c00c730bb..22d7f03c449 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/OperatorChangeApplicationMaintainerTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/OperatorChangeApplicationMaintainerTest.java @@ -56,7 +56,7 @@ public class OperatorChangeApplicationMaintainerTest { this.nodeRepository = new NodeRepository(nodeFlavors, curator, clock, zone, new MockNameResolver().mockAnyLookup(), DockerImage.fromString("docker-registry.domain.tld:8080/dist/vespa"), - true); + true, new InMemoryFlagSource()); this.fixture = new Fixture(zone, nodeRepository); createReadyNodes(15, this.fixture.nodeResources, nodeRepository); diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/PeriodicApplicationMaintainerTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/PeriodicApplicationMaintainerTest.java index b1c3b23016c..913b8b53c46 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/PeriodicApplicationMaintainerTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/PeriodicApplicationMaintainerTest.java @@ -62,7 +62,7 @@ public class PeriodicApplicationMaintainerTest { this.nodeRepository = new NodeRepository(nodeFlavors, curator, clock, zone, new MockNameResolver().mockAnyLookup(), DockerImage.fromString("docker-registry.domain.tld:8080/dist/vespa"), - true); + true, new InMemoryFlagSource()); this.fixture = new Fixture(zone, nodeRepository); createReadyNodes(15, fixture.nodeResources, nodeRepository); diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/ReservationExpirerTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/ReservationExpirerTest.java index 11ee6637720..96c3cc09b6b 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/ReservationExpirerTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/ReservationExpirerTest.java @@ -47,7 +47,7 @@ public class ReservationExpirerTest { NodeRepository nodeRepository = new NodeRepository(flavors, curator, clock, Zone.defaultZone(), new MockNameResolver().mockAnyLookup(), DockerImage.fromString("docker-registry.domain.tld:8080/dist/vespa"), - true); + true, new InMemoryFlagSource()); NodeRepositoryProvisioner provisioner = new NodeRepositoryProvisioner(nodeRepository, Zone.defaultZone(), new MockProvisionServiceProvider(), new InMemoryFlagSource()); List<Node> nodes = new ArrayList<>(2); diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/RetiredExpirerTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/RetiredExpirerTest.java index 67af2df36e7..bdbe046fbdf 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/RetiredExpirerTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/RetiredExpirerTest.java @@ -64,8 +64,8 @@ public class RetiredExpirerTest { private final Zone zone = new Zone(Environment.prod, RegionName.from("us-east")); private final NodeFlavors nodeFlavors = FlavorConfigBuilder.createDummies("default"); private final NodeRepository nodeRepository = new NodeRepository(nodeFlavors, curator, clock, zone, - new MockNameResolver().mockAnyLookup(), - DockerImage.fromString("docker-registry.domain.tld:8080/dist/vespa"), true); + new MockNameResolver().mockAnyLookup(), + DockerImage.fromString("docker-registry.domain.tld:8080/dist/vespa"), true, new InMemoryFlagSource()); private final NodeRepositoryProvisioner provisioner = new NodeRepositoryProvisioner(nodeRepository, zone, new MockProvisionServiceProvider(), new InMemoryFlagSource()); private final Orchestrator orchestrator = mock(Orchestrator.class); diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/AclProvisioningTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/AclProvisioningTest.java index 92d066e5f16..1d028f13340 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/AclProvisioningTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/AclProvisioningTest.java @@ -4,8 +4,6 @@ package com.yahoo.vespa.hosted.provision.provisioning; import com.yahoo.component.Version; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.Capacity; -import com.yahoo.config.provision.ClusterSpec; -import com.yahoo.config.provision.HostSpec; import com.yahoo.config.provision.NodeResources; import com.yahoo.config.provision.NodeType; import com.yahoo.vespa.hosted.provision.Node; @@ -40,14 +38,14 @@ public class AclProvisioningTest { tester.makeReadyNodes(10, new NodeResources(1, 4, 10, 1)); List<Node> dockerHost = tester.makeReadyNodes(1, new NodeResources(1, 4, 10, 1), NodeType.host); ApplicationId zoneApplication = tester.makeApplicationId(); - deploy(zoneApplication, Capacity.fromRequiredNodeType(NodeType.host)); + tester.deploy(zoneApplication, Capacity.fromRequiredNodeType(NodeType.host)); tester.makeReadyVirtualDockerNodes(1,new NodeResources(1, 4, 10, 1), dockerHost.get(0).hostname()); List<Node> proxyNodes = tester.makeReadyNodes(3, new NodeResources(1, 4, 10, 1), NodeType.proxy); // Allocate 2 nodes ApplicationId application = tester.makeApplicationId(); - List<Node> activeNodes = deploy(application, Capacity.fromCount(2, new NodeResources(1, 4, 10, 1), false, true)); + List<Node> activeNodes = tester.deploy(application, Capacity.fromCount(2, new NodeResources(1, 4, 10, 1), false, true)); assertEquals(2, activeNodes.size()); // Get trusted nodes for the first active node @@ -112,7 +110,7 @@ public class AclProvisioningTest { // Deploy zone application ApplicationId zoneApplication = tester.makeApplicationId(); - deploy(zoneApplication, Capacity.fromRequiredNodeType(NodeType.proxy)); + tester.deploy(zoneApplication, Capacity.fromRequiredNodeType(NodeType.proxy)); // Get trusted nodes for first proxy node List<Node> proxyNodes = tester.nodeRepository().getNodes(zoneApplication); @@ -154,7 +152,7 @@ public class AclProvisioningTest { // Allocate ApplicationId controllerApplication = tester.makeApplicationId(); - List<Node> controllers = deploy(controllerApplication, Capacity.fromRequiredNodeType(NodeType.controller)); + List<Node> controllers = tester.deploy(controllerApplication, Capacity.fromRequiredNodeType(NodeType.controller)); // Controllers and hosts all trust each other List<NodeAcl> controllerAcls = tester.nodeRepository().getNodeAcls(controllers.get(0), false); @@ -164,12 +162,33 @@ public class AclProvisioningTest { @Test public void trusted_nodes_for_application_with_load_balancer() { - // Populate repo - tester.makeReadyNodes(10, nodeResources); + // Provision hosts and containers + var hosts = tester.makeReadyNodes(2, "default", NodeType.host); + tester.deployZoneApp(); + for (var host : hosts) { + tester.makeReadyVirtualDockerNodes(2, new NodeResources(2, 8, 50, 1), + host.hostname()); + } - // Allocate 2 nodes - List<Node> activeNodes = deploy(2); + // Deploy application + var application = tester.makeApplicationId(); + List<Node> activeNodes = deploy(application, 2); assertEquals(2, activeNodes.size()); + + // Load balancer is allocated to application + var loadBalancers = tester.nodeRepository().loadBalancers(application); + assertEquals(1, loadBalancers.asList().size()); + var lbNetworks = loadBalancers.asList().get(0).instance().networks(); + assertEquals(2, lbNetworks.size()); + + // ACL for nodes with allocation trust their respective load balancer networks, if any + for (var host : hosts) { + var acls = tester.nodeRepository().getNodeAcls(host, true); + assertEquals(2, acls.size()); + assertEquals(Set.of(), acls.get(0).trustedNetworks()); + assertEquals(application, acls.get(1).node().allocation().get().owner()); + assertEquals(lbNetworks, acls.get(1).trustedNetworks()); + } } @Test @@ -191,15 +210,7 @@ public class AclProvisioningTest { } private List<Node> deploy(ApplicationId application, int nodeCount) { - return deploy(application, Capacity.fromCount(nodeCount, nodeResources)); - } - - private List<Node> deploy(ApplicationId application, Capacity capacity) { - ClusterSpec cluster = ClusterSpec.request(ClusterSpec.Type.container, ClusterSpec.Id.from("test"), - Version.fromString("6.42"), false); - List<HostSpec> prepared = tester.prepare(application, cluster, capacity, 1); - tester.activate(application, Set.copyOf(prepared)); - return tester.getNodes(application, Node.State.active).asList(); + return tester.deploy(application, Capacity.fromCount(nodeCount, nodeResources)); } private static void assertAcls(List<List<Node>> expected, NodeAcl actual) { diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DockerImagesTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DockerImagesTest.java new file mode 100644 index 00000000000..70a57715d13 --- /dev/null +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DockerImagesTest.java @@ -0,0 +1,51 @@ +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.provision.provisioning; + +import com.yahoo.config.provision.Capacity; +import com.yahoo.config.provision.DockerImage; +import com.yahoo.config.provision.NodeResources; +import com.yahoo.config.provision.NodeType; +import com.yahoo.vespa.flags.Flags; +import com.yahoo.vespa.flags.InMemoryFlagSource; +import org.junit.Test; + +import static org.junit.Assert.assertEquals; + +/** + * @author mpolden + */ +public class DockerImagesTest { + + @Test + public void image_selection() { + var flagSource = new InMemoryFlagSource(); + var tester = new ProvisioningTester.Builder().flagSource(flagSource).build(); + + // Host uses tenant default image (for preload purposes) + var defaultImage = DockerImage.fromString("docker-registry.domain.tld:8080/dist/vespa"); + var hosts = tester.makeReadyNodes(2, "default", NodeType.host); + tester.deployZoneApp(); + for (var host : hosts) { + assertEquals(defaultImage, tester.nodeRepository().dockerImages().dockerImageFor(host)); + } + + // Tenant node uses tenant default image + var resources = new NodeResources(2, 8, 50, 1); + for (var host : hosts) { + var nodes = tester.makeReadyVirtualDockerNodes(2, resources, host.hostname()); + for (var node : nodes) { + assertEquals(defaultImage, tester.nodeRepository().dockerImages().dockerImageFor(node)); + } + } + + // Allocated containers uses overridden image when feature flag is set + var app = tester.makeApplicationId(); + var nodes = tester.deploy(app, Capacity.fromCount(2, resources)); + var customImage = DockerImage.fromString("docker.example.com/vespa/hosted"); + flagSource.withStringFlag(Flags.DOCKER_IMAGE_OVERRIDE.id(), customImage.asString()); + for (var node : nodes) { + assertEquals(customImage, tester.nodeRepository().dockerImages().dockerImageFor(node)); + } + } + +} diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/LoadBalancerProvisionerTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/LoadBalancerProvisionerTest.java index a26e802dfe8..ee9a582c4db 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/LoadBalancerProvisionerTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/LoadBalancerProvisionerTest.java @@ -1,4 +1,4 @@ -// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.provision.provisioning; import com.google.common.collect.Iterators; @@ -40,15 +40,14 @@ public class LoadBalancerProvisionerTest { private final ApplicationId app1 = ApplicationId.from("tenant1", "application1", "default"); private final ApplicationId app2 = ApplicationId.from("tenant2", "application2", "default"); - private final ApplicationId infraApp1 = ApplicationId.from("vespa", "tenant-host", "default"); - private ProvisioningTester tester = new ProvisioningTester.Builder().build(); + private final ProvisioningTester tester = new ProvisioningTester.Builder().build(); @Test public void provision_load_balancer() { - Supplier<List<LoadBalancer>> lbApp1 = () -> tester.nodeRepository().loadBalancers().owner(app1).asList(); - Supplier<List<LoadBalancer>> lbApp2 = () -> tester.nodeRepository().loadBalancers().owner(app2).asList(); + Supplier<List<LoadBalancer>> lbApp1 = () -> tester.nodeRepository().loadBalancers(app1).asList(); + Supplier<List<LoadBalancer>> lbApp2 = () -> tester.nodeRepository().loadBalancers(app2).asList(); ClusterSpec.Id containerCluster1 = ClusterSpec.Id.from("qrs1"); ClusterSpec.Id contentCluster = ClusterSpec.Id.from("content"); @@ -80,7 +79,7 @@ public class LoadBalancerProvisionerTest { tester.activate(app1, prepare(app1, clusterRequest(ClusterSpec.Type.container, containerCluster1), clusterRequest(ClusterSpec.Type.content, contentCluster))); - LoadBalancer loadBalancer = tester.nodeRepository().loadBalancers().owner(app1).asList().get(0); + LoadBalancer loadBalancer = tester.nodeRepository().loadBalancers(app1).asList().get(0); assertEquals(2, loadBalancer.instance().reals().size()); assertTrue("Failed node is removed", loadBalancer.instance().reals().stream() .map(Real::hostname) @@ -158,7 +157,7 @@ public class LoadBalancerProvisionerTest { var nodes = prepare(app1, Capacity.fromCount(2, new NodeResources(1, 4, 10, 0.3), false, true), true, clusterRequest(ClusterSpec.Type.container, ClusterSpec.Id.from("qrs"))); - Supplier<LoadBalancer> lb = () -> tester.nodeRepository().loadBalancers().owner(app1).asList().get(0); + Supplier<LoadBalancer> lb = () -> tester.nodeRepository().loadBalancers(app1).asList().get(0); assertTrue("Load balancer provisioned with empty reals", tester.loadBalancerService().instances().get(lb.get().id()).reals().isEmpty()); assignIps(tester.nodeRepository().getNodes(app1)); tester.activate(app1, nodes); @@ -189,7 +188,7 @@ public class LoadBalancerProvisionerTest { clusterRequest(ClusterSpec.Type.container, ClusterSpec.Id.from("tenant-host")))); assertTrue("No load balancer provisioned", tester.loadBalancerService().instances().isEmpty()); - assertEquals(List.of(), tester.nodeRepository().loadBalancers().owner(infraApp1).asList()); + assertEquals(List.of(), tester.nodeRepository().loadBalancers(infraApp1).asList()); } @Test @@ -197,12 +196,12 @@ public class LoadBalancerProvisionerTest { tester.activate(app1, prepare(app1, clusterRequest(ClusterSpec.Type.content, ClusterSpec.Id.from("tenant-host")))); assertTrue("No load balancer provisioned", tester.loadBalancerService().instances().isEmpty()); - assertEquals(List.of(), tester.nodeRepository().loadBalancers().owner(app1).asList()); + assertEquals(List.of(), tester.nodeRepository().loadBalancers(app1).asList()); } @Test public void provision_load_balancer_combined_cluster() { - Supplier<List<LoadBalancer>> lbs = () -> tester.nodeRepository().loadBalancers().owner(app1).asList(); + Supplier<List<LoadBalancer>> lbs = () -> tester.nodeRepository().loadBalancers(app1).asList(); ClusterSpec.Id cluster = ClusterSpec.Id.from("foo"); var nodes = prepare(app1, clusterRequest(ClusterSpec.Type.combined, cluster)); diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTester.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTester.java index 4e63d3cc79f..e464ed07472 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTester.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTester.java @@ -87,7 +87,7 @@ public class ProvisioningTester { this.nodeFlavors = nodeFlavors; this.clock = new ManualClock(); this.nodeRepository = new NodeRepository(nodeFlavors, curator, clock, zone, nameResolver, - DockerImage.fromString("docker-registry.domain.tld:8080/dist/vespa"), true); + DockerImage.fromString("docker-registry.domain.tld:8080/dist/vespa"), true, flagSource); this.orchestrator = orchestrator; ProvisionServiceProvider provisionServiceProvider = new MockProvisionServiceProvider(loadBalancerService, hostProvisioner); this.provisioner = new NodeRepositoryProvisioner(nodeRepository, zone, provisionServiceProvider, flagSource); @@ -418,21 +418,20 @@ public class ProvisioningTester { activate(applicationId, Set.copyOf(list)); } + public List<Node> deploy(ApplicationId application, Capacity capacity) { + ClusterSpec cluster = ClusterSpec.request(ClusterSpec.Type.container, ClusterSpec.Id.from("test"), + Version.fromString("6.42"), false); + List<HostSpec> prepared = prepare(application, cluster, capacity, 1); + activate(application, Set.copyOf(prepared)); + return getNodes(application, Node.State.active).asList(); + } + + /** Returns the hosts from the input list which are not retired */ public List<HostSpec> nonRetired(Collection<HostSpec> hosts) { return hosts.stream().filter(host -> ! host.membership().get().retired()).collect(Collectors.toList()); } - public void assertNumberOfNodesWithFlavor(List<HostSpec> hostSpecs, String flavor, int expectedCount) { - long actualNodesWithFlavor = hostSpecs.stream() - .map(HostSpec::hostname) - .map(this::getNodeFlavor) - .map(Flavor::name) - .filter(name -> name.equals(flavor)) - .count(); - assertEquals(expectedCount, actualNodesWithFlavor); - } - public void assertAllocatedOn(String explanation, String hostFlavor, ApplicationId app) { for (Node node : nodeRepository.getNodes(app)) { Node parent = nodeRepository.getNode(node.parentHostname().get()).get(); @@ -440,17 +439,6 @@ public class ProvisioningTester { } } - public void printFreeResources() { - for (Node host : nodeRepository().getNodes(NodeType.host)) { - NodeResources free = host.flavor().resources(); - for (Node child : nodeRepository().getNodes(NodeType.tenant)) { - if (child.parentHostname().get().equals(host.hostname())) - free = free.subtract(child.flavor().resources()); - } - System.out.println(host.flavor().name() + " node. Free resources: " + free); - } - } - public int hostFlavorCount(String hostFlavor, ApplicationId app) { return (int)nodeRepository().getNodes(app).stream() .map(n -> nodeRepository().getNode(n.parentHostname().get()).get()) @@ -458,10 +446,6 @@ public class ProvisioningTester { .count(); } - private Flavor getNodeFlavor(String hostname) { - return nodeRepository.getNode(hostname).map(Node::flavor).orElseThrow(() -> new RuntimeException("No flavor for host " + hostname)); - } - public static final class Builder { private Curator curator; private FlavorsConfig flavorsConfig; diff --git a/persistence/src/vespa/persistence/spi/abstractpersistenceprovider.cpp b/persistence/src/vespa/persistence/spi/abstractpersistenceprovider.cpp index 7f6bd835cd5..e35a6a74bde 100644 --- a/persistence/src/vespa/persistence/spi/abstractpersistenceprovider.cpp +++ b/persistence/src/vespa/persistence/spi/abstractpersistenceprovider.cpp @@ -6,9 +6,7 @@ #include <vespa/document/fieldset/fieldsets.h> #include <vespa/document/fieldvalue/document.h> -namespace storage { - -namespace spi { +namespace storage::spi { UpdateResult AbstractPersistenceProvider::update(const Bucket& bucket, Timestamp ts, @@ -66,5 +64,3 @@ AbstractPersistenceProvider::move(const Bucket& source, PartitionId target, Cont } } - -} diff --git a/persistence/src/vespa/persistence/spi/abstractpersistenceprovider.h b/persistence/src/vespa/persistence/spi/abstractpersistenceprovider.h index 27197c2adb8..557a9ec2edd 100644 --- a/persistence/src/vespa/persistence/spi/abstractpersistenceprovider.h +++ b/persistence/src/vespa/persistence/spi/abstractpersistenceprovider.h @@ -1,7 +1,7 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #pragma once -#include <vespa/persistence/spi/persistenceprovider.h> +#include "persistenceprovider.h" namespace storage::spi { diff --git a/persistence/src/vespa/persistence/spi/bucket.cpp b/persistence/src/vespa/persistence/spi/bucket.cpp index 9265f995a45..ef94519cdb0 100644 --- a/persistence/src/vespa/persistence/spi/bucket.cpp +++ b/persistence/src/vespa/persistence/spi/bucket.cpp @@ -4,8 +4,7 @@ #include <ostream> #include <vespa/vespalib/stllike/asciistream.h> -namespace storage { -namespace spi { +namespace storage::spi { vespalib::string Bucket::toString() const { @@ -30,5 +29,4 @@ operator<<(std::ostream& os, const Bucket& bucket) { return os << bucket.toString(); } -} // spi -} // storage +} diff --git a/persistence/src/vespa/persistence/spi/bucket.h b/persistence/src/vespa/persistence/spi/bucket.h index 54760304694..874074d7e24 100644 --- a/persistence/src/vespa/persistence/spi/bucket.h +++ b/persistence/src/vespa/persistence/spi/bucket.h @@ -17,8 +17,7 @@ #include <persistence/spi/types.h> #include <vespa/document/bucket/bucket.h> -namespace storage { -namespace spi { +namespace storage::spi { class Bucket { document::Bucket _bucket; @@ -47,6 +46,4 @@ public: vespalib::asciistream& operator<<(vespalib::asciistream& out, const Bucket& bucket); std::ostream& operator<<(std::ostream& out, const Bucket& bucket); -} // spi -} // storage - +} diff --git a/persistence/src/vespa/persistence/spi/bucketinfo.cpp b/persistence/src/vespa/persistence/spi/bucketinfo.cpp index e60d6152058..5bef59a65f1 100644 --- a/persistence/src/vespa/persistence/spi/bucketinfo.cpp +++ b/persistence/src/vespa/persistence/spi/bucketinfo.cpp @@ -3,8 +3,7 @@ #include "bucketinfo.h" #include <vespa/vespalib/stllike/asciistream.h> -namespace storage { -namespace spi { +namespace storage::spi { BucketInfo::BucketInfo() : _checksum(0), @@ -73,5 +72,4 @@ std::ostream& operator<<(std::ostream& out, const BucketInfo& info) { return out << info.toString(); } -} // spi -} // storage +} diff --git a/persistence/src/vespa/persistence/spi/bucketinfo.h b/persistence/src/vespa/persistence/spi/bucketinfo.h index fd4605229f8..827aad48d7f 100644 --- a/persistence/src/vespa/persistence/spi/bucketinfo.h +++ b/persistence/src/vespa/persistence/spi/bucketinfo.h @@ -8,9 +8,7 @@ #include <persistence/spi/types.h> -namespace vespalib { - class asciistream; -} +namespace vespalib { class asciistream; } namespace storage::spi { diff --git a/persistence/src/vespa/persistence/spi/clusterstate.cpp b/persistence/src/vespa/persistence/spi/clusterstate.cpp index bc30197978f..567d7ebc1ce 100644 --- a/persistence/src/vespa/persistence/spi/clusterstate.cpp +++ b/persistence/src/vespa/persistence/spi/clusterstate.cpp @@ -12,9 +12,9 @@ namespace storage::spi { ClusterState::ClusterState(const lib::ClusterState& state, uint16_t nodeIndex, const lib::Distribution& distribution) - : _state(new lib::ClusterState(state)), + : _state(std::make_unique<lib::ClusterState>(state)), _nodeIndex(nodeIndex), - _distribution(new lib::Distribution(distribution.serialize())) + _distribution(std::make_unique<lib::Distribution>(distribution.serialize())) { } @@ -26,8 +26,8 @@ void ClusterState::deserialize(vespalib::nbostream& i) { i >> _nodeIndex; i >> distribution; - _state.reset(new lib::ClusterState(clusterState)); - _distribution.reset(new lib::Distribution(distribution)); + _state = std::make_unique<lib::ClusterState>(clusterState); + _distribution = std::make_unique<lib::Distribution>(distribution); } ClusterState::ClusterState(vespalib::nbostream& i) { diff --git a/persistence/src/vespa/persistence/spi/clusterstateimpl.h b/persistence/src/vespa/persistence/spi/clusterstateimpl.h deleted file mode 100644 index 281c37eb9d5..00000000000 --- a/persistence/src/vespa/persistence/spi/clusterstateimpl.h +++ /dev/null @@ -1,66 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#pragma once - -#include <vespa/persistence/spi/bucket.h> -#include <vespa/persistence/spi/clusterstate.h> - -namespace storage { - -namespace spi { - -/** - * Used to determine the state of the current node and its buckets. - */ -class ClusterStateImpl : public ClusterState{ -public: - ClusterStateImpl(); - - ClusterStateImpl(const lib::ClusterState& state, - uint16_t nodeIndex, - const lib::Distribution& distribution); - - ClusterStateImpl(vespalib::nbostream& i); - - ClusterStateImpl(const ClusterStateImpl& other); - - ClusterStateImpl& operator=(const ClusterStateImpl& other); - - /** - * Returns true if the given bucket is in the ideal state - * for readiness. - * - * @param b The bucket to check. - */ - bool shouldBeReady(const Bucket& b) const; - - /** - * Returns false if the cluster has been deemed down. This can happen - * if the fleet controller has detected that too many nodes are down - * compared to the complete list of nodes, and deigns the system to be - * unusable. - */ - bool clusterUp() const; - - /** - * Returns false if this node has been set in a state where it should not - * receive external load. - */ - bool nodeUp() const; - - /** - * Returns a serialized form of this object. - */ - void serialize(vespalib::nbostream& o) const; - -private: - std::unique_ptr<lib::ClusterState> _state; - uint16_t _nodeIndex; - std::unique_ptr<lib::Distribution> _distribution; - - void deserialize(vespalib::nbostream&); -}; - -} - -} - diff --git a/persistence/src/vespa/persistence/spi/context.cpp b/persistence/src/vespa/persistence/spi/context.cpp index 5ce34d5d139..429e2fb9d4e 100644 --- a/persistence/src/vespa/persistence/spi/context.cpp +++ b/persistence/src/vespa/persistence/spi/context.cpp @@ -2,8 +2,7 @@ #include "context.h" -namespace storage { -namespace spi { +namespace storage::spi { Context::Context(const LoadType& loadType, Priority pri, int maxTraceLevel) : _loadType(&loadType), @@ -12,7 +11,6 @@ Context::Context(const LoadType& loadType, Priority pri, int maxTraceLevel) _readConsistency(ReadConsistency::STRONG) { } -Context::~Context() { } +Context::~Context() = default; -} // spi -} // storage +} diff --git a/persistence/src/vespa/persistence/spi/context.h b/persistence/src/vespa/persistence/spi/context.h index ca4c79e3005..8c31439ee75 100644 --- a/persistence/src/vespa/persistence/spi/context.h +++ b/persistence/src/vespa/persistence/spi/context.h @@ -29,13 +29,10 @@ #pragma once -#include <persistence/spi/types.h> -#include <vespa/persistence/spi/read_consistency.h> +#include "read_consistency.h" #include <vespa/vespalib/trace/trace.h> -namespace metrics { - class LoadType; -} +namespace metrics { class LoadType; } namespace storage::spi { @@ -62,10 +59,6 @@ public: const LoadType& getLoadType() const { return *_loadType; } Priority getPriority() const { return _priority; } - int getMaxTraceLevel() const { return _trace.getLevel(); } - void addTrace(const vespalib::TraceNode& traceNode) { - _trace.getRoot().addChild(traceNode); - } /** * A read operation might choose to relax its consistency requirements, diff --git a/persistence/src/vespa/persistence/spi/exceptions.cpp b/persistence/src/vespa/persistence/spi/exceptions.cpp index a1b9d57270c..d17c0f90ca0 100644 --- a/persistence/src/vespa/persistence/spi/exceptions.cpp +++ b/persistence/src/vespa/persistence/spi/exceptions.cpp @@ -2,11 +2,8 @@ #include "exceptions.h" -namespace storage { -namespace spi { +namespace storage::spi { VESPA_IMPLEMENT_EXCEPTION(HandledException, vespalib::Exception); -} // spi -} // storage - +} diff --git a/persistence/src/vespa/persistence/spi/exceptions.h b/persistence/src/vespa/persistence/spi/exceptions.h index 1c434fd8f00..e972e304567 100644 --- a/persistence/src/vespa/persistence/spi/exceptions.h +++ b/persistence/src/vespa/persistence/spi/exceptions.h @@ -3,8 +3,7 @@ #include <vespa/vespalib/util/exceptions.h> -namespace storage { -namespace spi { +namespace storage::spi { /** * Exception used where the cause has already been reported to the user, so @@ -16,6 +15,4 @@ namespace spi { */ VESPA_DEFINE_EXCEPTION(HandledException, vespalib::Exception); -} // spi -} // storage - +} diff --git a/persistence/src/vespa/persistence/spi/matcher.h b/persistence/src/vespa/persistence/spi/matcher.h index 02dc6db2261..bf989530f35 100644 --- a/persistence/src/vespa/persistence/spi/matcher.h +++ b/persistence/src/vespa/persistence/spi/matcher.h @@ -8,12 +8,11 @@ #pragma once -#include <vespa/persistence/spi/docentry.h> +#include "docentry.h" #include <persistence/spi/documentsubset.h> #include <persistence/spi/types.h> -namespace storage { -namespace spi { +namespace storage::spi { class Matcher { DocumentSubset _subset; @@ -37,6 +36,4 @@ struct AllMatcher : public Matcher { bool match(const DocEntry&) const { return true; } }; -} // spi -} // storage - +} diff --git a/persistence/src/vespa/persistence/spi/partitionstate.cpp b/persistence/src/vespa/persistence/spi/partitionstate.cpp index 123a82829ef..7cc42742019 100644 --- a/persistence/src/vespa/persistence/spi/partitionstate.cpp +++ b/persistence/src/vespa/persistence/spi/partitionstate.cpp @@ -4,8 +4,7 @@ #include <vespa/vespalib/util/exceptions.h> #include <vespa/vespalib/stllike/asciistream.h> -namespace storage { -namespace spi { +namespace storage::spi { PartitionState::PartitionState() : _state(UP), @@ -21,7 +20,7 @@ PartitionStateList::PartitionStateList(PartitionId::Type partitionCount) : _states(partitionCount) { } -PartitionStateList::~PartitionStateList() { } +PartitionStateList::~PartitionStateList() = default; PartitionState& PartitionStateList::operator[](PartitionId::Type index) @@ -34,5 +33,4 @@ PartitionStateList::operator[](PartitionId::Type index) return _states[index]; } -} // spi -} // storage +} diff --git a/persistence/src/vespa/persistence/spi/partitionstate.h b/persistence/src/vespa/persistence/spi/partitionstate.h index 6296a70b2c1..e5ce24abbe6 100644 --- a/persistence/src/vespa/persistence/spi/partitionstate.h +++ b/persistence/src/vespa/persistence/spi/partitionstate.h @@ -16,8 +16,7 @@ #include <persistence/spi/types.h> -namespace storage { -namespace spi { +namespace storage::spi { struct PartitionState { enum State { UP, DOWN }; @@ -49,6 +48,4 @@ public: PartitionId size() const { return PartitionId(_states.size()); } }; -} // spi -} // storage - +} diff --git a/persistence/src/vespa/persistence/spi/persistenceprovider.cpp b/persistence/src/vespa/persistence/spi/persistenceprovider.cpp index ec71928baba..61d141c0229 100644 --- a/persistence/src/vespa/persistence/spi/persistenceprovider.cpp +++ b/persistence/src/vespa/persistence/spi/persistenceprovider.cpp @@ -2,11 +2,8 @@ #include "persistenceprovider.h" -namespace storage { -namespace spi { +namespace storage::spi { -PersistenceProvider::~PersistenceProvider() { } - -} // spi -} // storage +PersistenceProvider::~PersistenceProvider() = default; +} diff --git a/persistence/src/vespa/persistence/spi/persistenceprovider.h b/persistence/src/vespa/persistence/spi/persistenceprovider.h index 96b3d385b87..857630e2606 100644 --- a/persistence/src/vespa/persistence/spi/persistenceprovider.h +++ b/persistence/src/vespa/persistence/spi/persistenceprovider.h @@ -11,9 +11,7 @@ #include "selection.h" #include "clusterstate.h" -namespace document { - class FieldSet; -} +namespace document { class FieldSet; } namespace storage::spi { diff --git a/persistence/src/vespa/persistence/spi/providerfactory.h b/persistence/src/vespa/persistence/spi/providerfactory.h index 6b4f0b20ae9..8be143851dd 100644 --- a/persistence/src/vespa/persistence/spi/providerfactory.h +++ b/persistence/src/vespa/persistence/spi/providerfactory.h @@ -8,14 +8,11 @@ #pragma once -#include <vespa/persistence/spi/persistenceprovider.h> +#include "persistenceprovider.h" -namespace document { - class DocumentTypeRepo; -} +namespace document { class DocumentTypeRepo; } -namespace storage { -namespace spi { +namespace storage::spi { struct ProviderFactory { virtual ~ProviderFactory() {} diff --git a/persistence/src/vespa/persistence/spi/read_consistency.cpp b/persistence/src/vespa/persistence/spi/read_consistency.cpp index cad15a5263d..c3b55e5a261 100644 --- a/persistence/src/vespa/persistence/spi/read_consistency.cpp +++ b/persistence/src/vespa/persistence/spi/read_consistency.cpp @@ -1,12 +1,11 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "read_consistency.h" -#include <iostream> +#include <ostream> #include <vespa/log/log.h> LOG_SETUP(".persistence.spi.read_consistency"); -namespace storage { -namespace spi { +namespace storage::spi { std::ostream& operator<<(std::ostream& os, ReadConsistency consistency) @@ -24,6 +23,4 @@ operator<<(std::ostream& os, ReadConsistency consistency) return os; } -} // spi -} // storage - +} diff --git a/persistence/src/vespa/persistence/spi/read_consistency.h b/persistence/src/vespa/persistence/spi/read_consistency.h index 507e05027cc..d90c43af407 100644 --- a/persistence/src/vespa/persistence/spi/read_consistency.h +++ b/persistence/src/vespa/persistence/spi/read_consistency.h @@ -2,10 +2,9 @@ #pragma once #include <iosfwd> -#include <stdint.h> +#include <cstdint> -namespace storage { -namespace spi { +namespace storage::spi { enum class ReadConsistency : uint8_t { /** @@ -28,9 +27,7 @@ enum class ReadConsistency : uint8_t { WEAK }; -std::ostream& -operator<<(std::ostream&, ReadConsistency); +std::ostream& operator<<(std::ostream&, ReadConsistency); -} // spi -} // storage +} diff --git a/persistence/src/vespa/persistence/spi/result.cpp b/persistence/src/vespa/persistence/spi/result.cpp index 024f5595102..f5131f08b1f 100644 --- a/persistence/src/vespa/persistence/spi/result.cpp +++ b/persistence/src/vespa/persistence/spi/result.cpp @@ -9,7 +9,7 @@ namespace storage::spi { Result::Result(const Result &) = default; Result & Result::operator = (const Result &) = default; -Result::~Result() { } +Result::~Result() = default; vespalib::string Result::toString() const { @@ -33,10 +33,10 @@ GetResult::GetResult(Document::UP doc, Timestamp timestamp) _doc(std::move(doc)) { } -GetResult::~GetResult() { } -BucketIdListResult::~BucketIdListResult() { } +GetResult::~GetResult() = default; +BucketIdListResult::~BucketIdListResult() = default; -IterateResult::~IterateResult() { } +IterateResult::~IterateResult() = default; } diff --git a/persistence/src/vespa/persistence/spi/selection.cpp b/persistence/src/vespa/persistence/spi/selection.cpp index 0cd50446488..c7ed98b9d92 100644 --- a/persistence/src/vespa/persistence/spi/selection.cpp +++ b/persistence/src/vespa/persistence/spi/selection.cpp @@ -2,8 +2,7 @@ #include "selection.h" -namespace storage { -namespace spi { +namespace storage::spi { Selection::Selection(const DocumentSelection& docSel) : _documentSelection(docSel), @@ -12,8 +11,7 @@ Selection::Selection(const DocumentSelection& docSel) _timestampSubset() { } -Selection::~Selection() { } +Selection::~Selection() = default; -} // spi -} // storage +} diff --git a/persistence/src/vespa/persistence/spi/selection.h b/persistence/src/vespa/persistence/spi/selection.h index 5c562cabd34..0f809cf1641 100644 --- a/persistence/src/vespa/persistence/spi/selection.h +++ b/persistence/src/vespa/persistence/spi/selection.h @@ -10,12 +10,7 @@ #include "documentselection.h" -namespace storage { -namespace spi { - -class MetaData { - Timestamp timestamp; -}; +namespace storage::spi { class Selection { public: @@ -67,13 +62,6 @@ public: Timestamp getFromTimestamp() const { return _fromTimestamp; } Timestamp getToTimestamp() const { return _toTimestamp; } - - /** - * Regular usage. - */ - bool match(const Document& doc, const MetaData& metaData) const; }; -} // spi -} // storage - +} diff --git a/persistencetypes/src/persistence/spi/types.cpp b/persistencetypes/src/persistence/spi/types.cpp index 00aa95b9707..e746f02e869 100644 --- a/persistencetypes/src/persistence/spi/types.cpp +++ b/persistencetypes/src/persistence/spi/types.cpp @@ -1,11 +1,8 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "types.h" - #include <vespa/vespalib/objects/nbostream.h> -namespace storage { - -namespace spi { +namespace storage::spi { DEFINE_PRIMITIVE_WRAPPER_NBOSTREAM(NodeIndex); DEFINE_PRIMITIVE_WRAPPER_NBOSTREAM(PartitionId); @@ -14,5 +11,3 @@ DEFINE_PRIMITIVE_WRAPPER_NBOSTREAM(Timestamp); DEFINE_PRIMITIVE_WRAPPER_NBOSTREAM(BucketChecksum); } - -} diff --git a/processing/abi-spec.json b/processing/abi-spec.json index 78058f1a8b7..8f77672faec 100644 --- a/processing/abi-spec.json +++ b/processing/abi-spec.json @@ -329,6 +329,10 @@ "public final void set(java.lang.String, java.lang.Object, java.util.Map)", "public final void set(com.yahoo.processing.request.CompoundName, java.lang.Object)", "public final void set(java.lang.String, java.lang.Object)", + "public void clearAll(com.yahoo.processing.request.CompoundName, java.util.Map)", + "public final void clearAll(java.lang.String, java.lang.Object, java.util.Map)", + "public final void clearAll(com.yahoo.processing.request.CompoundName)", + "public final void clearAll(java.lang.String)", "public final boolean getBoolean(com.yahoo.processing.request.CompoundName)", "public final boolean getBoolean(java.lang.String)", "public final boolean getBoolean(com.yahoo.processing.request.CompoundName, boolean)", diff --git a/processing/src/main/java/com/yahoo/processing/request/Properties.java b/processing/src/main/java/com/yahoo/processing/request/Properties.java index 095a0e51ce0..cadc658417b 100644 --- a/processing/src/main/java/com/yahoo/processing/request/Properties.java +++ b/processing/src/main/java/com/yahoo/processing/request/Properties.java @@ -206,8 +206,8 @@ public class Properties implements Cloneable { * This default implementation forwards to the chained instance or throws * a RuntimeException if there is not chained instance. * - * @param name the name of the value - * @param value the value to set. Setting a name to null explicitly is legal. + * @param name the name of the property + * @param value the value to set. Setting a property to null clears it. * @param context the context used to resolve where the values should be set, or null if none * @throws RuntimeException if no instance in the chain accepted this name-value pair */ @@ -223,8 +223,8 @@ public class Properties implements Cloneable { * This default implementation forwards to the chained instance or throws * a RuntimeException if there is not chained instance. * - * @param name the name of the value - * @param value the value to set. Setting a name to null explicitly is legal. + * @param name the name of the property + * @param value the value to set. Setting a property to null clears it. * @param context the context used to resolve where the values should be set, or null if none * @throws RuntimeException if no instance in the chain accepted this name-value pair */ @@ -235,8 +235,8 @@ public class Properties implements Cloneable { /** * Sets a value to the first chained instance which accepts it by calling set(name,value,null). * - * @param name the name of the value - * @param value the value to set. Setting a name to null explicitly is legal. + * @param name the name of the property + * @param value the value to set. Setting a property to null clears it. * @throws RuntimeException if no instance in the chain accepted this name-value pair */ public final void set(CompoundName name, Object value) { @@ -246,8 +246,8 @@ public class Properties implements Cloneable { /** * Sets a value to the first chained instance which accepts it by calling set(name,value,null). * - * @param name the name of the value - * @param value the value to set. Setting a name to null explicitly is legal. + * @param name the name of the property + * @param value the value to set. Setting a property to null clears it. * @throws RuntimeException if no instance in the chain accepted this name-value pair */ public final void set(String name, Object value) { @@ -255,6 +255,56 @@ public class Properties implements Cloneable { } /** + * Sets all properties having this name as a compound prefix to null. + * I.e clearAll("a") will clear the value of "a" and "a.b" but not "ab". + * This default implementation forwards to the chained instance or throws + * a RuntimeException if there is not chained instance. + * + * @param name the compound prefix of the properties to clear + * @param context the context used to resolve where the values should be cleared, or null if none + * @throws RuntimeException if no instance in the chain accepted this name-value pair + */ + public void clearAll(CompoundName name, Map<String, String> context) { + if (chained == null) throw new RuntimeException("Property '" + name + + "' was not accepted in this property chain"); + chained.clearAll(name, context); + } + + /** + * Sets all properties having this name as a compound prefix to null. + * I.e clearAll("a") will clear the value of "a" and "a.b" but not "ab". + * + * @param name the compound prefix of the properties to clear + * @param context the context used to resolve where the values should be cleared, or null if none + * @throws RuntimeException if no instance in the chain accepted this name-value pair + */ + public final void clearAll(String name, Object value, Map<String, String> context) { + set(new CompoundName(name), value, context); + } + + /** + * Sets all properties having this name as a compound prefix to null. + * I.e clearAll("a") will clear the value of "a" and "a.b" but not "ab". + * + * @param name the compound prefix of the properties to clear + * @throws RuntimeException if no instance in the chain accepted this name-value pair + */ + public final void clearAll(CompoundName name) { + clearAll(name, null); + } + + /** + * Sets all properties having this name as a compound prefix to null. + * I.e clearAll("a") will clear the value of "a" and "a.b" but not "ab". + * + * @param name the compound prefix of the properties to clear + * @throws RuntimeException if no instance in the chain accepted this name-value pair + */ + public final void clearAll(String name) { + clearAll(new CompoundName(name), Collections.<String,String>emptyMap()); + } + + /** * Gets a property as a boolean - if this value can reasonably be interpreted as a boolean, this will return * the value. Returns false if this property is null. */ @@ -571,14 +621,14 @@ public class Properties implements Cloneable { } } - /** - * Clones a map by deep cloning each value which is cloneable and shallow copying all other values. - */ + /** Clones a map by deep cloning each value which is cloneable and shallow copying all other values. */ public static Map<CompoundName, Object> cloneMap(Map<CompoundName, Object> map) { return cloneHelper.cloneMap(map); } + /** Clones this object if it is clonable, and the clone is public. Returns null if not */ public static Object clone(Object object) { return cloneHelper.clone(object); } + } diff --git a/searchcore/src/apps/tests/persistenceconformance_test.cpp b/searchcore/src/apps/tests/persistenceconformance_test.cpp index ba0d2047b88..4056e49e37c 100644 --- a/searchcore/src/apps/tests/persistenceconformance_test.cpp +++ b/searchcore/src/apps/tests/persistenceconformance_test.cpp @@ -331,7 +331,7 @@ public: LOG(info, "putHandler(%s)", itr->first.toString().c_str()); IPersistenceHandler::SP proxy( new PersistenceHandlerProxy(itr->second)); - putHandler(itr->second->getBucketSpace(), itr->first, proxy); + putHandler(getWLock(), itr->second->getBucketSpace(), itr->first, proxy); } } @@ -343,7 +343,7 @@ public: const DocumentDBMap &docDbs = _docDbRepo->getDocDbs(); for (DocumentDBMap::const_iterator itr = docDbs.begin(); itr != docDbs.end(); ++itr) { - IPersistenceHandler::SP proxy(removeHandler(itr->second->getBucketSpace(), itr->first)); + IPersistenceHandler::SP proxy(removeHandler(getWLock(), itr->second->getBucketSpace(), itr->first)); } } diff --git a/searchcore/src/tests/proton/attribute/attributes_state_explorer/attributes_state_explorer_test.cpp b/searchcore/src/tests/proton/attribute/attributes_state_explorer/attributes_state_explorer_test.cpp index da5a8b31671..76dcf53c51d 100644 --- a/searchcore/src/tests/proton/attribute/attributes_state_explorer/attributes_state_explorer_test.cpp +++ b/searchcore/src/tests/proton/attribute/attributes_state_explorer/attributes_state_explorer_test.cpp @@ -10,6 +10,7 @@ #include <vespa/searchlib/index/dummyfileheadercontext.h> #include <vespa/searchlib/test/directory_handler.h> #include <vespa/vespalib/test/insertion_operators.h> +#include <vespa/searchlib/attribute/singlenumericattribute.hpp> #include <vespa/log/log.h> LOG_SETUP("attributes_state_explorer_test"); diff --git a/searchcore/src/tests/proton/persistenceengine/persistence_handler_map/persistence_handler_map_test.cpp b/searchcore/src/tests/proton/persistenceengine/persistence_handler_map/persistence_handler_map_test.cpp index ef12b694187..35590cc68f6 100644 --- a/searchcore/src/tests/proton/persistenceengine/persistence_handler_map/persistence_handler_map_test.cpp +++ b/searchcore/src/tests/proton/persistenceengine/persistence_handler_map/persistence_handler_map_test.cpp @@ -46,23 +46,31 @@ DummyPersistenceHandler::SP handler_c(std::make_shared<DummyPersistenceHandler>( DummyPersistenceHandler::SP handler_a_new(std::make_shared<DummyPersistenceHandler>()); + +void +assertHandler(const IPersistenceHandler::SP & lhs, const IPersistenceHandler * rhs) +{ + EXPECT_EQUAL(lhs.get(), rhs); +} + void assertHandler(const IPersistenceHandler::SP &lhs, const IPersistenceHandler::SP &rhs) { EXPECT_EQUAL(lhs.get(), rhs.get()); } +template <typename T> void -assertNullHandler(const IPersistenceHandler::SP &handler) +assertNullHandler(const T & handler) { - EXPECT_TRUE(handler.get() == nullptr); + EXPECT_TRUE(! handler); } void -assertSnapshot(const std::vector<IPersistenceHandler::SP> &exp, const HandlerSnapshot::UP &snapshot) +assertSnapshot(const std::vector<IPersistenceHandler::SP> &exp, HandlerSnapshot snapshot) { - EXPECT_EQUAL(exp.size(), snapshot->size()); - auto &sequence = snapshot->handlers(); + EXPECT_EQUAL(exp.size(), snapshot.size()); + auto &sequence = snapshot.handlers(); for (size_t i = 0; i < exp.size() && sequence.valid(); ++i, sequence.next()) { EXPECT_EQUAL(exp[i].get(), sequence.get()); } diff --git a/searchcore/src/tests/proton/persistenceengine/persistenceengine_test.cpp b/searchcore/src/tests/proton/persistenceengine/persistenceengine_test.cpp index 569b36a425d..19d9d41c3e4 100644 --- a/searchcore/src/tests/proton/persistenceengine/persistenceengine_test.cpp +++ b/searchcore/src/tests/proton/persistenceengine/persistenceengine_test.cpp @@ -404,8 +404,8 @@ struct SimpleFixture { engine(_owner, _writeFilter, -1, false), hset() { - engine.putHandler(makeBucketSpace(), DocTypeName(doc1->getType()), hset.phandler1); - engine.putHandler(bucketSpace2, DocTypeName(doc2->getType()), hset.phandler2); + engine.putHandler(engine.getWLock(), makeBucketSpace(), DocTypeName(doc1->getType()), hset.phandler1); + engine.putHandler(engine.getWLock(), bucketSpace2, DocTypeName(doc2->getType()), hset.phandler2); } SimpleFixture() : SimpleFixture(makeBucketSpace()) diff --git a/searchcore/src/vespa/searchcore/proton/bucketdb/bucketdb.cpp b/searchcore/src/vespa/searchcore/proton/bucketdb/bucketdb.cpp index 3ced0f509b5..e80543368d4 100644 --- a/searchcore/src/vespa/searchcore/proton/bucketdb/bucketdb.cpp +++ b/searchcore/src/vespa/searchcore/proton/bucketdb/bucketdb.cpp @@ -119,6 +119,19 @@ BucketDB::cachedGet(const BucketId &bucketId) const return get(bucketId); } +storage::spi::BucketInfo +BucketDB::cachedGetBucketInfo(const BucketId &bucketId) const +{ + if (isCachedBucket(bucketId)) { + return _cachedBucketState; + } + auto itr = _map.find(bucketId); + if (itr != _map.end()) { + return itr->second; + } + return BucketState(); +} + bool BucketDB::hasBucket(const BucketId &bucketId) const { diff --git a/searchcore/src/vespa/searchcore/proton/bucketdb/bucketdb.h b/searchcore/src/vespa/searchcore/proton/bucketdb/bucketdb.h index e64cc53d4a2..05388931e20 100644 --- a/searchcore/src/vespa/searchcore/proton/bucketdb/bucketdb.h +++ b/searchcore/src/vespa/searchcore/proton/bucketdb/bucketdb.h @@ -51,6 +51,7 @@ public: void cacheBucket(const BucketId &bucketId); void uncacheBucket(); bool isCachedBucket(const BucketId &bucketId) const; + storage::spi::BucketInfo cachedGetBucketInfo(const BucketId &bucketId) const; BucketState cachedGet(const BucketId &bucketId) const; bool hasBucket(const BucketId &bucketId) const; void getBuckets(BucketId::List & buckets) const; diff --git a/searchcore/src/vespa/searchcore/proton/common/handlermap.hpp b/searchcore/src/vespa/searchcore/proton/common/handlermap.hpp index bddbfed371f..3fb32a9d1e0 100644 --- a/searchcore/src/vespa/searchcore/proton/common/handlermap.hpp +++ b/searchcore/src/vespa/searchcore/proton/common/handlermap.hpp @@ -5,7 +5,6 @@ #include <vespa/vespalib/util/exceptions.h> #include <vespa/vespalib/util/sequence.h> #include <vespa/vespalib/stllike/string.h> -#include <vespa/vespalib/stllike/hash_map.h> #include <map> #include <vector> @@ -18,9 +17,8 @@ namespace proton { template <typename T> class HandlerMap { private: - typedef typename std::shared_ptr<T> HandlerSP; - typedef std::map<DocTypeName, HandlerSP> StdMap; - typedef typename StdMap::iterator MapIterator; + using HandlerSP = typename std::shared_ptr<T>; + using StdMap = std::map<DocTypeName, HandlerSP>; StdMap _handlers; @@ -40,8 +38,7 @@ public: size_t _offset; public: - typedef std::unique_ptr<Snapshot> UP; - + Snapshot() : _handlers(), _offset(0) { } Snapshot(const StdMap &map) : _handlers(), _offset(0) { _handlers.reserve(map.size()); for (auto itr : map) { @@ -49,6 +46,11 @@ public: } } Snapshot(std::vector<HandlerSP> &&handlers) : _handlers(std::move(handlers)), _offset(0) {} + Snapshot(Snapshot &&) noexcept = default; + Snapshot & operator = (Snapshot &&) noexcept = default; + Snapshot(const Snapshot &) = delete; + Snapshot & operator = (const Snapshot &) = delete; + bool valid() const override { return (_offset < _handlers.size()); } T *get() const override { return _handlers[_offset].get(); } HandlerSP getSP() const { return _handlers[_offset]; } @@ -64,11 +66,7 @@ public: /** * Constructs a new instance of this class. */ - HandlerMap() - : _handlers() - { - // empty - } + HandlerMap() = default; /** * Registers a new handler for the given document type. If another handler @@ -83,7 +81,7 @@ public: putHandler(const DocTypeName &docTypeNameVer, const HandlerSP &handler) { - if (handler.get() == NULL) { + if ( ! handler) { throw vespalib::IllegalArgumentException(vespalib::make_string( "Handler is null for docType '%s'", docTypeNameVer.toString().c_str())); @@ -115,6 +113,20 @@ public: return HandlerSP(); } + /** + * Returns the handler for the given document type. If no handler was + * registered, this method returns a null pointer. + * + * @param docType The document type whose handler to return. + * @return The registered handler, if any. + */ + T * + getHandlerPtr(const DocTypeName &docTypeNameVer) const + { + const_iterator it = _handlers.find(docTypeNameVer); + return (it != _handlers.end()) ? it->second.get() : nullptr; + } + bool hasHandler(const HandlerSP &handler) const { bool found = false; for (const auto &kv : _handlers) { @@ -148,11 +160,7 @@ public: /** * Clear all handlers. */ - void - clear() - { - _handlers.clear(); - } + void clear() { _handlers.clear(); } /** * Create a snapshot of the handlers currently contained in this @@ -161,68 +169,15 @@ public: * * @return handler sequence **/ - std::unique_ptr<Snapshot> - snapshot() const - { - return std::unique_ptr<Snapshot>(new Snapshot(_handlers)); - } + Snapshot snapshot() const { return Snapshot(_handlers); } // we want to use snapshots rather than direct iteration to reduce locking; // the below functions should be deprecated when possible. - /** - * Returns a bidirectional iterator that points at the first element of the - * sequence (or just beyond the end of an empty sequence). - * - * @return The beginning of this map. - */ - iterator - begin() - { - return _handlers.begin(); - } - - /** - * Returns a const bidirectional iterator that points at the first element - * of the sequence (or just beyond the end of an empty sequence). - * - * @return The beginning of this map. - */ - const_iterator - begin() const - { - return _handlers.begin(); - } - - /** - * Returns a bidirectional iterator that points just beyond the end of the - * sequence. - * - * @return The end of this map. - */ - iterator - end() - { - return _handlers.end(); - } - - /** - * Returns a const bidirectional iterator that points just beyond the end of - * the sequence. - * - * @return The end of this map. - */ - const_iterator - end() const - { - return _handlers.end(); - } - - /** - * Returns the number of handlers in this map. - * - * @return the number of handlers. - */ + iterator begin() { return _handlers.begin(); } + const_iterator begin() const { return _handlers.begin(); } + iterator end() { return _handlers.end(); } + const_iterator end() const { return _handlers.end(); } size_t size() const { return _handlers.size(); } }; diff --git a/searchcore/src/vespa/searchcore/proton/matchengine/matchengine.cpp b/searchcore/src/vespa/searchcore/proton/matchengine/matchengine.cpp index 480359f8382..b170f60d71f 100644 --- a/searchcore/src/vespa/searchcore/proton/matchengine/matchengine.cpp +++ b/searchcore/src/vespa/searchcore/proton/matchengine/matchengine.cpp @@ -128,13 +128,13 @@ MatchEngine::performSearch(search::engine::SearchRequest::Source req, if (searchHandler) { ret = searchHandler->match(searchHandler, *searchRequest, *threadBundle); } else { - HandlerMap<ISearchHandler>::Snapshot::UP snapshot; + HandlerMap<ISearchHandler>::Snapshot snapshot; { std::lock_guard<std::mutex> guard(_lock); snapshot = _handlers.snapshot(); } - if (snapshot->valid()) { - ISearchHandler::SP handler = snapshot->getSP(); + if (snapshot.valid()) { + ISearchHandler::SP handler = snapshot.getSP(); ret = handler->match(handler, *searchRequest, *threadBundle); // use the first handler } } diff --git a/searchcore/src/vespa/searchcore/proton/metrics/metrics_engine.cpp b/searchcore/src/vespa/searchcore/proton/metrics/metrics_engine.cpp index d34ec8d05a8..bf97bdda29a 100644 --- a/searchcore/src/vespa/searchcore/proton/metrics/metrics_engine.cpp +++ b/searchcore/src/vespa/searchcore/proton/metrics/metrics_engine.cpp @@ -106,7 +106,7 @@ void doCleanAttributes(AttributeMetrics &attributes) { auto entries = attributes.release(); - for (const auto entry : entries) { + for (const auto &entry : entries) { attributes.parent()->unregisterMetric(*entry); } } diff --git a/searchcore/src/vespa/searchcore/proton/persistenceengine/document_iterator.cpp b/searchcore/src/vespa/searchcore/proton/persistenceengine/document_iterator.cpp index 1bcbe4e9683..8175e612bd4 100644 --- a/searchcore/src/vespa/searchcore/proton/persistenceengine/document_iterator.cpp +++ b/searchcore/src/vespa/searchcore/proton/persistenceengine/document_iterator.cpp @@ -1,6 +1,8 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "document_iterator.h" +#include <vespa/searchcore/proton/common/cachedselect.h> +#include <vespa/searchcore/proton/common/selectcontext.h> #include <vespa/document/select/gid_filter.h> #include <vespa/document/select/node.h> #include <vespa/document/fieldvalue/document.h> @@ -42,13 +44,6 @@ DocEntry *createDocEntry(Timestamp timestamp, bool removed, Document::UP doc, ss } // namespace proton::<unnamed> bool -DocumentIterator::useDocumentSelection() const -{ - return (!_metaOnly && - !_selection.getDocumentSelection().getDocumentSelection().empty()); -} - -bool DocumentIterator::checkMeta(const search::DocumentMetaData &meta) const { if (!meta.valid()) { diff --git a/searchcore/src/vespa/searchcore/proton/persistenceengine/document_iterator.h b/searchcore/src/vespa/searchcore/proton/persistenceengine/document_iterator.h index 285b3cb2afa..67242e8220f 100644 --- a/searchcore/src/vespa/searchcore/proton/persistenceengine/document_iterator.h +++ b/searchcore/src/vespa/searchcore/proton/persistenceengine/document_iterator.h @@ -3,8 +3,6 @@ #pragma once #include "i_document_retriever.h" -#include <vespa/searchcore/proton/common/cachedselect.h> -#include <vespa/searchcore/proton/common/selectcontext.h> #include <vespa/searchlib/common/idocumentmetastore.h> #include <vespa/persistence/spi/bucket.h> #include <vespa/persistence/spi/selection.h> @@ -32,10 +30,7 @@ private: storage::spi::IterateResult::List _list; - bool useDocumentSelection() const; bool checkMeta(const search::DocumentMetaData &meta) const; - bool checkDoc(const document::Document &doc) const; - bool checkDoc(const SelectContext &sc) const; void fetchCompleteSource(const IDocumentRetriever & source, storage::spi::IterateResult::List & list); bool isWeakRead() const { return _readConsistency == ReadConsistency::WEAK; } diff --git a/searchcore/src/vespa/searchcore/proton/persistenceengine/persistence_handler_map.cpp b/searchcore/src/vespa/searchcore/proton/persistenceengine/persistence_handler_map.cpp index 06969167ab3..5b62a290353 100644 --- a/searchcore/src/vespa/searchcore/proton/persistenceengine/persistence_handler_map.cpp +++ b/searchcore/src/vespa/searchcore/proton/persistenceengine/persistence_handler_map.cpp @@ -20,15 +20,15 @@ PersistenceHandlerMap::putHandler(document::BucketSpace bucketSpace, return _map[bucketSpace].putHandler(docType, handler); } -IPersistenceHandler::SP +IPersistenceHandler * PersistenceHandlerMap::getHandler(document::BucketSpace bucketSpace, const DocTypeName &docType) const { auto itr = _map.find(bucketSpace); if (itr != _map.end()) { - return itr->second.getHandler(docType); + return itr->second.getHandlerPtr(docType); } - return IPersistenceHandler::SP(); + return nullptr; } IPersistenceHandler::SP @@ -42,7 +42,7 @@ PersistenceHandlerMap::removeHandler(document::BucketSpace bucketSpace, return IPersistenceHandler::SP(); } -HandlerSnapshot::UP +HandlerSnapshot PersistenceHandlerMap::getHandlerSnapshot() const { std::vector<IPersistenceHandler::SP> handlers; @@ -52,47 +52,17 @@ PersistenceHandlerMap::getHandlerSnapshot() const } } size_t handlersSize = handlers.size(); - return std::make_unique<HandlerSnapshot> - (std::make_unique<DocTypeToHandlerMap::Snapshot>(std::move(handlers)), - handlersSize); -} - -namespace { - -struct EmptySequence : public vespalib::Sequence<IPersistenceHandler *> { - virtual bool valid() const override { return false; } - virtual IPersistenceHandler *get() const override { return nullptr; } - virtual void next() override { } - static EmptySequence::UP make() { return std::make_unique<EmptySequence>(); } -}; - + return HandlerSnapshot(DocTypeToHandlerMap::Snapshot(std::move(handlers)), handlersSize); } -HandlerSnapshot::UP +HandlerSnapshot PersistenceHandlerMap::getHandlerSnapshot(document::BucketSpace bucketSpace) const { auto itr = _map.find(bucketSpace); if (itr != _map.end()) { - return std::make_unique<HandlerSnapshot>(itr->second.snapshot(), itr->second.size()); + return HandlerSnapshot(itr->second.snapshot(), itr->second.size()); } - return std::make_unique<HandlerSnapshot>(EmptySequence::make(), 0); -} - -namespace { - -class SequenceOfOne : public vespalib::Sequence<IPersistenceHandler *> { -private: - bool _done; - IPersistenceHandler *_value; -public: - SequenceOfOne(IPersistenceHandler *value) : _done(false), _value(value) {} - - virtual bool valid() const override { return !_done; } - virtual IPersistenceHandler *get() const override { return _value; } - virtual void next() override { _done = true; } - static SequenceOfOne::UP make(IPersistenceHandler *value) { return std::make_unique<SequenceOfOne>(value); } -}; - + return HandlerSnapshot(); } } diff --git a/searchcore/src/vespa/searchcore/proton/persistenceengine/persistence_handler_map.h b/searchcore/src/vespa/searchcore/proton/persistenceengine/persistence_handler_map.h index 003c378d6a7..64241e1ad2b 100644 --- a/searchcore/src/vespa/searchcore/proton/persistenceengine/persistence_handler_map.h +++ b/searchcore/src/vespa/searchcore/proton/persistenceengine/persistence_handler_map.h @@ -20,16 +20,17 @@ class IPersistenceHandler; */ class PersistenceHandlerMap { public: - using PersistenceHandlerSequence = vespalib::Sequence<IPersistenceHandler *>; + using DocTypeToHandlerMap = HandlerMap<IPersistenceHandler>; + using PersistenceHandlerSequence = DocTypeToHandlerMap::Snapshot; using PersistenceHandlerSP = std::shared_ptr<IPersistenceHandler>; class HandlerSnapshot { private: - PersistenceHandlerSequence::UP _handlers; - size_t _size; + PersistenceHandlerSequence _handlers; + size_t _size; public: - using UP = std::unique_ptr<HandlerSnapshot>; - HandlerSnapshot(PersistenceHandlerSequence::UP handlers_, size_t size_) + HandlerSnapshot() : _handlers(), _size(0) {} + HandlerSnapshot(DocTypeToHandlerMap::Snapshot handlers_, size_t size_) : _handlers(std::move(handlers_)), _size(size_) {} @@ -37,12 +38,12 @@ public: HandlerSnapshot & operator = (const HandlerSnapshot &) = delete; size_t size() const { return _size; } - PersistenceHandlerSequence &handlers() { return *_handlers; } - static PersistenceHandlerSequence::UP release(HandlerSnapshot &&rhs) { return std::move(rhs._handlers); } + PersistenceHandlerSequence &handlers() { return _handlers; } + static PersistenceHandlerSequence release(HandlerSnapshot &&rhs) { return std::move(rhs._handlers); } }; private: - using DocTypeToHandlerMap = HandlerMap<IPersistenceHandler>; + struct BucketSpaceHash { std::size_t operator() (const document::BucketSpace &bucketSpace) const { return bucketSpace.getId(); } @@ -53,15 +54,11 @@ private: public: PersistenceHandlerMap(); - PersistenceHandlerSP putHandler(document::BucketSpace bucketSpace, - const DocTypeName &docType, - const PersistenceHandlerSP &handler); - PersistenceHandlerSP removeHandler(document::BucketSpace bucketSpace, - const DocTypeName &docType); - PersistenceHandlerSP getHandler(document::BucketSpace bucketSpace, - const DocTypeName &docType) const; - HandlerSnapshot::UP getHandlerSnapshot() const; - HandlerSnapshot::UP getHandlerSnapshot(document::BucketSpace bucketSpace) const; + PersistenceHandlerSP putHandler(document::BucketSpace bucketSpace, const DocTypeName &docType, const PersistenceHandlerSP &handler); + PersistenceHandlerSP removeHandler(document::BucketSpace bucketSpace, const DocTypeName &docType); + IPersistenceHandler * getHandler(document::BucketSpace bucketSpace, const DocTypeName &docType) const; + HandlerSnapshot getHandlerSnapshot() const; + HandlerSnapshot getHandlerSnapshot(document::BucketSpace bucketSpace) const; }; } diff --git a/searchcore/src/vespa/searchcore/proton/persistenceengine/persistenceengine.cpp b/searchcore/src/vespa/searchcore/proton/persistenceengine/persistenceengine.cpp index 1f862b07048..2ede5d45f7e 100644 --- a/searchcore/src/vespa/searchcore/proton/persistenceengine/persistenceengine.cpp +++ b/searchcore/src/vespa/searchcore/proton/persistenceengine/persistenceengine.cpp @@ -40,7 +40,7 @@ protected: std::mutex _lock; vespalib::CountDownLatch _latch; public: - ResultHandlerBase(uint32_t waitCnt); + explicit ResultHandlerBase(uint32_t waitCnt); ~ResultHandlerBase(); void await() { _latch.await(); } }; @@ -55,11 +55,11 @@ class GenericResultHandler : public ResultHandlerBase, public IGenericResultHand private: Result _result; public: - GenericResultHandler(uint32_t waitCnt) : + explicit GenericResultHandler(uint32_t waitCnt) : ResultHandlerBase(waitCnt), _result() { } - ~GenericResultHandler(); + ~GenericResultHandler() override; void handle(const Result &result) override { if (result.hasError()) { std::lock_guard<std::mutex> guard(_lock); @@ -109,7 +109,7 @@ class SynchronizedBucketIdListResultHandler : public ResultHandlerBase, public BucketIdListResultHandler { public: - SynchronizedBucketIdListResultHandler(uint32_t waitCnt) + explicit SynchronizedBucketIdListResultHandler(uint32_t waitCnt) : ResultHandlerBase(waitCnt), BucketIdListResultHandler() { } @@ -163,17 +163,21 @@ BucketInfoResultHandler::~BucketInfoResultHandler() = default; } -PersistenceEngine::HandlerSnapshot::UP -PersistenceEngine::getHandlerSnapshot() const +PersistenceEngine::HandlerSnapshot +PersistenceEngine::getHandlerSnapshot(const WriteGuard &) const { - std::lock_guard<std::mutex> guard(_lock); return _handlers.getHandlerSnapshot(); } -PersistenceEngine::HandlerSnapshot::UP -PersistenceEngine::getHandlerSnapshot(document::BucketSpace bucketSpace) const +PersistenceEngine::HandlerSnapshot +PersistenceEngine::getHandlerSnapshot(const ReadGuard &, document::BucketSpace bucketSpace) const +{ + return _handlers.getHandlerSnapshot(bucketSpace); +} + +PersistenceEngine::HandlerSnapshot +PersistenceEngine::getHandlerSnapshot(const WriteGuard &, document::BucketSpace bucketSpace) const { - std::lock_guard<std::mutex> guard(_lock); return _handlers.getHandlerSnapshot(bucketSpace); } @@ -202,27 +206,23 @@ PersistenceEngine::~PersistenceEngine() IPersistenceHandler::SP -PersistenceEngine::putHandler(document::BucketSpace bucketSpace, const DocTypeName &docType, - const IPersistenceHandler::SP &handler) +PersistenceEngine::putHandler(const WriteGuard &, document::BucketSpace bucketSpace, const DocTypeName &docType,const IPersistenceHandler::SP &handler) { - std::lock_guard<std::mutex> guard(_lock); return _handlers.putHandler(bucketSpace, docType, handler); } -IPersistenceHandler::SP -PersistenceEngine::getHandler(document::BucketSpace bucketSpace, const DocTypeName &docType) const +IPersistenceHandler * +PersistenceEngine::getHandler(const ReadGuard &, document::BucketSpace bucketSpace, const DocTypeName &docType) const { - std::lock_guard<std::mutex> guard(_lock); return _handlers.getHandler(bucketSpace, docType); } IPersistenceHandler::SP -PersistenceEngine::removeHandler(document::BucketSpace bucketSpace, const DocTypeName &docType) +PersistenceEngine::removeHandler(const WriteGuard &, document::BucketSpace bucketSpace, const DocTypeName &docType) { // TODO: Grab bucket list and treat them as modified - std::lock_guard<std::mutex> guard(_lock); return _handlers.removeHandler(bucketSpace, docType); } @@ -232,9 +232,9 @@ PersistenceEngine::initialize() { std::unique_lock<std::shared_timed_mutex> wguard(getWLock()); LOG(debug, "Begin initializing persistence handlers"); - HandlerSnapshot::UP snap = getHandlerSnapshot(); - for (; snap->handlers().valid(); snap->handlers().next()) { - IPersistenceHandler *handler = snap->handlers().get(); + HandlerSnapshot snap = getHandlerSnapshot(wguard); + for (; snap.handlers().valid(); snap.handlers().next()) { + IPersistenceHandler *handler = snap.handlers().get(); handler->initialize(); } LOG(debug, "Done initializing persistence handlers"); @@ -260,10 +260,10 @@ PersistenceEngine::listBuckets(BucketSpace bucketSpace, PartitionId id) const BucketIdListResult::List emptyList; return BucketIdListResult(emptyList); } - HandlerSnapshot::UP snap = getHandlerSnapshot(bucketSpace); + HandlerSnapshot snap = getHandlerSnapshot(rguard, bucketSpace); BucketIdListResultHandler resultHandler; - for (; snap->handlers().valid(); snap->handlers().next()) { - IPersistenceHandler *handler = snap->handlers().get(); + for (; snap.handlers().valid(); snap.handlers().next()) { + IPersistenceHandler *handler = snap.handlers().get(); handler->handleListBuckets(resultHandler); } return resultHandler.getResult(); @@ -275,10 +275,10 @@ PersistenceEngine::setClusterState(BucketSpace bucketSpace, const ClusterState & { std::shared_lock<std::shared_timed_mutex> rguard(_rwMutex); saveClusterState(bucketSpace, calc); - HandlerSnapshot::UP snap = getHandlerSnapshot(bucketSpace); - GenericResultHandler resultHandler(snap->size()); - for (; snap->handlers().valid(); snap->handlers().next()) { - IPersistenceHandler *handler = snap->handlers().get(); + HandlerSnapshot snap = getHandlerSnapshot(rguard, bucketSpace); + GenericResultHandler resultHandler(snap.size()); + for (; snap.handlers().valid(); snap.handlers().next()) { + IPersistenceHandler *handler = snap.handlers().get(); handler->handleSetClusterState(calc, resultHandler); } resultHandler.await(); @@ -292,10 +292,10 @@ PersistenceEngine::setActiveState(const Bucket& bucket, storage::spi::BucketInfo::ActiveState newState) { std::shared_lock<std::shared_timed_mutex> rguard(_rwMutex); - HandlerSnapshot::UP snap = getHandlerSnapshot(bucket.getBucketSpace()); - GenericResultHandler resultHandler(snap->size()); - for (; snap->handlers().valid(); snap->handlers().next()) { - IPersistenceHandler *handler = snap->handlers().get(); + HandlerSnapshot snap = getHandlerSnapshot(rguard, bucket.getBucketSpace()); + GenericResultHandler resultHandler(snap.size()); + for (; snap.handlers().valid(); snap.handlers().next()) { + IPersistenceHandler *handler = snap.handlers().get(); handler->handleSetActiveState(bucket, newState, resultHandler); } resultHandler.await(); @@ -309,10 +309,10 @@ PersistenceEngine::getBucketInfo(const Bucket& b) const // Runs in SPI thread. // No handover to write threads in persistence handlers. std::shared_lock<std::shared_timed_mutex> rguard(_rwMutex); - HandlerSnapshot::UP snap = getHandlerSnapshot(b.getBucketSpace()); + HandlerSnapshot snap = getHandlerSnapshot(rguard, b.getBucketSpace()); BucketInfoResultHandler resultHandler; - for (; snap->handlers().valid(); snap->handlers().next()) { - IPersistenceHandler *handler = snap->handlers().get(); + for (; snap.handlers().valid(); snap.handlers().next()) { + IPersistenceHandler *handler = snap.handlers().get(); handler->handleGetBucketInfo(b, resultHandler); } return resultHandler.getResult(); @@ -338,7 +338,7 @@ PersistenceEngine::put(const Bucket& b, Timestamp t, const document::Document::S return Result(Result::ErrorType::PERMANENT_ERROR, make_string("Old id scheme not supported in elastic mode (%s)", doc->getId().toString().c_str())); } - IPersistenceHandler::SP handler = getHandler(b.getBucketSpace(), docType); + IPersistenceHandler * handler = getHandler(rguard, b.getBucketSpace(), docType); if (!handler) { return Result(Result::ErrorType::PERMANENT_ERROR, make_string("No handler for document type '%s'", docType.toString().c_str())); @@ -360,7 +360,7 @@ PersistenceEngine::remove(const Bucket& b, Timestamp t, const DocumentId& did, C make_string("Old id scheme not supported in elastic mode (%s)", did.toString().c_str())); } DocTypeName docType(did.getDocType()); - IPersistenceHandler::SP handler = getHandler(b.getBucketSpace(), docType); + IPersistenceHandler * handler = getHandler(rguard, b.getBucketSpace(), docType); if (!handler) { return RemoveResult(Result::ErrorType::PERMANENT_ERROR, make_string("No handler for document type '%s'", docType.toString().c_str())); @@ -414,7 +414,7 @@ PersistenceEngine::update(const Bucket& b, Timestamp t, const DocumentUpdate::SP return UpdateResult(Result::ErrorType::PERMANENT_ERROR, make_string("Update operation rejected due to bad id (%s, %s)", upd->getId().toString().c_str(), docType.getName().c_str())); } - IPersistenceHandler::SP handler = getHandler(b.getBucketSpace(), docType); + IPersistenceHandler * handler = getHandler(rguard, b.getBucketSpace(), docType); if (handler) { TransportLatch latch(1); @@ -432,9 +432,9 @@ PersistenceEngine::GetResult PersistenceEngine::get(const Bucket& b, const document::FieldSet& fields, const DocumentId& did, Context& context) const { std::shared_lock<std::shared_timed_mutex> rguard(_rwMutex); - HandlerSnapshot::UP snapshot = getHandlerSnapshot(b.getBucketSpace()); + HandlerSnapshot snapshot = getHandlerSnapshot(rguard, b.getBucketSpace()); - for (PersistenceHandlerSequence & handlers = snapshot->handlers(); handlers.valid(); handlers.next()) { + for (PersistenceHandlerSequence & handlers = snapshot.handlers(); handlers.valid(); handlers.next()) { BucketGuard::UP bucket_guard = handlers.get()->lockBucket(b); IPersistenceHandler::RetrieversSP retrievers = handlers.get()->getDocumentRetrievers(context.getReadConsistency()); for (size_t i = 0; i < retrievers->size(); ++i) { @@ -462,19 +462,19 @@ PersistenceEngine::createIterator(const Bucket &bucket, const document::FieldSet IncludedVersions versions, Context & context) { std::shared_lock<std::shared_timed_mutex> rguard(_rwMutex); - HandlerSnapshot::UP snapshot = getHandlerSnapshot(bucket.getBucketSpace()); + HandlerSnapshot snapshot = getHandlerSnapshot(rguard, bucket.getBucketSpace()); auto entry = std::make_unique<IteratorEntry>(context.getReadConsistency(), bucket, fields, selection, versions, _defaultSerializedSize, _ignoreMaxBytes); - entry->bucket_guards.reserve(snapshot->size()); - for (PersistenceHandlerSequence & handlers = snapshot->handlers(); handlers.valid(); handlers.next()) { + entry->bucket_guards.reserve(snapshot.size()); + for (PersistenceHandlerSequence & handlers = snapshot.handlers(); handlers.valid(); handlers.next()) { entry->bucket_guards.push_back(handlers.get()->lockBucket(bucket)); IPersistenceHandler::RetrieversSP retrievers = handlers.get()->getDocumentRetrievers(context.getReadConsistency()); for (size_t i = 0; i < retrievers->size(); ++i) { entry->it.add((*retrievers)[i]); } } - entry->handler_sequence = HandlerSnapshot::release(std::move(*snapshot)); + entry->handler_sequence = HandlerSnapshot::release(std::move(snapshot)); std::lock_guard<std::mutex> guard(_iterators_lock); static IteratorId id_counter(0); @@ -541,10 +541,10 @@ PersistenceEngine::createBucket(const Bucket &b, Context &) { std::shared_lock<std::shared_timed_mutex> rguard(_rwMutex); LOG(spam, "createBucket(%s)", b.toString().c_str()); - HandlerSnapshot::UP snap = getHandlerSnapshot(b.getBucketSpace()); - TransportLatch latch(snap->size()); - for (; snap->handlers().valid(); snap->handlers().next()) { - IPersistenceHandler *handler = snap->handlers().get(); + HandlerSnapshot snap = getHandlerSnapshot(rguard, b.getBucketSpace()); + TransportLatch latch(snap.size()); + for (; snap.handlers().valid(); snap.handlers().next()) { + IPersistenceHandler *handler = snap.handlers().get(); handler->handleCreateBucket(feedtoken::make(latch), b); } latch.await(); @@ -557,10 +557,10 @@ PersistenceEngine::deleteBucket(const Bucket& b, Context&) { std::shared_lock<std::shared_timed_mutex> rguard(_rwMutex); LOG(spam, "deleteBucket(%s)", b.toString().c_str()); - HandlerSnapshot::UP snap = getHandlerSnapshot(b.getBucketSpace()); - TransportLatch latch(snap->size()); - for (; snap->handlers().valid(); snap->handlers().next()) { - IPersistenceHandler *handler = snap->handlers().get(); + HandlerSnapshot snap = getHandlerSnapshot(rguard, b.getBucketSpace()); + TransportLatch latch(snap.size()); + for (; snap.handlers().valid(); snap.handlers().next()) { + IPersistenceHandler *handler = snap.handlers().get(); handler->handleDeleteBucket(feedtoken::make(latch), b); } latch.await(); @@ -578,10 +578,10 @@ PersistenceEngine::getModifiedBuckets(BucketSpace bucketSpace) const std::lock_guard<std::mutex> guard(_lock); extraModifiedBuckets.swap(_extraModifiedBuckets[bucketSpace]); } - HandlerSnapshot::UP snap = getHandlerSnapshot(bucketSpace); - SynchronizedBucketIdListResultHandler resultHandler(snap->size() + extraModifiedBuckets.size()); - for (; snap->handlers().valid(); snap->handlers().next()) { - IPersistenceHandler *handler = snap->handlers().get(); + HandlerSnapshot snap = getHandlerSnapshot(rguard, bucketSpace); + SynchronizedBucketIdListResultHandler resultHandler(snap.size() + extraModifiedBuckets.size()); + for (; snap.handlers().valid(); snap.handlers().next()) { + IPersistenceHandler *handler = snap.handlers().get(); handler->handleGetModifiedBuckets(resultHandler); } for (const auto & item : extraModifiedBuckets) { @@ -599,10 +599,10 @@ PersistenceEngine::split(const Bucket& source, const Bucket& target1, const Buck LOG(spam, "split(%s, %s, %s)", source.toString().c_str(), target1.toString().c_str(), target2.toString().c_str()); assert(source.getBucketSpace() == target1.getBucketSpace()); assert(source.getBucketSpace() == target2.getBucketSpace()); - HandlerSnapshot::UP snap = getHandlerSnapshot(source.getBucketSpace()); - TransportLatch latch(snap->size()); - for (; snap->handlers().valid(); snap->handlers().next()) { - IPersistenceHandler *handler = snap->handlers().get(); + HandlerSnapshot snap = getHandlerSnapshot(rguard, source.getBucketSpace()); + TransportLatch latch(snap.size()); + for (; snap.handlers().valid(); snap.handlers().next()) { + IPersistenceHandler *handler = snap.handlers().get(); handler->handleSplit(feedtoken::make(latch), source, target1, target2); } latch.await(); @@ -617,10 +617,10 @@ PersistenceEngine::join(const Bucket& source1, const Bucket& source2, const Buck LOG(spam, "join(%s, %s, %s)", source1.toString().c_str(), source2.toString().c_str(), target.toString().c_str()); assert(source1.getBucketSpace() == target.getBucketSpace()); assert(source2.getBucketSpace() == target.getBucketSpace()); - HandlerSnapshot::UP snap = getHandlerSnapshot(target.getBucketSpace()); - TransportLatch latch(snap->size()); - for (; snap->handlers().valid(); snap->handlers().next()) { - IPersistenceHandler *handler = snap->handlers().get(); + HandlerSnapshot snap = getHandlerSnapshot(rguard, target.getBucketSpace()); + TransportLatch latch(snap.size()); + for (; snap.handlers().valid(); snap.handlers().next()) { + IPersistenceHandler *handler = snap.handlers().get(); handler->handleJoin(feedtoken::make(latch), source1, source2, target); } latch.await(); @@ -722,19 +722,18 @@ public: }; void -PersistenceEngine::populateInitialBucketDB(BucketSpace bucketSpace, - IPersistenceHandler &targetHandler) +PersistenceEngine::populateInitialBucketDB(const WriteGuard & guard, BucketSpace bucketSpace, IPersistenceHandler &targetHandler) { - HandlerSnapshot::UP snap = getHandlerSnapshot(bucketSpace); + HandlerSnapshot snap = getHandlerSnapshot(guard, bucketSpace); - size_t snapSize(snap->size()); + size_t snapSize(snap.size()); size_t flawed = 0; // handleListActiveBuckets() runs in SPI thread. // No handover to write threads in persistence handlers. ActiveBucketIdListResultHandler resultHandler; - for (; snap->handlers().valid(); snap->handlers().next()) { - IPersistenceHandler *handler = snap->handlers().get(); + for (; snap.handlers().valid(); snap.handlers().next()) { + IPersistenceHandler *handler = snap.handlers().get(); handler->handleListActiveBuckets(resultHandler); } typedef std::map<document::BucketId, size_t> BucketIdMap; diff --git a/searchcore/src/vespa/searchcore/proton/persistenceengine/persistenceengine.h b/searchcore/src/vespa/searchcore/proton/persistenceengine/persistenceengine.h index a6c696d08fb..5d3be07c532 100644 --- a/searchcore/src/vespa/searchcore/proton/persistenceengine/persistenceengine.h +++ b/searchcore/src/vespa/searchcore/proton/persistenceengine/persistenceengine.h @@ -5,12 +5,9 @@ #include "i_resource_write_filter.h" #include "persistence_handler_map.h" #include "ipersistencehandler.h" -#include <vespa/document/bucket/bucketspace.h> #include <vespa/persistence/spi/abstractpersistenceprovider.h> -#include <vespa/searchcore/proton/common/handlermap.hpp> #include <mutex> #include <shared_mutex> -#include <unordered_map> namespace proton { @@ -18,7 +15,7 @@ class IPersistenceEngineOwner; class PersistenceEngine : public storage::spi::AbstractPersistenceProvider { private: - using PersistenceHandlerSequence = vespalib::Sequence<IPersistenceHandler *>; + using PersistenceHandlerSequence = PersistenceHandlerMap::PersistenceHandlerSequence; using HandlerSnapshot = PersistenceHandlerMap::HandlerSnapshot; using DocumentUpdate = document::DocumentUpdate; using Bucket = storage::spi::Bucket; @@ -43,7 +40,7 @@ private: using UpdateResult = storage::spi::UpdateResult; struct IteratorEntry { - PersistenceHandlerSequence::UP handler_sequence; + PersistenceHandlerSequence handler_sequence; DocumentIterator it; bool in_use; std::vector<BucketGuard::UP> bucket_guards; @@ -75,9 +72,13 @@ private: mutable ExtraModifiedBuckets _extraModifiedBuckets; mutable std::shared_timed_mutex _rwMutex; - IPersistenceHandler::SP getHandler(document::BucketSpace bucketSpace, const DocTypeName &docType) const; - HandlerSnapshot::UP getHandlerSnapshot() const; - HandlerSnapshot::UP getHandlerSnapshot(document::BucketSpace bucketSpace) const; + using ReadGuard = std::shared_lock<std::shared_timed_mutex>; + using WriteGuard = std::unique_lock<std::shared_timed_mutex>; + + IPersistenceHandler * getHandler(const ReadGuard & guard, document::BucketSpace bucketSpace, const DocTypeName &docType) const; + HandlerSnapshot getHandlerSnapshot(const WriteGuard & guard) const; + HandlerSnapshot getHandlerSnapshot(const ReadGuard & guard, document::BucketSpace bucketSpace) const; + HandlerSnapshot getHandlerSnapshot(const WriteGuard & guard, document::BucketSpace bucketSpace) const; void saveClusterState(BucketSpace bucketSpace, const ClusterState &calc); ClusterState::SP savedClusterState(BucketSpace bucketSpace) const; @@ -89,9 +90,8 @@ public: ssize_t defaultSerializedSize, bool ignoreMaxBytes); ~PersistenceEngine() override; - IPersistenceHandler::SP putHandler(document::BucketSpace bucketSpace, const DocTypeName &docType, - const IPersistenceHandler::SP &handler); - IPersistenceHandler::SP removeHandler(document::BucketSpace bucketSpace, const DocTypeName &docType); + IPersistenceHandler::SP putHandler(const WriteGuard &, document::BucketSpace bucketSpace, const DocTypeName &docType, const IPersistenceHandler::SP &handler); + IPersistenceHandler::SP removeHandler(const WriteGuard &, document::BucketSpace bucketSpace, const DocTypeName &docType); // Implements PersistenceProvider Result initialize() override; @@ -121,8 +121,8 @@ public: void destroyIterators(); void propagateSavedClusterState(BucketSpace bucketSpace, IPersistenceHandler &handler); void grabExtraModifiedBuckets(BucketSpace bucketSpace, IPersistenceHandler &handler); - void populateInitialBucketDB(BucketSpace bucketSpace, IPersistenceHandler &targetHandler); - std::unique_lock<std::shared_timed_mutex> getWLock() const; + void populateInitialBucketDB(const WriteGuard & guard, BucketSpace bucketSpace, IPersistenceHandler &targetHandler); + WriteGuard getWLock() const; }; } diff --git a/searchcore/src/vespa/searchcore/proton/server/buckethandler.cpp b/searchcore/src/vespa/searchcore/proton/server/buckethandler.cpp index 1d3b2165c80..57f1ee74e60 100644 --- a/searchcore/src/vespa/searchcore/proton/server/buckethandler.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/buckethandler.cpp @@ -116,7 +116,7 @@ BucketHandler::handleGetBucketInfo(const Bucket &bucket, // Called by SPI thread. // BucketDBOwner ensures synchronization between SPI thread and // master write thread in document database. - BucketInfo bucketInfo = _ready->getBucketDB().takeGuard()->cachedGet(bucket); + BucketInfo bucketInfo = _ready->getBucketDB().takeGuard()->cachedGetBucketInfo(bucket); LOG(spam, "handleGetBucketInfo(%s): %s", bucket.toString().c_str(), bucketInfo.toString().c_str()); resultHandler.handle(BucketInfoResult(bucketInfo)); diff --git a/searchcore/src/vespa/searchcore/proton/server/proton.cpp b/searchcore/src/vespa/searchcore/proton/server/proton.cpp index 4daf3e895af..28de4dff917 100644 --- a/searchcore/src/vespa/searchcore/proton/server/proton.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/proton.cpp @@ -594,10 +594,10 @@ Proton::addDocumentDB(const document::DocumentType &docType, auto persistenceHandler = std::make_shared<PersistenceHandlerProxy>(ret); if (!_isInitializing) { _persistenceEngine->propagateSavedClusterState(bucketSpace, *persistenceHandler); - _persistenceEngine->populateInitialBucketDB(bucketSpace, *persistenceHandler); + _persistenceEngine->populateInitialBucketDB(persistenceWGuard, bucketSpace, *persistenceHandler); } // TODO: Fix race with new cluster state setting. - _persistenceEngine->putHandler(bucketSpace, docTypeName, persistenceHandler); + _persistenceEngine->putHandler(persistenceWGuard, bucketSpace, docTypeName, persistenceHandler); } auto searchHandler = std::make_shared<SearchHandlerProxy>(ret); _summaryEngine->putSearchHandler(docTypeName, searchHandler); @@ -629,7 +629,7 @@ Proton::removeDocumentDB(const DocTypeName &docTypeName) // Not allowed to get to service layer to call pause(). std::unique_lock<std::shared_timed_mutex> persistenceWguard(_persistenceEngine->getWLock()); IPersistenceHandler::SP oldHandler; - oldHandler = _persistenceEngine->removeHandler(old->getBucketSpace(), docTypeName); + oldHandler = _persistenceEngine->removeHandler(persistenceWguard, old->getBucketSpace(), docTypeName); if (_initComplete && oldHandler) { // TODO: Fix race with bucket db modifying ops. _persistenceEngine->grabExtraModifiedBuckets(old->getBucketSpace(), *oldHandler); diff --git a/searchcore/src/vespa/searchcore/proton/summaryengine/summaryengine.cpp b/searchcore/src/vespa/searchcore/proton/summaryengine/summaryengine.cpp index e154c6761e2..692b05899f4 100644 --- a/searchcore/src/vespa/searchcore/proton/summaryengine/summaryengine.cpp +++ b/searchcore/src/vespa/searchcore/proton/summaryengine/summaryengine.cpp @@ -128,13 +128,13 @@ SummaryEngine::getDocsums(DocsumRequest::UP req) if (searchHandler) { reply = searchHandler->getDocsums(*req); } else { - vespalib::Sequence<ISearchHandler*>::UP snapshot; + HandlerMap<ISearchHandler>::Snapshot snapshot; { std::lock_guard<std::mutex> guard(_lock); snapshot = _handlers.snapshot(); } - if (snapshot->valid()) { - reply = snapshot->get()->getDocsums(*req); // use the first handler + if (snapshot.valid()) { + reply = snapshot.get()->getDocsums(*req); // use the first handler } } updateDocsumMetrics(vespalib::to_s(req->getTimeUsed()), getNumDocs(*reply)); diff --git a/searchlib/src/apps/docstore/benchmarkdatastore.cpp b/searchlib/src/apps/docstore/benchmarkdatastore.cpp index 620a139d451..4f8a4ad7345 100644 --- a/searchlib/src/apps/docstore/benchmarkdatastore.cpp +++ b/searchlib/src/apps/docstore/benchmarkdatastore.cpp @@ -9,6 +9,7 @@ #include <vespa/vespalib/data/databuffer.h> #include <vespa/fastos/app.h> #include <unistd.h> +#include <random> #include <vespa/log/log.h> LOG_SETUP("documentstore.benchmark"); @@ -65,17 +66,13 @@ BenchmarkDataStoreApp::Main() void BenchmarkDataStoreApp::read(size_t numReads, size_t perChunk, const IDataStore * dataStore) { vespalib::DataBuffer buf; - struct random_data rstate; - char state[8]; - memset(state, 0, sizeof(state)); - memset(&rstate, 0, sizeof(rstate)); + std::minstd_rand rng; const size_t docIdLimit(dataStore->getDocIdLimit()); assert(docIdLimit > 0); - initstate_r(getpid(), state, sizeof(state), &rstate); - assert(srandom_r(getpid(), &rstate) == 0); + rng.seed(getpid()); int32_t rnd(0); for ( size_t i(0); i < numReads; i++) { - random_r(&rstate, &rnd); + rnd = rng(); uint32_t lid(rnd%docIdLimit); for (uint32_t j(lid); j < std::min(docIdLimit, lid+perChunk); j++) { dataStore->read(j, buf); diff --git a/searchlib/src/tests/aggregator/perdocexpr.cpp b/searchlib/src/tests/aggregator/perdocexpr.cpp index 610fc58e98f..41465f991e3 100644 --- a/searchlib/src/tests/aggregator/perdocexpr.cpp +++ b/searchlib/src/tests/aggregator/perdocexpr.cpp @@ -1209,7 +1209,7 @@ TEST("testArithmeticOperations") { testAdd(createScalarInt(I1), createScalarInt(I2), 3469774562ull, 3469774562ull); testAdd(createScalarInt(I1), createScalarFloat(F2), 1793253251ull, 1793253250.767681239); testAdd(createScalarFloat(F1), createScalarFloat(F2), 11, 10.878668839 ); - testMultiply(createScalarInt(I1), createScalarInt(I2), 3006427292488851361ull, 3006427292488851361ull); + testMultiply(createScalarInt(I1), createScalarInt(I2), 3006427292488851361ull, static_cast<double>(3006427292488851361ull)); testMultiply(createScalarInt(I1), createScalarFloat(F2), 17515926039ull, 1793253241.0*9.767681239); testMultiply(createScalarFloat(F1), createScalarFloat(F2), 11, 10.8517727372816364 ); diff --git a/searchlib/src/tests/expression/attributenode/attribute_node_test.cpp b/searchlib/src/tests/expression/attributenode/attribute_node_test.cpp index 503922bdb13..02d459dd0d8 100644 --- a/searchlib/src/tests/expression/attributenode/attribute_node_test.cpp +++ b/searchlib/src/tests/expression/attributenode/attribute_node_test.cpp @@ -267,7 +267,7 @@ Fixture::assertBools(std::vector<bool> expVals, const vespalib::string &attribut { auto node = makeNode(attributeName, false, preserveAccurateTypes); uint32_t docId = 0; - for (const auto &expDocVal : expVals) { + for (const auto expDocVal : expVals) { ++docId; node->setDocId(docId); node->execute(); diff --git a/searchlib/src/tests/queryeval/multibitvectoriterator/multibitvectoriterator_test.cpp b/searchlib/src/tests/queryeval/multibitvectoriterator/multibitvectoriterator_test.cpp index 28ba13bb4fc..12c38d3fe02 100644 --- a/searchlib/src/tests/queryeval/multibitvectoriterator/multibitvectoriterator_test.cpp +++ b/searchlib/src/tests/queryeval/multibitvectoriterator/multibitvectoriterator_test.cpp @@ -10,6 +10,7 @@ #include <vespa/searchlib/fef/termfieldmatchdata.h> #include <vespa/searchlib/fef/termfieldmatchdataarray.h> #include <vespa/searchlib/test/searchiteratorverifier.h> +#include <random> #include <vespa/log/log.h> LOG_SETUP("multibitvectoriterator_test"); @@ -52,6 +53,14 @@ private: BitVector * getBV(size_t index, bool inverted) { return inverted ? _bvs_inverted[index].get() : _bvs[index].get(); } + void fixup_bitvectors() { + // Restore from inverted bitvectors after tampering + for (int i = 0; i < 3; ++i) { + if (_bvs_inverted[i]->testBit(1)) { + _bvs[i]->clearBit(1); + } + } + } std::vector< BitVector::UP > _bvs; std::vector< BitVector::UP > _bvs_inverted; }; @@ -61,12 +70,12 @@ Test::~Test() = default; void Test::setup() { - srand(7); + std::minstd_rand rnd(341); for(size_t i(0); i < 3; i++) { _bvs.push_back(BitVector::create(10000)); BitVector & bv(*_bvs.back()); for (size_t j(0); j < bv.size(); j++) { - int r = rand(); + int r = rnd(); if (r & 0x1) { bv.setBit(j); } @@ -132,7 +141,7 @@ Test::testAndWith(bool invert) s->initFullRange(); H firstHits3 = seekNoReset(*s, 1, 130); H lastHits3 = seekNoReset(*s, 130, _bvs[0]->size()); - //These constants will change if srand(7) is changed. + //These constants will change if rnd(341) is changed. EXPECT_EQUAL(30u, firstHits2.size()); EXPECT_EQUAL(19u, firstHits3.size()); EXPECT_EQUAL(1234u, lastHits2F.size()); @@ -208,6 +217,7 @@ Test::testBug7163266() EXPECT_TRUE(ms->needUnpack(i)); } EXPECT_TRUE(ms->needUnpack(28)); // NB: force unpack all + fixup_bitvectors(); } template<typename T> @@ -240,6 +250,7 @@ Test::testThatOptimizePreservesUnpack() EXPECT_TRUE(ms != nullptr); EXPECT_EQUAL(2u, ms->getChildren().size()); verifySelectiveUnpack(*s, tfmd); + fixup_bitvectors(); } void diff --git a/security-utils/src/main/java/com/yahoo/security/tls/TransportSecurityOptions.java b/security-utils/src/main/java/com/yahoo/security/tls/TransportSecurityOptions.java index c0e9e1053c3..5db6d551193 100644 --- a/security-utils/src/main/java/com/yahoo/security/tls/TransportSecurityOptions.java +++ b/security-utils/src/main/java/com/yahoo/security/tls/TransportSecurityOptions.java @@ -30,6 +30,7 @@ public class TransportSecurityOptions { private final Path caCertificatesFile; private final AuthorizedPeers authorizedPeers; private final List<String> acceptedCiphers; + private final boolean isHostnameValidationDisabled; private TransportSecurityOptions(Builder builder) { this.privateKeyFile = builder.privateKeyFile; @@ -37,6 +38,7 @@ public class TransportSecurityOptions { this.caCertificatesFile = builder.caCertificatesFile; this.authorizedPeers = builder.authorizedPeers; this.acceptedCiphers = builder.acceptedCiphers; + this.isHostnameValidationDisabled = builder.isHostnameValidationDisabled; } public Optional<Path> getPrivateKeyFile() { @@ -57,6 +59,8 @@ public class TransportSecurityOptions { public List<String> getAcceptedCiphers() { return acceptedCiphers; } + public boolean isHostnameValidationDisabled() { return isHostnameValidationDisabled; } + public static TransportSecurityOptions fromJsonFile(Path file) { try (InputStream in = Files.newInputStream(file)) { return new TransportSecurityOptionsJsonSerializer().deserialize(in); @@ -90,6 +94,7 @@ public class TransportSecurityOptions { private Path caCertificatesFile; private AuthorizedPeers authorizedPeers; private List<String> acceptedCiphers = new ArrayList<>(); + private boolean isHostnameValidationDisabled; public Builder() {} @@ -114,6 +119,11 @@ public class TransportSecurityOptions { return this; } + public Builder withHostnameValidationDisabled(boolean isDisabled) { + this.isHostnameValidationDisabled = isDisabled; + return this; + } + public TransportSecurityOptions build() { return new TransportSecurityOptions(this); } @@ -127,6 +137,7 @@ public class TransportSecurityOptions { ", caCertificatesFile=" + caCertificatesFile + ", authorizedPeers=" + authorizedPeers + ", acceptedCiphers=" + acceptedCiphers + + ", isHostnameValidationDisabled=" + isHostnameValidationDisabled + '}'; } @@ -135,7 +146,8 @@ public class TransportSecurityOptions { if (this == o) return true; if (o == null || getClass() != o.getClass()) return false; TransportSecurityOptions that = (TransportSecurityOptions) o; - return Objects.equals(privateKeyFile, that.privateKeyFile) && + return isHostnameValidationDisabled == that.isHostnameValidationDisabled && + Objects.equals(privateKeyFile, that.privateKeyFile) && Objects.equals(certificatesFile, that.certificatesFile) && Objects.equals(caCertificatesFile, that.caCertificatesFile) && Objects.equals(authorizedPeers, that.authorizedPeers) && @@ -144,6 +156,6 @@ public class TransportSecurityOptions { @Override public int hashCode() { - return Objects.hash(privateKeyFile, certificatesFile, caCertificatesFile, authorizedPeers, acceptedCiphers); + return Objects.hash(privateKeyFile, certificatesFile, caCertificatesFile, authorizedPeers, acceptedCiphers, isHostnameValidationDisabled); } }
\ No newline at end of file diff --git a/security-utils/src/main/java/com/yahoo/security/tls/json/TransportSecurityOptionsEntity.java b/security-utils/src/main/java/com/yahoo/security/tls/json/TransportSecurityOptionsEntity.java index 6594fa84255..2b001ca2ca0 100644 --- a/security-utils/src/main/java/com/yahoo/security/tls/json/TransportSecurityOptionsEntity.java +++ b/security-utils/src/main/java/com/yahoo/security/tls/json/TransportSecurityOptionsEntity.java @@ -8,6 +8,7 @@ import com.fasterxml.jackson.annotation.JsonProperty; import java.util.List; import static com.fasterxml.jackson.annotation.JsonInclude.Include.NON_EMPTY; +import static com.fasterxml.jackson.annotation.JsonInclude.Include.NON_NULL; /** * Jackson bindings for transport security options @@ -20,6 +21,7 @@ class TransportSecurityOptionsEntity { @JsonProperty("files") Files files; @JsonProperty("authorized-peers") @JsonInclude(NON_EMPTY) List<AuthorizedPeer> authorizedPeers; @JsonProperty("accepted-ciphers") @JsonInclude(NON_EMPTY) List<String> acceptedCiphers; + @JsonProperty("disable-hostname-validation") @JsonInclude(NON_NULL) Boolean isHostnameValidationDisabled; static class Files { @JsonProperty("private-key") String privateKeyFile; diff --git a/security-utils/src/main/java/com/yahoo/security/tls/json/TransportSecurityOptionsJsonSerializer.java b/security-utils/src/main/java/com/yahoo/security/tls/json/TransportSecurityOptionsJsonSerializer.java index 5487bad24e7..3cba434912c 100644 --- a/security-utils/src/main/java/com/yahoo/security/tls/json/TransportSecurityOptionsJsonSerializer.java +++ b/security-utils/src/main/java/com/yahoo/security/tls/json/TransportSecurityOptionsJsonSerializer.java @@ -77,6 +77,9 @@ public class TransportSecurityOptionsJsonSerializer { } builder.withAcceptedCiphers(entity.acceptedCiphers); } + if (entity.isHostnameValidationDisabled != null) { + builder.withHostnameValidationDisabled(entity.isHostnameValidationDisabled); + } return builder.build(); } @@ -158,6 +161,9 @@ public class TransportSecurityOptionsJsonSerializer { if (!options.getAcceptedCiphers().isEmpty()) { entity.acceptedCiphers = options.getAcceptedCiphers(); } + if (options.isHostnameValidationDisabled()) { + entity.isHostnameValidationDisabled = true; + } return entity; } diff --git a/security-utils/src/test/java/com/yahoo/security/tls/TransportSecurityOptionsTest.java b/security-utils/src/test/java/com/yahoo/security/tls/TransportSecurityOptionsTest.java index 28dc10d31d5..f2d2b932cd0 100644 --- a/security-utils/src/test/java/com/yahoo/security/tls/TransportSecurityOptionsTest.java +++ b/security-utils/src/test/java/com/yahoo/security/tls/TransportSecurityOptionsTest.java @@ -21,6 +21,7 @@ public class TransportSecurityOptionsTest { .withCertificates(Paths.get("certs.pem"), Paths.get("myhost.key")) .withCaCertificates(Paths.get("my_cas.pem")) .withAcceptedCiphers(com.yahoo.vespa.jdk8compat.List.of("TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384" , "TLS_AES_256_GCM_SHA384")) + .withHostnameValidationDisabled(true) .build(); @Test diff --git a/security-utils/src/test/java/com/yahoo/security/tls/json/TransportSecurityOptionsJsonSerializerTest.java b/security-utils/src/test/java/com/yahoo/security/tls/json/TransportSecurityOptionsJsonSerializerTest.java index 078aa58c948..0dec75fa711 100644 --- a/security-utils/src/test/java/com/yahoo/security/tls/json/TransportSecurityOptionsJsonSerializerTest.java +++ b/security-utils/src/test/java/com/yahoo/security/tls/json/TransportSecurityOptionsJsonSerializerTest.java @@ -22,7 +22,6 @@ import java.nio.file.Paths; import java.util.Arrays; import java.util.Collections; import java.util.HashSet; -import java.util.List; import static com.yahoo.security.tls.policy.RequiredPeerCredential.Field.CN; import static com.yahoo.security.tls.policy.RequiredPeerCredential.Field.SAN_DNS; @@ -44,6 +43,7 @@ public class TransportSecurityOptionsJsonSerializerTest { TransportSecurityOptions options = new TransportSecurityOptions.Builder() .withCaCertificates(Paths.get("/path/to/ca-certs.pem")) .withCertificates(Paths.get("/path/to/cert.pem"), Paths.get("/path/to/key.pem")) + .withHostnameValidationDisabled(false) .withAuthorizedPeers( new AuthorizedPeers( new HashSet<>(Arrays.asList( @@ -66,6 +66,7 @@ public class TransportSecurityOptionsJsonSerializerTest { .withCertificates(Paths.get("certs.pem"), Paths.get("myhost.key")) .withCaCertificates(Paths.get("my_cas.pem")) .withAcceptedCiphers(com.yahoo.vespa.jdk8compat.List.of("TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384" , "TLS_AES_256_GCM_SHA384")) + .withHostnameValidationDisabled(true) .build(); File outputFile = tempDirectory.newFile(); try (OutputStream out = Files.newOutputStream(outputFile.toPath())) { @@ -76,4 +77,22 @@ public class TransportSecurityOptionsJsonSerializerTest { assertJsonEquals(expectedOutput, actualOutput); } + @Test + public void disable_hostname_validation_is_not_serialized_if_false() throws IOException { + TransportSecurityOptions options = new TransportSecurityOptions.Builder() + .withCertificates(Paths.get("certs.pem"), Paths.get("myhost.key")) + .withCaCertificates(Paths.get("my_cas.pem")) + .withHostnameValidationDisabled(false) + .build(); + File outputFile = tempDirectory.newFile(); + try (OutputStream out = Files.newOutputStream(outputFile.toPath())) { + new TransportSecurityOptionsJsonSerializer().serialize(out, options); + } + + String expectedOutput = new String(Files.readAllBytes( + Paths.get("src/test/resources/transport-security-options-with-disable-hostname-validation-set-to-false.json"))); + String actualOutput = new String(Files.readAllBytes(outputFile.toPath())); + assertJsonEquals(expectedOutput, actualOutput); + } + } diff --git a/security-utils/src/test/resources/transport-security-options-with-disable-hostname-validation-set-to-false.json b/security-utils/src/test/resources/transport-security-options-with-disable-hostname-validation-set-to-false.json new file mode 100644 index 00000000000..0506c130722 --- /dev/null +++ b/security-utils/src/test/resources/transport-security-options-with-disable-hostname-validation-set-to-false.json @@ -0,0 +1,7 @@ +{ + "files": { + "private-key": "myhost.key", + "ca-certificates": "my_cas.pem", + "certificates": "certs.pem" + } +}
\ No newline at end of file diff --git a/security-utils/src/test/resources/transport-security-options.json b/security-utils/src/test/resources/transport-security-options.json index 2e55c8fd931..7983982f644 100644 --- a/security-utils/src/test/resources/transport-security-options.json +++ b/security-utils/src/test/resources/transport-security-options.json @@ -1,8 +1,9 @@ { + "disable-hostname-validation": true, "files": { "private-key": "myhost.key", "ca-certificates": "my_cas.pem", "certificates": "certs.pem" - }, - "accepted-ciphers": ["TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384", "TLS_AES_256_GCM_SHA384"] + }, + "accepted-ciphers": ["TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384", "TLS_AES_256_GCM_SHA384"] }
\ No newline at end of file diff --git a/storage/src/tests/frameworkimpl/status/statustest.cpp b/storage/src/tests/frameworkimpl/status/statustest.cpp index 81d91e2f08a..7c259f00899 100644 --- a/storage/src/tests/frameworkimpl/status/statustest.cpp +++ b/storage/src/tests/frameworkimpl/status/statustest.cpp @@ -19,7 +19,7 @@ vespalib::string fetch(int port, const vespalib::string &path) { auto crypto = vespalib::CryptoEngine::get_default(); auto socket = vespalib::SocketSpec::from_port(port).client_address().connect(); assert(socket.valid()); - auto conn = vespalib::SyncCryptoSocket::create(*crypto, std::move(socket), false); + auto conn = vespalib::SyncCryptoSocket::create_client(*crypto, std::move(socket), vespalib::SocketSpec::from_host_port("localhost", port)); vespalib::string http_req = vespalib::make_string("GET %s HTTP/1.1\r\n" "Host: localhost:%d\r\n" "\r\n", path.c_str(), port); diff --git a/storage/src/vespa/storage/persistence/filestorage/filestorhandler.cpp b/storage/src/vespa/storage/persistence/filestorage/filestorhandler.cpp index 350cdad791c..543accc80ae 100644 --- a/storage/src/vespa/storage/persistence/filestorage/filestorhandler.cpp +++ b/storage/src/vespa/storage/persistence/filestorage/filestorhandler.cpp @@ -6,22 +6,18 @@ namespace storage { FileStorHandler::FileStorHandler(MessageSender& sender, FileStorMetrics& metrics, const spi::PartitionStateList& partitions, ServiceLayerComponentRegister& compReg) - : _impl(new FileStorHandlerImpl(1, 1, sender, metrics, partitions, compReg)) + : _impl(std::make_unique<FileStorHandlerImpl>(1, 1, sender, metrics, partitions, compReg)) { } FileStorHandler::FileStorHandler(uint32_t numThreads, uint32_t numStripes, MessageSender& sender, FileStorMetrics& metrics, const spi::PartitionStateList& partitions, ServiceLayerComponentRegister& compReg) - : _impl(new FileStorHandlerImpl(numThreads, numStripes, sender, metrics, partitions, compReg)) + : _impl(std::make_unique<FileStorHandlerImpl>(numThreads, numStripes, sender, metrics, partitions, compReg)) { } -FileStorHandler::~FileStorHandler() -{ - delete _impl; -} - +FileStorHandler::~FileStorHandler() = default; void FileStorHandler::flush(bool flushMerges) { diff --git a/storage/src/vespa/storage/persistence/filestorage/filestorhandler.h b/storage/src/vespa/storage/persistence/filestorage/filestorhandler.h index ab3d03a5e9a..db294d7c39f 100644 --- a/storage/src/vespa/storage/persistence/filestorage/filestorhandler.h +++ b/storage/src/vespa/storage/persistence/filestorage/filestorhandler.h @@ -35,7 +35,6 @@ namespace framework { class FileStorHandlerImpl; struct FileStorMetrics; struct MessageSender; -class MountPointList; struct ServiceLayerComponentRegister; class AbortBucketOperationsCommand; @@ -259,7 +258,7 @@ public: std::string dumpQueue(uint16_t disk) const; private: - FileStorHandlerImpl* _impl; + std::unique_ptr<FileStorHandlerImpl> _impl; }; } // storage diff --git a/storage/src/vespa/storage/persistence/filestorage/filestorhandlerimpl.cpp b/storage/src/vespa/storage/persistence/filestorage/filestorhandlerimpl.cpp index f773ee774bb..080446c1c92 100644 --- a/storage/src/vespa/storage/persistence/filestorage/filestorhandlerimpl.cpp +++ b/storage/src/vespa/storage/persistence/filestorage/filestorhandlerimpl.cpp @@ -879,15 +879,15 @@ FileStorHandlerImpl::MessageEntry::MessageEntry(MessageEntry && entry) noexcept _priority(entry._priority) { } -FileStorHandlerImpl::MessageEntry::~MessageEntry() { } +FileStorHandlerImpl::MessageEntry::~MessageEntry() = default; -FileStorHandlerImpl::Disk::Disk(const FileStorHandlerImpl & owner, MessageSender & messageSender, uint32_t numThreads) +FileStorHandlerImpl::Disk::Disk(const FileStorHandlerImpl & owner, MessageSender & messageSender, uint32_t numStripes) : metrics(0), _nextStripeId(0), - _stripes(numThreads, Stripe(owner, messageSender)), + _stripes(numStripes, Stripe(owner, messageSender)), state(FileStorHandler::AVAILABLE) { - assert(numThreads > 0); + assert(numStripes > 0); } FileStorHandlerImpl::Disk::Disk(Disk && rhs) noexcept diff --git a/storage/src/vespa/storage/persistence/filestorage/filestorhandlerimpl.h b/storage/src/vespa/storage/persistence/filestorage/filestorhandlerimpl.h index 5fc592e11cb..44756ed5891 100644 --- a/storage/src/vespa/storage/persistence/filestorage/filestorhandlerimpl.h +++ b/storage/src/vespa/storage/persistence/filestorage/filestorhandlerimpl.h @@ -169,7 +169,7 @@ public: state.store(s, std::memory_order_relaxed); } - Disk(const FileStorHandlerImpl & owner, MessageSender & messageSender, uint32_t numThreads); + Disk(const FileStorHandlerImpl & owner, MessageSender & messageSender, uint32_t numStripes); Disk(Disk &&) noexcept; ~Disk(); diff --git a/storage/src/vespa/storage/persistence/filestorage/filestormanager.cpp b/storage/src/vespa/storage/persistence/filestorage/filestormanager.cpp index baa6523cbb2..b9fb85e0e80 100644 --- a/storage/src/vespa/storage/persistence/filestorage/filestormanager.cpp +++ b/storage/src/vespa/storage/persistence/filestorage/filestormanager.cpp @@ -38,11 +38,11 @@ FileStorManager(const config::ConfigUri & configUri, const spi::PartitionStateLi _bucketIdFactory(_component.getBucketIdFactory()), _configUri(configUri), _disks(), - _bucketOwnershipNotifier(new BucketOwnershipNotifier(_component, *this)), + _bucketOwnershipNotifier(std::make_unique<BucketOwnershipNotifier>(_component, *this)), _configFetcher(_configUri.getContext()), _threadLockCheckInterval(60), _failDiskOnError(false), - _metrics(new FileStorMetrics(_component.getLoadTypes()->getMetricLoadTypes())), + _metrics(std::make_unique<FileStorMetrics>(_component.getLoadTypes()->getMetricLoadTypes())), _threadMonitor(), _closed(false) { @@ -105,10 +105,10 @@ FileStorManager::configure(std::unique_ptr<vespa::config::content::StorFilestorC _config = std::move(config); _disks.resize(_component.getDiskCount()); size_t numThreads = _config->numThreads; - size_t numStripes = std::min(2ul, numThreads); + size_t numStripes = std::min(4ul, numThreads); _metrics->initDiskMetrics(_disks.size(), _component.getLoadTypes()->getMetricLoadTypes(), numStripes, numThreads); - _filestorHandler.reset(new FileStorHandler(numThreads, numStripes, *this, *_metrics, _partitions, _compReg)); + _filestorHandler = std::make_unique<FileStorHandler>(numThreads, numStripes, *this, *_metrics, _partitions, _compReg); for (uint32_t i=0; i<_component.getDiskCount(); ++i) { if (_partitions[i].isUp()) { LOG(spam, "Setting up disk %u", i); @@ -849,12 +849,6 @@ FileStorManager::reportHtmlStatus(std::ostream& out, const framework::HttpUrlPat _filestorHandler->getStatus(out, path); } -bool -FileStorManager::isMerging(const document::Bucket& bucket) const -{ - return _filestorHandler->isMerging(bucket); -} - namespace { struct Deactivator { StorBucketDatabase::Decision operator()(document::BucketId::Type, StorBucketDatabase::Entry& data) diff --git a/storage/src/vespa/storage/persistence/filestorage/filestormanager.h b/storage/src/vespa/storage/persistence/filestorage/filestormanager.h index 65d4035a3dd..433b9ddbd39 100644 --- a/storage/src/vespa/storage/persistence/filestorage/filestormanager.h +++ b/storage/src/vespa/storage/persistence/filestorage/filestormanager.h @@ -45,65 +45,43 @@ class FileStorManager : public StorageLinkQueued, public framework::HtmlStatusReporter, public StateListener, private config::IFetcherCallback<vespa::config::content::StorFilestorConfig>, - private MessageSender + public MessageSender { - ServiceLayerComponentRegister& _compReg; - ServiceLayerComponent _component; - const spi::PartitionStateList& _partitions; - spi::PersistenceProvider& _providerCore; - ProviderErrorWrapper _providerErrorWrapper; - spi::PersistenceProvider* _provider; + ServiceLayerComponentRegister & _compReg; + ServiceLayerComponent _component; + const spi::PartitionStateList & _partitions; + spi::PersistenceProvider & _providerCore; + ProviderErrorWrapper _providerErrorWrapper; + spi::PersistenceProvider * _provider; const document::BucketIdFactory& _bucketIdFactory; - config::ConfigUri _configUri; + config::ConfigUri _configUri; typedef std::vector<DiskThread::SP> DiskThreads; - std::vector<DiskThreads> _disks; + std::vector<DiskThreads> _disks; std::unique_ptr<BucketOwnershipNotifier> _bucketOwnershipNotifier; std::unique_ptr<vespa::config::content::StorFilestorConfig> _config; config::ConfigFetcher _configFetcher; - uint32_t _threadLockCheckInterval; // In seconds - bool _failDiskOnError; - int _killSignal; + uint32_t _threadLockCheckInterval; // In seconds + bool _failDiskOnError; std::shared_ptr<FileStorMetrics> _metrics; std::unique_ptr<FileStorHandler> _filestorHandler; - lib::ClusterState _lastState; - struct ReplyHolder { - int refCount; - std::unique_ptr<api::StorageReply> reply; - - ReplyHolder(int rc, std::unique_ptr<api::StorageReply> r) - : refCount(rc), reply(std::move(r)) {}; - }; - - std::map<api::StorageMessage::Id, - std::shared_ptr<ReplyHolder> > _splitMessages; - vespalib::Lock _splitLock; mutable vespalib::Monitor _threadMonitor; // Notify to stop sleeping - bool _closed; - - FileStorManager(const FileStorManager &); - FileStorManager& operator=(const FileStorManager &); + bool _closed; - std::vector<DiskThreads> getThreads() { return _disks; } - - friend class BucketMergeTest; friend struct FileStorManagerTest; - friend class MessageTest; public: - explicit FileStorManager(const config::ConfigUri &, - const spi::PartitionStateList&, - spi::PersistenceProvider&, - ServiceLayerComponentRegister&); - ~FileStorManager(); + FileStorManager(const config::ConfigUri &, const spi::PartitionStateList&, + spi::PersistenceProvider&, ServiceLayerComponentRegister&); + FileStorManager(const FileStorManager &) = delete; + FileStorManager& operator=(const FileStorManager &) = delete; - void print(std::ostream& out, bool verbose, const std::string& indent) const override; + ~FileStorManager() override; - // Return true if we are currently merging the given bucket. - bool isMerging(const document::Bucket& bucket) const; + void print(std::ostream& out, bool verbose, const std::string& indent) const override; FileStorHandler& getFileStorHandler() { return *_filestorHandler; diff --git a/storage/src/vespa/storage/persistence/persistencethread.cpp b/storage/src/vespa/storage/persistence/persistencethread.cpp index 7d0ce26b83d..dd44c96555b 100644 --- a/storage/src/vespa/storage/persistence/persistencethread.cpp +++ b/storage/src/vespa/storage/persistence/persistencethread.cpp @@ -795,8 +795,8 @@ PersistenceThread::handleCommand(api::StorageCommand& msg) { _context = spi::Context(msg.getLoadType(), msg.getPriority(), msg.getTrace().getLevel()); MessageTracker::UP mtracker(handleCommandSplitByType(msg)); - if (mtracker.get() != 0) { - if (mtracker->getReply().get() != 0) { + if (mtracker) { + if (mtracker->getReply()) { mtracker->getReply()->getTrace().getRoot().addChild(_context.getTrace().getRoot()); } else { msg.getTrace().getRoot().addChild(_context.getTrace().getRoot()); diff --git a/storage/src/vespa/storage/tools/analyzedistribution.cpp b/storage/src/vespa/storage/tools/analyzedistribution.cpp index 52c26c4bb17..472aa63fff6 100644 --- a/storage/src/vespa/storage/tools/analyzedistribution.cpp +++ b/storage/src/vespa/storage/tools/analyzedistribution.cpp @@ -132,7 +132,7 @@ Node::Node(const lib::NodeState& dstate, const lib::NodeState& sstate, uint32_t disks.push_back(Disk(storageState.getDiskState(i))); } } -Node::~Node() {} +Node::~Node() = default; struct Distribution { std::vector<Node> nodes; diff --git a/storageframework/src/vespa/storageframework/generic/status/httpurlpath.cpp b/storageframework/src/vespa/storageframework/generic/status/httpurlpath.cpp index 337709cc591..10691d37b9e 100644 --- a/storageframework/src/vespa/storageframework/generic/status/httpurlpath.cpp +++ b/storageframework/src/vespa/storageframework/generic/status/httpurlpath.cpp @@ -77,7 +77,7 @@ HttpUrlPath::print(std::ostream& out, bool, const std::string&) const if (!_attributes.empty()) { out << "?"; size_t cnt = 0; - for (const auto attr: _attributes) { + for (const auto &attr: _attributes) { if (cnt++ > 0) { out << "&"; } diff --git a/vbench/src/vbench/core/socket.cpp b/vbench/src/vbench/core/socket.cpp index 822b96b2c07..0431b6889a8 100644 --- a/vbench/src/vbench/core/socket.cpp +++ b/vbench/src/vbench/core/socket.cpp @@ -29,7 +29,8 @@ Socket::Socket(SyncCryptoSocket::UP socket) } Socket::Socket(CryptoEngine &crypto, const string &host, int port) - : _socket(SyncCryptoSocket::create(crypto, connect(host, port), false)), + : _socket(SyncCryptoSocket::create_client(crypto, connect(host, port), + vespalib::SocketSpec::from_host_port(host, port))), _input(), _output(), _taint(), diff --git a/vbench/src/vbench/core/socket.h b/vbench/src/vbench/core/socket.h index 0e8848e8292..4b5721c0fb7 100644 --- a/vbench/src/vbench/core/socket.h +++ b/vbench/src/vbench/core/socket.h @@ -52,7 +52,7 @@ struct ServerSocket { Stream::UP accept(CryptoEngine &crypto) { vespalib::SocketHandle handle = server_socket.accept(); if (handle.valid()) { - return std::make_unique<Socket>(SyncCryptoSocket::create(crypto, std::move(handle), true)); + return std::make_unique<Socket>(SyncCryptoSocket::create_server(crypto, std::move(handle))); } else { return Stream::UP(); } diff --git a/vespalib/src/tests/net/crypto_socket/crypto_socket_test.cpp b/vespalib/src/tests/net/crypto_socket/crypto_socket_test.cpp index 62bad716597..e938e15f4e6 100644 --- a/vespalib/src/tests/net/crypto_socket/crypto_socket_test.cpp +++ b/vespalib/src/tests/net/crypto_socket/crypto_socket_test.cpp @@ -17,6 +17,7 @@ #include <fcntl.h> using namespace vespalib; +using namespace vespalib::test; struct SocketPair { SocketHandle client; @@ -204,7 +205,9 @@ void verify_crypto_socket(SocketPair &sockets, CryptoEngine &engine, bool is_ser SocketHandle &my_handle = is_server ? sockets.server : sockets.client; my_handle.set_blocking(false); SmartBuffer read_buffer(4096); - CryptoSocket::UP my_socket = engine.create_crypto_socket(std::move(my_handle), is_server); + CryptoSocket::UP my_socket = is_server + ? engine.create_server_crypto_socket(std::move(my_handle)) + : engine.create_client_crypto_socket(std::move(my_handle), local_spec); TEST_DO(verify_handshake(*my_socket)); drain(*my_socket, read_buffer); TEST_DO(verify_socket_io(*my_socket, read_buffer, is_server)); @@ -226,19 +229,19 @@ TEST_MT_FFF("require that encrypted async socket io works with XorCryptoEngine", } TEST_MT_FFF("require that encrypted async socket io works with TlsCryptoEngine", - 2, SocketPair(), TlsCryptoEngine(vespalib::test::make_tls_options_for_testing()), TimeBomb(60)) + 2, SocketPair(), TlsCryptoEngine(make_tls_options_for_testing()), TimeBomb(60)) { TEST_DO(verify_crypto_socket(f1, f2, (thread_id == 0))); } TEST_MT_FFF("require that encrypted async socket io works with MaybeTlsCryptoEngine(true)", - 2, SocketPair(), MaybeTlsCryptoEngine(std::make_shared<TlsCryptoEngine>(vespalib::test::make_tls_options_for_testing()), true), TimeBomb(60)) + 2, SocketPair(), MaybeTlsCryptoEngine(std::make_shared<TlsCryptoEngine>(make_tls_options_for_testing()), true), TimeBomb(60)) { TEST_DO(verify_crypto_socket(f1, f2, (thread_id == 0))); } TEST_MT_FFF("require that encrypted async socket io works with MaybeTlsCryptoEngine(false)", - 2, SocketPair(), MaybeTlsCryptoEngine(std::make_shared<TlsCryptoEngine>(vespalib::test::make_tls_options_for_testing()), false), TimeBomb(60)) + 2, SocketPair(), MaybeTlsCryptoEngine(std::make_shared<TlsCryptoEngine>(make_tls_options_for_testing()), false), TimeBomb(60)) { TEST_DO(verify_crypto_socket(f1, f2, (thread_id == 0))); } diff --git a/vespalib/src/tests/net/socket_spec/socket_spec_test.cpp b/vespalib/src/tests/net/socket_spec/socket_spec_test.cpp index 0c2b92bbb53..f2da6b70bf3 100644 --- a/vespalib/src/tests/net/socket_spec/socket_spec_test.cpp +++ b/vespalib/src/tests/net/socket_spec/socket_spec_test.cpp @@ -123,4 +123,8 @@ TEST("require that replace_host gives invalid spec when used with less than 2 ho TEST_DO(verify_invalid(SocketSpec("ipc/name:my_socket").replace_host("foo"))); } +TEST("require that invalid socket spec is not valid") { + EXPECT_FALSE(SocketSpec::invalid.valid()); +} + TEST_MAIN() { TEST_RUN_ALL(); } diff --git a/vespalib/src/tests/net/sync_crypto_socket/sync_crypto_socket_test.cpp b/vespalib/src/tests/net/sync_crypto_socket/sync_crypto_socket_test.cpp index 76ecd9453b6..56767051dad 100644 --- a/vespalib/src/tests/net/sync_crypto_socket/sync_crypto_socket_test.cpp +++ b/vespalib/src/tests/net/sync_crypto_socket/sync_crypto_socket_test.cpp @@ -17,6 +17,7 @@ #include <fcntl.h> using namespace vespalib; +using namespace vespalib::test; struct SocketPair { SocketHandle client; @@ -97,7 +98,9 @@ void verify_socket_io(SyncCryptoSocket &socket, bool is_server) { void verify_crypto_socket(SocketPair &sockets, CryptoEngine &engine, bool is_server) { SocketHandle &my_handle = is_server ? sockets.server : sockets.client; my_handle.set_blocking(false); - SyncCryptoSocket::UP my_socket = SyncCryptoSocket::create(engine, std::move(my_handle), is_server); + SyncCryptoSocket::UP my_socket = is_server + ? SyncCryptoSocket::create_server(engine, std::move(my_handle)) + : SyncCryptoSocket::create_client(engine, std::move(my_handle), local_spec); ASSERT_TRUE(my_socket); TEST_DO(verify_socket_io(*my_socket, is_server)); TEST_DO(verify_graceful_shutdown(*my_socket, is_server)); @@ -118,19 +121,19 @@ TEST_MT_FFF("require that encrypted sync socket io works with XorCryptoEngine", } TEST_MT_FFF("require that encrypted sync socket io works with TlsCryptoEngine", - 2, SocketPair(), TlsCryptoEngine(vespalib::test::make_tls_options_for_testing()), TimeBomb(60)) + 2, SocketPair(), TlsCryptoEngine(make_tls_options_for_testing()), TimeBomb(60)) { TEST_DO(verify_crypto_socket(f1, f2, (thread_id == 0))); } TEST_MT_FFF("require that encrypted sync socket io works with MaybeTlsCryptoEngine(true)", - 2, SocketPair(), MaybeTlsCryptoEngine(std::make_shared<TlsCryptoEngine>(vespalib::test::make_tls_options_for_testing()), true), TimeBomb(60)) + 2, SocketPair(), MaybeTlsCryptoEngine(std::make_shared<TlsCryptoEngine>(make_tls_options_for_testing()), true), TimeBomb(60)) { TEST_DO(verify_crypto_socket(f1, f2, (thread_id == 0))); } TEST_MT_FFF("require that encrypted sync socket io works with MaybeTlsCryptoEngine(false)", - 2, SocketPair(), MaybeTlsCryptoEngine(std::make_shared<TlsCryptoEngine>(vespalib::test::make_tls_options_for_testing()), false), TimeBomb(60)) + 2, SocketPair(), MaybeTlsCryptoEngine(std::make_shared<TlsCryptoEngine>(make_tls_options_for_testing()), false), TimeBomb(60)) { TEST_DO(verify_crypto_socket(f1, f2, (thread_id == 0))); } diff --git a/vespalib/src/tests/portal/portal_test.cpp b/vespalib/src/tests/portal/portal_test.cpp index e54700306fe..0bd029c0c3a 100644 --- a/vespalib/src/tests/portal/portal_test.cpp +++ b/vespalib/src/tests/portal/portal_test.cpp @@ -14,13 +14,14 @@ #include <vespa/vespalib/util/latch.h> using namespace vespalib; +using namespace vespalib::test; //----------------------------------------------------------------------------- vespalib::string do_http(int port, CryptoEngine::SP crypto, const vespalib::string &method, const vespalib::string &uri, bool send_host = true) { auto socket = SocketSpec::from_port(port).client_address().connect(); ASSERT_TRUE(socket.valid()); - auto conn = SyncCryptoSocket::create(*crypto, std::move(socket), false); + auto conn = SyncCryptoSocket::create_client(*crypto, std::move(socket), local_spec); vespalib::string http_req = vespalib::make_string("%s %s HTTP/1.1\r\n" "My-Header: my value\r\n" "%s" @@ -75,7 +76,7 @@ Encryption::~Encryption() = default; auto null_crypto() { return std::make_shared<NullCryptoEngine>(); } auto xor_crypto() { return std::make_shared<XorCryptoEngine>(); } -auto tls_crypto() { return std::make_shared<TlsCryptoEngine>(vespalib::test::make_tls_options_for_testing()); } +auto tls_crypto() { return std::make_shared<TlsCryptoEngine>(make_tls_options_for_testing()); } auto maybe_tls_crypto(bool client_tls) { return std::make_shared<MaybeTlsCryptoEngine>(tls_crypto(), client_tls); } std::vector<Encryption> crypto_list = {{"no encryption", null_crypto()}, @@ -260,18 +261,18 @@ TEST("require that connection errors do not block shutdown by leaking resources" auto bound = portal->bind("/test", handler); { // close before sending anything auto socket = SocketSpec::from_port(portal->listen_port()).client_address().connect(); - auto conn = SyncCryptoSocket::create(*crypto.engine, std::move(socket), false); + auto conn = SyncCryptoSocket::create_client(*crypto.engine, std::move(socket), local_spec); } { // send partial request then close connection auto socket = SocketSpec::from_port(portal->listen_port()).client_address().connect(); - auto conn = SyncCryptoSocket::create(*crypto.engine, std::move(socket), false); + auto conn = SyncCryptoSocket::create_client(*crypto.engine, std::move(socket), local_spec); vespalib::string req = "GET /test HTTP/1.1\r\n" "Host: local"; ASSERT_EQUAL(conn->write(req.data(), req.size()), ssize_t(req.size())); } { // send request then close without reading response auto socket = SocketSpec::from_port(portal->listen_port()).client_address().connect(); - auto conn = SyncCryptoSocket::create(*crypto.engine, std::move(socket), false); + auto conn = SyncCryptoSocket::create_client(*crypto.engine, std::move(socket), local_spec); vespalib::string req = "GET /test HTTP/1.1\r\n" "Host: localhost\r\n" "\r\n"; diff --git a/vespalib/src/vespa/vespalib/net/crypto_engine.cpp b/vespalib/src/vespa/vespalib/net/crypto_engine.cpp index a52d4eeb690..a92b0e06bbe 100644 --- a/vespalib/src/vespa/vespalib/net/crypto_engine.cpp +++ b/vespalib/src/vespa/vespalib/net/crypto_engine.cpp @@ -250,16 +250,29 @@ CryptoEngine::get_default() } CryptoSocket::UP -NullCryptoEngine::create_crypto_socket(SocketHandle socket, bool is_server) +NullCryptoEngine::create_client_crypto_socket(SocketHandle socket, const SocketSpec &) { - net::tls::ConnectionStatistics::get(is_server).inc_insecure_connections(); + net::tls::ConnectionStatistics::get(false).inc_insecure_connections(); return std::make_unique<NullCryptoSocket>(std::move(socket)); } CryptoSocket::UP -XorCryptoEngine::create_crypto_socket(SocketHandle socket, bool is_server) +NullCryptoEngine::create_server_crypto_socket(SocketHandle socket) { - return std::make_unique<XorCryptoSocket>(std::move(socket), is_server); + net::tls::ConnectionStatistics::get(true).inc_insecure_connections(); + return std::make_unique<NullCryptoSocket>(std::move(socket)); +} + +CryptoSocket::UP +XorCryptoEngine::create_client_crypto_socket(SocketHandle socket, const SocketSpec &) +{ + return std::make_unique<XorCryptoSocket>(std::move(socket), false); +} + +CryptoSocket::UP +XorCryptoEngine::create_server_crypto_socket(SocketHandle socket) +{ + return std::make_unique<XorCryptoSocket>(std::move(socket), true); } } // namespace vespalib diff --git a/vespalib/src/vespa/vespalib/net/crypto_engine.h b/vespalib/src/vespa/vespalib/net/crypto_engine.h index 1cb1305e039..4deacf9a6c7 100644 --- a/vespalib/src/vespa/vespalib/net/crypto_engine.h +++ b/vespalib/src/vespa/vespalib/net/crypto_engine.h @@ -9,6 +9,8 @@ namespace vespalib { +class SocketSpec; + /** * Component responsible for wrapping low-level sockets into * appropriate CryptoSocket instances. This is the top-level interface @@ -17,7 +19,8 @@ namespace vespalib { **/ struct CryptoEngine { using SP = std::shared_ptr<CryptoEngine>; - virtual CryptoSocket::UP create_crypto_socket(SocketHandle socket, bool is_server) = 0; + virtual CryptoSocket::UP create_client_crypto_socket(SocketHandle socket, const SocketSpec &spec) = 0; + virtual CryptoSocket::UP create_server_crypto_socket(SocketHandle socket) = 0; virtual ~CryptoEngine(); static CryptoEngine::SP get_default(); }; @@ -26,7 +29,8 @@ struct CryptoEngine { * Crypto engine without encryption. **/ struct NullCryptoEngine : public CryptoEngine { - CryptoSocket::UP create_crypto_socket(SocketHandle socket, bool is_server) override; + CryptoSocket::UP create_client_crypto_socket(SocketHandle socket, const SocketSpec &spec) override; + CryptoSocket::UP create_server_crypto_socket(SocketHandle socket) override; }; /** @@ -35,7 +39,8 @@ struct NullCryptoEngine : public CryptoEngine { * from TLS. **/ struct XorCryptoEngine : public CryptoEngine { - CryptoSocket::UP create_crypto_socket(SocketHandle socket, bool is_server) override; + CryptoSocket::UP create_client_crypto_socket(SocketHandle socket, const SocketSpec &spec) override; + CryptoSocket::UP create_server_crypto_socket(SocketHandle socket) override; }; } // namespace vespalib diff --git a/vespalib/src/vespa/vespalib/net/socket_spec.cpp b/vespalib/src/vespa/vespalib/net/socket_spec.cpp index d1376ce1dd7..06682086670 100644 --- a/vespalib/src/vespa/vespalib/net/socket_spec.cpp +++ b/vespalib/src/vespa/vespalib/net/socket_spec.cpp @@ -41,6 +41,8 @@ SocketSpec::address(bool server) const return SocketAddress(); } +SocketSpec SocketSpec::invalid; + SocketSpec::SocketSpec(const vespalib::string &spec) : SocketSpec() { diff --git a/vespalib/src/vespa/vespalib/net/socket_spec.h b/vespalib/src/vespa/vespalib/net/socket_spec.h index f28b14573ac..01af382d638 100644 --- a/vespalib/src/vespa/vespalib/net/socket_spec.h +++ b/vespalib/src/vespa/vespalib/net/socket_spec.h @@ -24,6 +24,7 @@ private: : _type(type), _node(node), _port(port) {} SocketAddress address(bool server) const; public: + static SocketSpec invalid; explicit SocketSpec(const vespalib::string &spec); vespalib::string spec() const; SocketSpec replace_host(const vespalib::string &new_host) const; diff --git a/vespalib/src/vespa/vespalib/net/sync_crypto_socket.cpp b/vespalib/src/vespa/vespalib/net/sync_crypto_socket.cpp index 29388035bda..3aa2d3b0683 100644 --- a/vespalib/src/vespa/vespalib/net/sync_crypto_socket.cpp +++ b/vespalib/src/vespa/vespalib/net/sync_crypto_socket.cpp @@ -29,6 +29,25 @@ void set_blocking(int fd) { } // namespace vespalib::<unnamed> +SyncCryptoSocket::UP +SyncCryptoSocket::create(CryptoSocket::UP socket) +{ + set_blocking(socket->get_fd()); + for (;;) { + switch (socket->handshake()) { + case CryptoSocket::HandshakeResult::FAIL: + return std::unique_ptr<SyncCryptoSocket>(nullptr); + case CryptoSocket::HandshakeResult::DONE: + return UP(new SyncCryptoSocket(std::move(socket))); + case CryptoSocket::HandshakeResult::NEED_READ: + case CryptoSocket::HandshakeResult::NEED_WRITE: + break; + case CryptoSocket::HandshakeResult::NEED_WORK: + socket->do_handshake_work(); + } + } +} + SyncCryptoSocket::~SyncCryptoSocket() = default; ssize_t @@ -90,23 +109,15 @@ SyncCryptoSocket::half_close() } SyncCryptoSocket::UP -SyncCryptoSocket::create(CryptoEngine &engine, SocketHandle socket, bool is_server) +SyncCryptoSocket::create_client(CryptoEngine &engine, SocketHandle socket, const SocketSpec &spec) { - auto crypto_socket = engine.create_crypto_socket(std::move(socket), is_server); - set_blocking(crypto_socket->get_fd()); - for (;;) { - switch (crypto_socket->handshake()) { - case CryptoSocket::HandshakeResult::FAIL: - return std::unique_ptr<SyncCryptoSocket>(nullptr); - case CryptoSocket::HandshakeResult::DONE: - return UP(new SyncCryptoSocket(std::move(crypto_socket))); - case CryptoSocket::HandshakeResult::NEED_READ: - case CryptoSocket::HandshakeResult::NEED_WRITE: - break; - case CryptoSocket::HandshakeResult::NEED_WORK: - crypto_socket->do_handshake_work(); - } - } + return create(engine.create_client_crypto_socket(std::move(socket), spec)); +} + +SyncCryptoSocket::UP +SyncCryptoSocket::create_server(CryptoEngine &engine, SocketHandle socket) +{ + return create(engine.create_server_crypto_socket(std::move(socket))); } } // namespace vespalib diff --git a/vespalib/src/vespa/vespalib/net/sync_crypto_socket.h b/vespalib/src/vespa/vespalib/net/sync_crypto_socket.h index 00d6cbca0db..36fcfe12ed9 100644 --- a/vespalib/src/vespa/vespalib/net/sync_crypto_socket.h +++ b/vespalib/src/vespa/vespalib/net/sync_crypto_socket.h @@ -19,17 +19,20 @@ namespace vespalib { **/ class SyncCryptoSocket { +public: + using UP = std::unique_ptr<SyncCryptoSocket>; private: CryptoSocket::UP _socket; SmartBuffer _buffer; SyncCryptoSocket(CryptoSocket::UP socket) : _socket(std::move(socket)), _buffer(0) {} + static UP create(CryptoSocket::UP socket); public: - using UP = std::unique_ptr<SyncCryptoSocket>; ~SyncCryptoSocket(); ssize_t read(char *buf, size_t len); ssize_t write(const char *buf, size_t len); ssize_t half_close(); - static UP create(CryptoEngine &engine, SocketHandle socket, bool is_server); + static UP create_client(CryptoEngine &engine, SocketHandle socket, const SocketSpec &spec); + static UP create_server(CryptoEngine &engine, SocketHandle socket); }; } // namespace vespalib diff --git a/vespalib/src/vespa/vespalib/net/tls/auto_reloading_tls_crypto_engine.cpp b/vespalib/src/vespa/vespalib/net/tls/auto_reloading_tls_crypto_engine.cpp index 5f20280e0e2..c425ab75ce8 100644 --- a/vespalib/src/vespa/vespalib/net/tls/auto_reloading_tls_crypto_engine.cpp +++ b/vespalib/src/vespa/vespalib/net/tls/auto_reloading_tls_crypto_engine.cpp @@ -91,13 +91,22 @@ AutoReloadingTlsCryptoEngine::EngineSP AutoReloadingTlsCryptoEngine::acquire_cur return _current_engine; } -CryptoSocket::UP AutoReloadingTlsCryptoEngine::create_crypto_socket(SocketHandle socket, bool is_server) { - return acquire_current_engine()->create_crypto_socket(std::move(socket), is_server); +CryptoSocket::UP AutoReloadingTlsCryptoEngine::create_client_crypto_socket(SocketHandle socket, const SocketSpec &spec) { + return acquire_current_engine()->create_client_crypto_socket(std::move(socket), spec); +} + +CryptoSocket::UP AutoReloadingTlsCryptoEngine::create_server_crypto_socket(SocketHandle socket) { + return acquire_current_engine()->create_server_crypto_socket(std::move(socket)); +} + +std::unique_ptr<TlsCryptoSocket> +AutoReloadingTlsCryptoEngine::create_tls_client_crypto_socket(SocketHandle socket, const SocketSpec &spec) { + return acquire_current_engine()->create_tls_client_crypto_socket(std::move(socket), spec); } std::unique_ptr<TlsCryptoSocket> -AutoReloadingTlsCryptoEngine::create_tls_crypto_socket(SocketHandle socket, bool is_server) { - return acquire_current_engine()->create_tls_crypto_socket(std::move(socket), is_server); +AutoReloadingTlsCryptoEngine::create_tls_server_crypto_socket(SocketHandle socket) { + return acquire_current_engine()->create_tls_server_crypto_socket(std::move(socket)); } } diff --git a/vespalib/src/vespa/vespalib/net/tls/auto_reloading_tls_crypto_engine.h b/vespalib/src/vespa/vespalib/net/tls/auto_reloading_tls_crypto_engine.h index 6287fdd4f63..e268cbc8f1a 100644 --- a/vespalib/src/vespa/vespalib/net/tls/auto_reloading_tls_crypto_engine.h +++ b/vespalib/src/vespa/vespalib/net/tls/auto_reloading_tls_crypto_engine.h @@ -45,8 +45,10 @@ public: EngineSP acquire_current_engine() const; - CryptoSocket::UP create_crypto_socket(SocketHandle socket, bool is_server) override; - std::unique_ptr<TlsCryptoSocket> create_tls_crypto_socket(SocketHandle socket, bool is_server) override; + CryptoSocket::UP create_client_crypto_socket(SocketHandle socket, const SocketSpec &spec) override; + CryptoSocket::UP create_server_crypto_socket(SocketHandle socket) override; + std::unique_ptr<TlsCryptoSocket> create_tls_client_crypto_socket(SocketHandle socket, const SocketSpec &spec) override; + std::unique_ptr<TlsCryptoSocket> create_tls_server_crypto_socket(SocketHandle socket) override; }; } diff --git a/vespalib/src/vespa/vespalib/net/tls/maybe_tls_crypto_engine.cpp b/vespalib/src/vespa/vespalib/net/tls/maybe_tls_crypto_engine.cpp index 891f8cdab23..f7f0284bded 100644 --- a/vespalib/src/vespa/vespalib/net/tls/maybe_tls_crypto_engine.cpp +++ b/vespalib/src/vespa/vespalib/net/tls/maybe_tls_crypto_engine.cpp @@ -6,15 +6,19 @@ namespace vespalib { CryptoSocket::UP -MaybeTlsCryptoEngine::create_crypto_socket(SocketHandle socket, bool is_server) +MaybeTlsCryptoEngine::create_client_crypto_socket(SocketHandle socket, const SocketSpec &spec) { - if (is_server) { - return std::make_unique<MaybeTlsCryptoSocket>(std::move(socket), _tls_engine); - } else if (_use_tls_when_client) { - return _tls_engine->create_crypto_socket(std::move(socket), false); + if (_use_tls_when_client) { + return _tls_engine->create_client_crypto_socket(std::move(socket), spec); } else { - return _null_engine->create_crypto_socket(std::move(socket), false); + return _null_engine->create_client_crypto_socket(std::move(socket), spec); } } +CryptoSocket::UP +MaybeTlsCryptoEngine::create_server_crypto_socket(SocketHandle socket) +{ + return std::make_unique<MaybeTlsCryptoSocket>(std::move(socket), _tls_engine); +} + } // namespace vespalib diff --git a/vespalib/src/vespa/vespalib/net/tls/maybe_tls_crypto_engine.h b/vespalib/src/vespa/vespalib/net/tls/maybe_tls_crypto_engine.h index 29909fa115d..147a770bc8f 100644 --- a/vespalib/src/vespa/vespalib/net/tls/maybe_tls_crypto_engine.h +++ b/vespalib/src/vespa/vespalib/net/tls/maybe_tls_crypto_engine.h @@ -28,7 +28,8 @@ public: : _null_engine(std::make_shared<NullCryptoEngine>()), _tls_engine(std::move(tls_engine)), _use_tls_when_client(use_tls_when_client) {} - CryptoSocket::UP create_crypto_socket(SocketHandle socket, bool is_server) override; + CryptoSocket::UP create_client_crypto_socket(SocketHandle socket, const SocketSpec &spec) override; + CryptoSocket::UP create_server_crypto_socket(SocketHandle socket) override; }; } // namespace vespalib diff --git a/vespalib/src/vespa/vespalib/net/tls/maybe_tls_crypto_socket.cpp b/vespalib/src/vespa/vespalib/net/tls/maybe_tls_crypto_socket.cpp index 9af6703acbc..8ab6adad2e5 100644 --- a/vespalib/src/vespa/vespalib/net/tls/maybe_tls_crypto_socket.cpp +++ b/vespalib/src/vespa/vespalib/net/tls/maybe_tls_crypto_socket.cpp @@ -50,7 +50,7 @@ public: } if (looksLikeTlsToMe(src.data)) { CryptoSocket::UP &self = _self; // need copy due to self destruction - auto tls_socket = _factory->create_tls_crypto_socket(std::move(_socket), true); + auto tls_socket = _factory->create_tls_server_crypto_socket(std::move(_socket)); tls_socket->inject_read_data(src.data, src.size); self = std::move(tls_socket); return self->handshake(); diff --git a/vespalib/src/vespa/vespalib/net/tls/tls_crypto_engine.cpp b/vespalib/src/vespa/vespalib/net/tls/tls_crypto_engine.cpp index 58d99cc7108..d0475f3e88d 100644 --- a/vespalib/src/vespa/vespalib/net/tls/tls_crypto_engine.cpp +++ b/vespalib/src/vespa/vespalib/net/tls/tls_crypto_engine.cpp @@ -12,9 +12,17 @@ TlsCryptoEngine::TlsCryptoEngine(net::tls::TransportSecurityOptions tls_opts, ne } std::unique_ptr<TlsCryptoSocket> -TlsCryptoEngine::create_tls_crypto_socket(SocketHandle socket, bool is_server) +TlsCryptoEngine::create_tls_client_crypto_socket(SocketHandle socket, const SocketSpec &) { - auto mode = is_server ? net::tls::CryptoCodec::Mode::Server : net::tls::CryptoCodec::Mode::Client; + auto mode = net::tls::CryptoCodec::Mode::Client; + auto codec = net::tls::CryptoCodec::create_default_codec(_tls_ctx, SocketAddress::peer_address(socket.get()), mode); + return std::make_unique<net::tls::CryptoCodecAdapter>(std::move(socket), std::move(codec)); +} + +std::unique_ptr<TlsCryptoSocket> +TlsCryptoEngine::create_tls_server_crypto_socket(SocketHandle socket) +{ + auto mode = net::tls::CryptoCodec::Mode::Server; auto codec = net::tls::CryptoCodec::create_default_codec(_tls_ctx, SocketAddress::peer_address(socket.get()), mode); return std::make_unique<net::tls::CryptoCodecAdapter>(std::move(socket), std::move(codec)); } diff --git a/vespalib/src/vespa/vespalib/net/tls/tls_crypto_engine.h b/vespalib/src/vespa/vespalib/net/tls/tls_crypto_engine.h index dc7d7eaf9ce..5e760cf5585 100644 --- a/vespalib/src/vespa/vespalib/net/tls/tls_crypto_engine.h +++ b/vespalib/src/vespa/vespalib/net/tls/tls_crypto_engine.h @@ -11,7 +11,8 @@ namespace vespalib { class AbstractTlsCryptoEngine : public CryptoEngine { public: - virtual std::unique_ptr<TlsCryptoSocket> create_tls_crypto_socket(SocketHandle socket, bool is_server) = 0; + virtual std::unique_ptr<TlsCryptoSocket> create_tls_client_crypto_socket(SocketHandle socket, const SocketSpec &spec) = 0; + virtual std::unique_ptr<TlsCryptoSocket> create_tls_server_crypto_socket(SocketHandle socket) = 0; }; /** @@ -24,9 +25,13 @@ private: public: explicit TlsCryptoEngine(net::tls::TransportSecurityOptions tls_opts, net::tls::AuthorizationMode authz_mode = net::tls::AuthorizationMode::Enforce); - std::unique_ptr<TlsCryptoSocket> create_tls_crypto_socket(SocketHandle socket, bool is_server) override; - CryptoSocket::UP create_crypto_socket(SocketHandle socket, bool is_server) override { - return create_tls_crypto_socket(std::move(socket), is_server); + std::unique_ptr<TlsCryptoSocket> create_tls_client_crypto_socket(SocketHandle socket, const SocketSpec &spec) override; + std::unique_ptr<TlsCryptoSocket> create_tls_server_crypto_socket(SocketHandle socket) override; + CryptoSocket::UP create_client_crypto_socket(SocketHandle socket, const SocketSpec &spec) override { + return create_tls_client_crypto_socket(std::move(socket), spec); + } + CryptoSocket::UP create_server_crypto_socket(SocketHandle socket) override { + return create_tls_server_crypto_socket(std::move(socket)); } std::shared_ptr<net::tls::TlsContext> tls_context() const noexcept { return _tls_ctx; }; diff --git a/vespalib/src/vespa/vespalib/portal/portal.cpp b/vespalib/src/vespa/vespalib/portal/portal.cpp index 47719ea4c69..a6d44348e5e 100644 --- a/vespalib/src/vespa/vespalib/portal/portal.cpp +++ b/vespalib/src/vespa/vespalib/portal/portal.cpp @@ -143,7 +143,7 @@ Portal::handle_accept(portal::HandleGuard guard, SocketHandle socket) { socket.set_blocking(false); socket.set_keepalive(true); - new HttpConnection(std::move(guard), _reactor, _crypto->create_crypto_socket(std::move(socket), true), + new HttpConnection(std::move(guard), _reactor, _crypto->create_server_crypto_socket(std::move(socket)), [this](HttpConnection *conn) { handle_http(conn); diff --git a/vespalib/src/vespa/vespalib/test/make_tls_options_for_testing.cpp b/vespalib/src/vespa/vespalib/test/make_tls_options_for_testing.cpp index c685bffc23e..dcd2ced8036 100644 --- a/vespalib/src/vespa/vespalib/test/make_tls_options_for_testing.cpp +++ b/vespalib/src/vespa/vespalib/test/make_tls_options_for_testing.cpp @@ -67,6 +67,8 @@ npxYSKVCyo3a/Vo33V8/H0WgOXioKEZJxA== namespace vespalib::test { +SocketSpec local_spec("tcp/localhost:123"); + vespalib::net::tls::TransportSecurityOptions make_tls_options_for_testing() { return vespalib::net::tls::TransportSecurityOptions(ca_pem, cert_pem, key_pem); } diff --git a/vespalib/src/vespa/vespalib/test/make_tls_options_for_testing.h b/vespalib/src/vespa/vespalib/test/make_tls_options_for_testing.h index a1f1d5958f9..41e5d7cc86d 100644 --- a/vespalib/src/vespa/vespalib/test/make_tls_options_for_testing.h +++ b/vespalib/src/vespa/vespalib/test/make_tls_options_for_testing.h @@ -2,11 +2,19 @@ #pragma once +#include <vespa/vespalib/net/socket_spec.h> #include <vespa/vespalib/net/tls/transport_security_options.h> namespace vespalib::test { /** + * A socket spec representing "tcp/localhost:123". Used by unit tests + * performing hostname verification against the tls options created + * below. + **/ +extern SocketSpec local_spec; + +/** * Make security options allowing you to talk to yourself using * TLS. This is intended for testing purposes only. **/ |