diff options
31 files changed, 135 insertions, 118 deletions
diff --git a/config-application-package/src/main/java/com/yahoo/config/application/Xml.java b/config-application-package/src/main/java/com/yahoo/config/application/Xml.java index c48a41083c7..f2a837026ea 100644 --- a/config-application-package/src/main/java/com/yahoo/config/application/Xml.java +++ b/config-application-package/src/main/java/com/yahoo/config/application/Xml.java @@ -1,4 +1,4 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.config.application; import com.yahoo.config.application.api.ApplicationPackage; @@ -75,10 +75,6 @@ public class Xml { return factory.newDocumentBuilder(); } - static File getServices(File app) { - return new File(app, "services.xml"); // TODO Do not hard-code - } - static Document copyDocument(Document input) throws TransformerException { Transformer transformer = TransformerFactory.newInstance().newTransformer(); DOMSource source = new DOMSource(input); @@ -142,9 +138,7 @@ public class Xml { List<Element> children = XML.getChildren(parent, name); List<Element> allFromFiles = allElemsFromPath(app, pathFromAppRoot); for (Element fromFile : allFromFiles) { - for (Element inThatFile : XML.getChildren(fromFile, name)) { - children.add(inThatFile); - } + children.addAll(XML.getChildren(fromFile, name)); } return children; } diff --git a/config-application-package/src/test/java/com/yahoo/config/application/IncludeProcessorTest.java b/config-application-package/src/test/java/com/yahoo/config/application/IncludeProcessorTest.java index 562970c266f..f8484a8e455 100644 --- a/config-application-package/src/test/java/com/yahoo/config/application/IncludeProcessorTest.java +++ b/config-application-package/src/test/java/com/yahoo/config/application/IncludeProcessorTest.java @@ -1,15 +1,16 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.config.application; +import com.yahoo.config.application.api.ApplicationPackage; import org.junit.Test; import org.w3c.dom.Document; import org.xml.sax.SAXException; import javax.xml.parsers.DocumentBuilder; import javax.xml.parsers.ParserConfigurationException; -import javax.xml.stream.XMLStreamException; -import javax.xml.transform.*; -import java.io.*; +import javax.xml.transform.TransformerException; +import java.io.File; +import java.io.IOException; import java.nio.file.NoSuchFileException; /** @@ -72,7 +73,7 @@ public class IncludeProcessorTest { " </nodes>\n" + "</container></services>"; - Document doc = new IncludeProcessor(app).process(docBuilder.parse(Xml.getServices(app))); + Document doc = new IncludeProcessor(app).process(docBuilder.parse(getServices(app))); // System.out.println(Xml.documentAsString(doc)); TestBase.assertDocument(expected, doc); } @@ -81,7 +82,11 @@ public class IncludeProcessorTest { public void testRequiredIncludeIsDefault() throws ParserConfigurationException, IOException, SAXException, TransformerException { File app = new File("src/test/resources/multienvapp_failrequired"); DocumentBuilder docBuilder = Xml.getPreprocessDocumentBuilder(); - new IncludeProcessor(app).process(docBuilder.parse(Xml.getServices(app))); + new IncludeProcessor(app).process(docBuilder.parse(getServices(app))); + } + + static File getServices(File app) { + return new File(app, ApplicationPackage.SERVICES); } } diff --git a/config-model-api/src/main/java/com/yahoo/config/application/api/ApplicationPackage.java b/config-model-api/src/main/java/com/yahoo/config/application/api/ApplicationPackage.java index bc7dbbe2069..2aefc985f4b 100644 --- a/config-model-api/src/main/java/com/yahoo/config/application/api/ApplicationPackage.java +++ b/config-model-api/src/main/java/com/yahoo/config/application/api/ApplicationPackage.java @@ -1,4 +1,4 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.config.application.api; import com.yahoo.component.Version; @@ -87,14 +87,14 @@ public interface ApplicationPackage { /** * Contents of services.xml. Caller must close reader after use. * - * @return a Reader, or null if no services.xml/vespa-services.xml present + * @return a Reader, or null if no services.xml present */ Reader getServices(); /** * Contents of hosts.xml. Caller must close reader after use. * - * @return a Reader, or null if no hosts.xml/vespa-hosts.xml present + * @return a Reader, or null if no hosts.xml present */ Reader getHosts(); @@ -146,10 +146,11 @@ public interface ApplicationPackage { /** Returns the major version this application is valid for, or empty if it is valid for all versions */ default Optional<Integer> getMajorVersion() { - if ( ! getDeployment().isPresent()) return Optional.empty(); + if (getDeployment().isEmpty()) return Optional.empty(); Element deployElement = XML.getDocument(getDeployment().get()).getDocumentElement(); if (deployElement == null) return Optional.empty(); + String majorVersionString = deployElement.getAttribute("major-version"); if (majorVersionString == null || majorVersionString.isEmpty()) return Optional.empty(); @@ -181,7 +182,6 @@ public interface ApplicationPackage { /** Returns handle for the file containing client certificate authorities */ default ApplicationFile getClientSecurityFile() { return getFile(SECURITY_DIR.append("clients.pem")); } - //For generating error messages String getHostSource(); String getServicesSource(); diff --git a/config-model-api/src/main/java/com/yahoo/config/application/api/ValidationOverrides.java b/config-model-api/src/main/java/com/yahoo/config/application/api/ValidationOverrides.java index 8845431c71b..ea27b7f70d8 100644 --- a/config-model-api/src/main/java/com/yahoo/config/application/api/ValidationOverrides.java +++ b/config-model-api/src/main/java/com/yahoo/config/application/api/ValidationOverrides.java @@ -1,4 +1,4 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.config.application.api; import com.google.common.collect.ImmutableList; @@ -67,7 +67,7 @@ public class ValidationOverrides { public boolean allows(String validationIdString, Instant now) { Optional<ValidationId> validationId = ValidationId.from(validationIdString); - if ( ! validationId.isPresent()) return false; // unknown id -> not allowed + if (validationId.isEmpty()) return false; // unknown id -> not allowed return allows(validationId.get(), now); } @@ -125,8 +125,8 @@ public class ValidationOverrides { .atStartOfDay().atZone(ZoneOffset.UTC).toInstant() .plus(Duration.ofDays(1)); // Make the override valid *on* the "until" date Optional<ValidationId> validationId = ValidationId.from(XML.getValue(allow)); - if (validationId.isPresent()) // skip unknown ids as they may be valid for other model versions - overrides.add(new ValidationOverrides.Allow(validationId.get(), until)); + // skip unknown ids as they may be valid for other model versions + validationId.ifPresent(id -> overrides.add(new Allow(id, until))); } return new ValidationOverrides(overrides, xmlForm); } diff --git a/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java b/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java index 5158f3ec488..d2892917a2e 100644 --- a/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java +++ b/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java @@ -96,6 +96,7 @@ public interface ModelContext { @ModelFeatureFlag(owners = {"tokle", "bjorncs"}) default boolean enableCustomAclMapping() { return false; } @ModelFeatureFlag(owners = {"geirst", "vekterli"}) default int numDistributorStripes() { return 0; } @ModelFeatureFlag(owners = {"arnej"}) default boolean requireConnectivityCheck() { return false; } + @ModelFeatureFlag(owners = {"hmusum"}) default boolean throwIfResourceLimitsSpecified() { return false; } } /** Warning: As elsewhere in this package, do not make backwards incompatible changes that will break old config models! */ diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/VespaMetricSet.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/VespaMetricSet.java index c5aba8388b4..114a3e380ef 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/VespaMetricSet.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/VespaMetricSet.java @@ -765,6 +765,15 @@ public class VespaMetricSet { metrics.add(new Metric("vds.bouncer.clock_skew_aborts.count")); + metrics.add(new Metric("vds.mergethrottler.averagequeuewaitingtime.max")); + metrics.add(new Metric("vds.mergethrottler.averagequeuewaitingtime.sum")); + metrics.add(new Metric("vds.mergethrottler.averagequeuewaitingtime.count")); + metrics.add(new Metric("vds.mergethrottler.queuesize.max")); + metrics.add(new Metric("vds.mergethrottler.queuesize.sum")); + metrics.add(new Metric("vds.mergethrottler.queuesize.count")); + metrics.add(new Metric("vds.mergethrottler.bounced_due_to_back_pressure.rate")); + metrics.add(new Metric("vds.mergethrottler.locallyexecutedmerges.ok.rate")); + metrics.add(new Metric("vds.mergethrottler.mergechains.ok.rate")); return metrics; } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/ClusterResourceLimits.java b/config-model/src/main/java/com/yahoo/vespa/model/content/ClusterResourceLimits.java index 70f2acd3c7b..8db656a5f2c 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/content/ClusterResourceLimits.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/content/ClusterResourceLimits.java @@ -1,7 +1,6 @@ // Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.model.content; -import com.yahoo.config.application.api.DeployLogger; import com.yahoo.vespa.model.builder.xml.dom.ModelElement; import com.yahoo.vespa.model.content.cluster.DomResourceLimitsBuilder; @@ -37,14 +36,14 @@ public class ClusterResourceLimits { private final boolean enableFeedBlockInDistributor; private final boolean hostedVespa; - private final DeployLogger deployLogger; + private final boolean throwIfSpecified; private ResourceLimits.Builder ctrlBuilder = new ResourceLimits.Builder(); private ResourceLimits.Builder nodeBuilder = new ResourceLimits.Builder(); - public Builder(boolean enableFeedBlockInDistributor, boolean hostedVespa, DeployLogger deployLogger) { + public Builder(boolean enableFeedBlockInDistributor, boolean hostedVespa, boolean throwIfSpecified) { this.enableFeedBlockInDistributor = enableFeedBlockInDistributor; this.hostedVespa = hostedVespa; - this.deployLogger = deployLogger; + this.throwIfSpecified = throwIfSpecified; } public ClusterResourceLimits build(ModelElement clusterElem) { @@ -58,7 +57,7 @@ public class ClusterResourceLimits { private ResourceLimits.Builder createBuilder(ModelElement element) { return element == null ? new ResourceLimits.Builder() - : DomResourceLimitsBuilder.createBuilder(element, hostedVespa, deployLogger); + : DomResourceLimitsBuilder.createBuilder(element, hostedVespa, throwIfSpecified); } public void setClusterControllerBuilder(ResourceLimits.Builder builder) { diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/ContentCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/ContentCluster.java index 96f2af66131..e9264a6d9fc 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/ContentCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/ContentCluster.java @@ -123,7 +123,7 @@ public class ContentCluster extends AbstractConfigProducer<AbstractConfigProduce boolean enableFeedBlockInDistributor = deployState.getProperties().featureFlags().enableFeedBlockInDistributor(); var resourceLimits = new ClusterResourceLimits.Builder(enableFeedBlockInDistributor, stateIsHosted(deployState), - deployState.getDeployLogger()) + deployState.featureFlags().throwIfResourceLimitsSpecified()) .build(contentElement); c.clusterControllerConfig = new ClusterControllerConfig.Builder(getClusterId(contentElement), contentElement, diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/DomResourceLimitsBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/DomResourceLimitsBuilder.java index 9f4852629d0..bab991efe51 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/DomResourceLimitsBuilder.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/DomResourceLimitsBuilder.java @@ -1,12 +1,9 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.model.content.cluster; -import com.yahoo.config.application.api.DeployLogger; import com.yahoo.vespa.model.builder.xml.dom.ModelElement; import com.yahoo.vespa.model.content.ResourceLimits; -import java.util.logging.Level; - /** * Builder for feed block resource limits. * @@ -14,18 +11,14 @@ import java.util.logging.Level; */ public class DomResourceLimitsBuilder { - public static ResourceLimits.Builder createBuilder(ModelElement contentXml, boolean hostedVespa, DeployLogger deployLogger) { + public static ResourceLimits.Builder createBuilder(ModelElement contentXml, boolean hostedVespa, boolean throwIfSpecified) { ResourceLimits.Builder builder = new ResourceLimits.Builder(); ModelElement resourceLimits = contentXml.child("resource-limits"); if (resourceLimits == null) { return builder; } - if (hostedVespa) { - deployLogger.logApplicationPackage(Level.WARNING, "Element " + resourceLimits + - " is not allowed, default limits will be used"); - // TODO: Throw exception when we are sure nobody is using this - //throw new IllegalArgumentException("Element " + element + " is not allowed to be set, default limits will be used"); - return builder; - } + if (hostedVespa && throwIfSpecified) + throw new IllegalArgumentException("Element '" + resourceLimits + "' is not allowed to be set"); + if (resourceLimits.child("disk") != null) { builder.setDiskLimit(resourceLimits.childAsDouble("disk")); } diff --git a/config-model/src/test/java/com/yahoo/vespa/model/content/ClusterResourceLimitsTest.java b/config-model/src/test/java/com/yahoo/vespa/model/content/ClusterResourceLimitsTest.java index 469e4649c14..ad1f5331a91 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/content/ClusterResourceLimitsTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/content/ClusterResourceLimitsTest.java @@ -1,15 +1,16 @@ // Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.model.content; -import com.yahoo.config.application.api.DeployLogger; -import com.yahoo.config.model.application.provider.BaseDeployLogger; -import com.yahoo.searchdefinition.derived.TestableDeployLogger; import com.yahoo.text.XML; import com.yahoo.vespa.model.builder.xml.dom.ModelElement; +import org.junit.Rule; import org.junit.Test; +import org.junit.rules.ExpectedException; +import org.w3c.dom.Document; import java.util.Optional; +import static org.hamcrest.CoreMatchers.containsString; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; @@ -48,13 +49,16 @@ public class ClusterResourceLimitsTest { return this; } public ClusterResourceLimits build() { - var builder = new ClusterResourceLimits.Builder(enableFeedBlockInDistributor, false, new BaseDeployLogger()); + var builder = new ClusterResourceLimits.Builder(enableFeedBlockInDistributor, false, false); builder.setClusterControllerBuilder(ctrlBuilder); builder.setContentNodeBuilder(nodeBuilder); return builder.build(); } } + @Rule + public ExpectedException expectedException = ExpectedException.none(); + @Test public void content_node_limits_are_derived_from_cluster_controller_limits_if_not_set() { assertLimits(0.4, 0.7, 0.7, 0.85, @@ -120,26 +124,25 @@ public class ClusterResourceLimitsTest { } @Test - // TODO: Change to expect exception being thrown when no one uses this in hosted - public void default_resource_limits_when_hosted_and_warning_is_logged() { - TestableDeployLogger logger = new TestableDeployLogger(); + public void exception_is_thrown_when_resource_limits_are_specified() { final boolean hosted = true; - ClusterResourceLimits.Builder builder = new ClusterResourceLimits.Builder(true, hosted, logger); - ClusterResourceLimits limits = builder.build(new ModelElement(XML.getDocument("<cluster id=\"test\">" + - " <tuning>\n" + - " <resource-limits>\n" + - " <memory>0.92</memory>\n" + - " </resource-limits>\n" + - " </tuning>\n" + - "</cluster>") - .getDocumentElement())); - - assertLimits(0.8, 0.8, limits.getClusterControllerLimits()); - assertLimits(0.9, 0.9, limits.getContentNodeLimits()); - - assertEquals(1, logger.warnings.size()); - assertEquals("Element resource-limits is not allowed, default limits will be used", logger.warnings.get(0)); + Document clusterXml = XML.getDocument("<cluster id=\"test\">" + + " <tuning>\n" + + " <resource-limits>\n" + + " <memory>0.92</memory>\n" + + " </resource-limits>\n" + + " </tuning>\n" + + "</cluster>"); + + expectedException.expect(IllegalArgumentException.class); + expectedException.expectMessage(containsString("Element 'resource-limits' is not allowed to be set")); + ClusterResourceLimits.Builder builder = new ClusterResourceLimits.Builder(true, hosted, true); + builder.build(new ModelElement(clusterXml.getDocumentElement())); + + expectedException = ExpectedException.none(); + ClusterResourceLimits.Builder builder2 = new ClusterResourceLimits.Builder(true, hosted, false); + builder2.build(new ModelElement(clusterXml.getDocumentElement())); } private void assertLimits(Double expCtrlDisk, Double expCtrlMemory, Double expNodeDisk, Double expNodeMemory, Fixture f) { diff --git a/config-model/src/test/java/com/yahoo/vespa/model/content/FleetControllerClusterTest.java b/config-model/src/test/java/com/yahoo/vespa/model/content/FleetControllerClusterTest.java index 22e38b30959..2eecfa9440e 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/content/FleetControllerClusterTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/content/FleetControllerClusterTest.java @@ -1,12 +1,11 @@ // Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.model.content; -import com.yahoo.config.model.application.provider.BaseDeployLogger; import com.yahoo.config.model.deploy.DeployState; import com.yahoo.config.model.deploy.TestProperties; -import com.yahoo.vespa.config.content.FleetcontrollerConfig; import com.yahoo.config.model.test.MockRoot; import com.yahoo.text.XML; +import com.yahoo.vespa.config.content.FleetcontrollerConfig; import com.yahoo.vespa.model.builder.xml.dom.ModelElement; import org.junit.Test; import org.w3c.dom.Document; @@ -24,9 +23,7 @@ public class FleetControllerClusterTest { var clusterElement = new ModelElement(doc.getDocumentElement()); return new ClusterControllerConfig.Builder("storage", clusterElement, - new ClusterResourceLimits.Builder(enableFeedBlockInDistributor, - false, - new BaseDeployLogger()) + new ClusterResourceLimits.Builder(enableFeedBlockInDistributor, false, false) .build(clusterElement).getClusterControllerLimits()) .build(root.getDeployState(), root, clusterElement.getXml()); } @@ -115,7 +112,7 @@ public class FleetControllerClusterTest { assertLimits(0.8, 0.7, getConfigForResourceLimitsTuning(null, 0.7)); } - private static double DELTA = 0.00001; + private static final double DELTA = 0.00001; private void assertLimits(double expDisk, double expMemory, FleetcontrollerConfig config) { var limits = config.cluster_feed_block_limit(); diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java index 8805c339482..5cd52e36339 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java @@ -180,6 +180,7 @@ public class ModelContextImpl implements ModelContext { private final boolean requireConnectivityCheck; private final int maxConcurrentMergesPerContentNode; private final int maxMergeQueueSize; + private final boolean throwIfResourceLimitsSpecified; public FeatureFlags(FlagSource source, ApplicationId appId) { this.dedicatedClusterControllerFlavor = parseDedicatedClusterControllerFlavor(flagValue(source, appId, Flags.DEDICATED_CLUSTER_CONTROLLER_FLAVOR)); @@ -205,6 +206,7 @@ public class ModelContextImpl implements ModelContext { this.requireConnectivityCheck = flagValue(source, appId, Flags.REQUIRE_CONNECTIVITY_CHECK); this.maxConcurrentMergesPerContentNode = flagValue(source, appId, Flags.MAX_CONCURRENT_MERGES_PER_NODE); this.maxMergeQueueSize = flagValue(source, appId, Flags.MAX_MERGE_QUEUE_SIZE); + this.throwIfResourceLimitsSpecified = flagValue(source, appId, Flags.THROW_EXCEPTION_IF_RESOURCE_LIMITS_SPECIFIED); } @Override public Optional<NodeResources> dedicatedClusterControllerFlavor() { return Optional.ofNullable(dedicatedClusterControllerFlavor); } @@ -232,6 +234,7 @@ public class ModelContextImpl implements ModelContext { @Override public boolean requireConnectivityCheck() { return requireConnectivityCheck; } @Override public int maxConcurrentMergesPerNode() { return maxConcurrentMergesPerContentNode; } @Override public int maxMergeQueueSize() { return maxMergeQueueSize; } + @Override public boolean throwIfResourceLimitsSpecified() { return throwIfResourceLimitsSpecified; } private static <V> V flagValue(FlagSource source, ApplicationId appId, UnboundFlag<? extends V, ?, ?> flag) { return flag.bindTo(source) diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ZooKeeperClient.java b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ZooKeeperClient.java index 42f4dbd5762..f0a63757477 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ZooKeeperClient.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ZooKeeperClient.java @@ -35,7 +35,7 @@ import static com.yahoo.vespa.config.server.zookeeper.ConfigCurator.USERAPP_ZK_S import static com.yahoo.vespa.config.server.zookeeper.ConfigCurator.USER_DEFCONFIGS_ZK_SUBPATH; /** - * A class used for reading and writing application data to zookeeper. + * Reads and writes application package to and from ZooKeeper. * * @author hmusum */ @@ -76,13 +76,13 @@ public class ZooKeeperClient { * * @param app the application package to feed to zookeeper */ - void write(ApplicationPackage app) { + void writeApplicationPackage(ApplicationPackage app) { try { writeUserDefs(app); writeSomeOf(app); writeSchemas(app); writeUserIncludeDirs(app, app.getUserIncludeDirs()); - write(app.getMetaData()); + writeMetadata(app.getMetaData()); } catch (Exception e) { throw new IllegalStateException("Unable to write vespa model to config server(s) " + System.getProperty("configsources") + "\n" + "Please ensure that config server is started " + @@ -153,7 +153,6 @@ public class ZooKeeperClient { for (ApplicationFile file : listFiles(dir, filenameFilter)) { String name = file.getPath().getName(); if (name.startsWith(".")) continue; //.svn , .git ... - if ("CVS".equals(name)) continue; if (file.isDirectory()) { configCurator.createNode(path.append(name).getAbsolute()); if (recurse) { @@ -198,7 +197,6 @@ public class ZooKeeperClient { } private void writeUserIncludeDirs(ApplicationPackage applicationPackage, List<String> userIncludeDirs) throws IOException { - // User defined include directories for (String userInclude : userIncludeDirs) { ApplicationFile dir = applicationPackage.getFile(Path.fromString(userInclude)); final List<ApplicationFile> files = dir.listFiles(); @@ -238,12 +236,12 @@ public class ZooKeeperClient { } /** - * Feeds application metadata to zookeeper. Used by vespamodel to create config - * for application metadata (used by ApplicationStatusHandler) + * Feeds application metadata to zookeeper. Used by config model to create config + * for application metadata * * @param metaData The application metadata. */ - private void write(ApplicationMetaData metaData) { + private void writeMetadata(ApplicationMetaData metaData) { configCurator.putData(getZooKeeperAppPath(META_ZK_PATH).getAbsolute(), metaData.asJsonBytes()); } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ZooKeeperDeployer.java b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ZooKeeperDeployer.java index 12aa5b7cc35..8c7d6ea28dd 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ZooKeeperDeployer.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ZooKeeperDeployer.java @@ -33,7 +33,7 @@ public class ZooKeeperDeployer { public void deploy(ApplicationPackage applicationPackage, Map<Version, FileRegistry> fileRegistryMap, AllocatedHosts allocatedHosts) throws IOException { zooKeeperClient.initialize(); - zooKeeperClient.write(applicationPackage); + zooKeeperClient.writeApplicationPackage(applicationPackage); zooKeeperClient.write(fileRegistryMap); zooKeeperClient.write(allocatedHosts); } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/session/PrepareParams.java b/configserver/src/main/java/com/yahoo/vespa/config/server/session/PrepareParams.java index 1b43e57c01a..071a0dd8f0c 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/session/PrepareParams.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/session/PrepareParams.java @@ -306,7 +306,7 @@ public final class PrepareParams { .athenzDomain(SlimeUtils.optionalString(params.field(ATHENZ_DOMAIN)).orElse(null)) .applicationRoles(ApplicationRoles.fromString(SlimeUtils.optionalString(params.field(APPLICATION_HOST_ROLE)).orElse(null), SlimeUtils.optionalString(params.field(APPLICATION_CONTAINER_ROLE)).orElse(null))) .quota(deserialize(params.field(QUOTA_PARAM_NAME), Quota::fromSlime)) - .tenantSecretStores(SlimeUtils.optionalString(params.field(TENANT_SECRET_STORES_PARAM_NAME)).orElse(null)) + .tenantSecretStores(deserialize(params.field(TENANT_SECRET_STORES_PARAM_NAME), TenantSecretStoreSerializer::listFromSlime, List.of())) .force(booleanValue(params, FORCE_PARAM_NAME)) .waitForResourcesInPrepare(booleanValue(params, WAIT_FOR_RESOURCES_IN_PREPARE)) .withOperatorCertificates(deserialize(params.field(OPERATOR_CERTIFICATES), PrepareParams::readOperatorCertificates, Collections.emptyList())) diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/deploy/ZooKeeperClientTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/deploy/ZooKeeperClientTest.java index 7d14b1996b0..e20363af4e9 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/deploy/ZooKeeperClientTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/deploy/ZooKeeperClientTest.java @@ -64,7 +64,7 @@ public class ZooKeeperClientTest { Map<Version, FileRegistry> fileRegistries = createFileRegistries(); app.writeMetaData(); zkc.initialize(); - zkc.write(app); + zkc.writeApplicationPackage(app); zkc.write(fileRegistries); } diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/session/PrepareParamsTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/session/PrepareParamsTest.java index 08794cf0b78..f68e79ae266 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/session/PrepareParamsTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/session/PrepareParamsTest.java @@ -4,6 +4,7 @@ package com.yahoo.vespa.config.server.session; import com.yahoo.config.model.api.ApplicationRoles; import com.yahoo.config.model.api.ContainerEndpoint; import com.yahoo.config.model.api.EndpointCertificateMetadata; +import com.yahoo.config.model.api.TenantSecretStore; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.TenantName; import com.yahoo.container.jdisc.HttpRequest; @@ -24,6 +25,7 @@ import com.yahoo.slime.SlimeInserter; import com.yahoo.slime.SlimeUtils; import com.yahoo.vespa.config.server.tenant.ContainerEndpointSerializer; import com.yahoo.vespa.config.server.tenant.EndpointCertificateMetadataSerializer; +import com.yahoo.vespa.config.server.tenant.TenantSecretStoreSerializer; import org.junit.Test; import javax.security.auth.x500.X500Principal; @@ -203,6 +205,26 @@ public class PrepareParamsTest { assertEquals(certificate, prepareParams.operatorCertificates().get(0)); } + @Test + public void testSecretStores() throws IOException { + List<TenantSecretStore> secretStores = List.of(new TenantSecretStore("name", "awsId", "role")); + Slime secretStoreSlime = TenantSecretStoreSerializer.toSlime(secretStores); + String secretStoreParam = new String(SlimeUtils.toJsonBytes(secretStoreSlime), StandardCharsets.UTF_8); + + var prepareParams = createParams(request + "&" + PrepareParams.TENANT_SECRET_STORES_PARAM_NAME + "=" + URLEncoder.encode(secretStoreParam, StandardCharsets.UTF_8), TenantName.from("foo")); + assertEquals(1, prepareParams.tenantSecretStores().size()); + TenantSecretStore tenantSecretStore = prepareParams.tenantSecretStores().get(0); + assertEquals("name", tenantSecretStore.getName()); + assertEquals("awsId", tenantSecretStore.getAwsId()); + assertEquals("role", tenantSecretStore.getRole()); + + // Verify using json object + var root = SlimeUtils.jsonToSlime(json); + new Injector().inject(secretStoreSlime.get(), new ObjectInserter(root.get(), PrepareParams.TENANT_SECRET_STORES_PARAM_NAME)); + PrepareParams prepareParamsJson = PrepareParams.fromJson(SlimeUtils.toJsonBytes(root), TenantName.from("foo"), Duration.ofSeconds(60)); + assertPrepareParamsEqual(prepareParams, prepareParamsJson); + } + private void assertPrepareParamsEqual(PrepareParams urlParams, PrepareParams jsonParams) { assertEquals(urlParams.ignoreValidationErrors(), jsonParams.ignoreValidationErrors()); assertEquals(urlParams.isDryRun(), jsonParams.isDryRun()); diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/zookeeper/ZKApplicationPackageTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/zookeeper/ZKApplicationPackageTest.java index a71f75f5035..458cdb82066 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/zookeeper/ZKApplicationPackageTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/zookeeper/ZKApplicationPackageTest.java @@ -124,8 +124,7 @@ public class ZKApplicationPackageTest { } /** - * Takes for instance the dir /app and puts the contents into the given ZK path. Ignores files starting with dot, - * and dirs called CVS. + * Takes for instance the dir /app and puts the contents into the given ZK path. Ignores files starting with dot. * * @param dir directory which holds the summary class part files * @param path zookeeper path @@ -142,7 +141,6 @@ public class ZKApplicationPackageTest { } for (File file : listFiles(dir, filenameFilter)) { if (file.getName().startsWith(".")) continue; //.svn , .git ... - if ("CVS".equals(file.getName())) continue; if (file.isFile()) { String contents = IOUtils.readFile(file); zk.putData(path, file.getName(), contents); diff --git a/flags/src/main/java/com/yahoo/vespa/flags/Flags.java b/flags/src/main/java/com/yahoo/vespa/flags/Flags.java index fdf2acb839e..9904ef77513 100644 --- a/flags/src/main/java/com/yahoo/vespa/flags/Flags.java +++ b/flags/src/main/java/com/yahoo/vespa/flags/Flags.java @@ -139,13 +139,13 @@ public class Flags { public static final UnboundBooleanFlag ENCRYPT_DISK = defineFeatureFlag( "encrypt-disk", false, - List.of("hakonhall"), "2021-05-05", "2021-06-05", + List.of("hakonhall"), "2021-05-05", "2021-08-05", "Allow migrating an unencrypted data partition to being encrypted.", "Takes effect on next host-admin tick."); public static final UnboundBooleanFlag ENCRYPT_DIRTY_DISK = defineFeatureFlag( "encrypt-dirty-disk", false, - List.of("hakonhall"), "2021-05-14", "2021-06-05", + List.of("hakonhall"), "2021-05-14", "2021-08-05", "Allow migrating an unencrypted data partition to being encrypted when (de)provisioned.", "Takes effect on next host-admin tick."); @@ -266,6 +266,13 @@ public class Flags { "Takes effect on next restart", ZONE_ID, APPLICATION_ID); + public static final UnboundBooleanFlag THROW_EXCEPTION_IF_RESOURCE_LIMITS_SPECIFIED = defineFeatureFlag( + "throw-exception-if-resource-limits-specified", false, + List.of("mpolden"), "2021-06-07", "2021-07-07", + "Whether to throw an exception in hosted Vespa if the application specifies resource limits in services.xml", + "Takes effect on next deployment through controller", + APPLICATION_ID); + /** WARNING: public for testing: All flags should be defined in {@link Flags}. */ public static UnboundBooleanFlag defineFeatureFlag(String flagId, boolean defaultValue, List<String> owners, String createdAt, String expiresAt, String description, diff --git a/metrics/src/vespa/metrics/countmetric.h b/metrics/src/vespa/metrics/countmetric.h index 02a6827d1ce..1701071104e 100644 --- a/metrics/src/vespa/metrics/countmetric.h +++ b/metrics/src/vespa/metrics/countmetric.h @@ -105,7 +105,7 @@ public: void addToSnapshot(Metric&, std::vector<Metric::UP> &) const override; }; -typedef CountMetric<uint64_t, true> LongCountMetric; +using LongCountMetric = CountMetric<uint64_t, true>; } // metrics diff --git a/metrics/src/vespa/metrics/metricvalueset.h b/metrics/src/vespa/metrics/metricvalueset.h index 2463990378e..c522876f5b1 100644 --- a/metrics/src/vespa/metrics/metricvalueset.h +++ b/metrics/src/vespa/metrics/metricvalueset.h @@ -76,12 +76,6 @@ public: */ bool setValues(const ValueClass& values); - /** - * Retrieve and reset in a single operation, to minimize chance of - * alteration in the process. - */ - ValueClass getValuesAndReset(); - void reset() { setFlag(RESET); } @@ -105,9 +99,6 @@ public: _flags.store(_flags.load(std::memory_order_relaxed) & ~flags, std::memory_order_relaxed); } - uint32_t getFlags() const { - return _flags.load(std::memory_order_relaxed); - } }; } // metrics diff --git a/metrics/src/vespa/metrics/metricvalueset.hpp b/metrics/src/vespa/metrics/metricvalueset.hpp index 8c5b32afcf8..57b3e7f9901 100644 --- a/metrics/src/vespa/metrics/metricvalueset.hpp +++ b/metrics/src/vespa/metrics/metricvalueset.hpp @@ -70,14 +70,6 @@ MetricValueSet<ValueClass>::setValues(const ValueClass& values) { } template<typename ValueClass> -ValueClass -MetricValueSet<ValueClass>::getValuesAndReset() { - ValueClass result(getValues()); - setFlag(RESET); - return result; -} - -template<typename ValueClass> std::string MetricValueSet<ValueClass>::toString() { std::ostringstream ost; diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/NodeRepository.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/NodeRepository.java index d113ca68d01..f084b83bf97 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/NodeRepository.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/NodeRepository.java @@ -109,7 +109,7 @@ public class NodeRepository extends AbstractComponent { "dynamicProvisioning property must be 1-to-1 with availability of HostProvisioner, was: dynamicProvisioning=%s, hostProvisioner=%s", zone.getCloud().dynamicProvisioning(), provisionServiceProvider.getHostProvisioner().map(__ -> "present").orElse("empty"))); - this.db = new CuratorDatabaseClient(flavors, curator, clock, zone, useCuratorClientCache, nodeCacheSize); + this.db = new CuratorDatabaseClient(flavors, curator, clock, useCuratorClientCache, nodeCacheSize); this.zone = zone; this.clock = clock; this.nodes = new Nodes(db, zone, clock); diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/CuratorDatabaseClient.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/CuratorDatabaseClient.java index d1f881f8b7a..fb4799be29c 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/CuratorDatabaseClient.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/CuratorDatabaseClient.java @@ -11,7 +11,6 @@ import com.yahoo.config.provision.HostName; import com.yahoo.config.provision.NodeFlavors; import com.yahoo.config.provision.NodeType; import com.yahoo.config.provision.TenantName; -import com.yahoo.config.provision.Zone; import com.yahoo.path.Path; import com.yahoo.transaction.NestedTransaction; import com.yahoo.transaction.Transaction; @@ -77,13 +76,10 @@ public class CuratorDatabaseClient { private final NodeSerializer nodeSerializer; private final CuratorDatabase db; private final Clock clock; - private final Zone zone; private final CuratorCounter provisionIndexCounter; - public CuratorDatabaseClient(NodeFlavors flavors, Curator curator, Clock clock, Zone zone, boolean useCache, - long nodeCacheSize) { + public CuratorDatabaseClient(NodeFlavors flavors, Curator curator, Clock clock, boolean useCache, long nodeCacheSize) { this.nodeSerializer = new NodeSerializer(flavors, nodeCacheSize); - this.zone = zone; this.db = new CuratorDatabase(curator, root, useCache); this.clock = clock; this.provisionIndexCounter = new CuratorCounter(curator, root.append("provisionIndexCounter").getAbsolute()); diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/persistence/CuratorDatabaseClientTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/persistence/CuratorDatabaseClientTest.java index f47fb7f23be..99f6ce4fb00 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/persistence/CuratorDatabaseClientTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/persistence/CuratorDatabaseClientTest.java @@ -1,4 +1,4 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.provision.persistence; import com.yahoo.config.provision.ApplicationId; @@ -6,7 +6,6 @@ import com.yahoo.config.provision.ApplicationName; import com.yahoo.config.provision.InstanceName; import com.yahoo.config.provision.NodeType; import com.yahoo.config.provision.TenantName; -import com.yahoo.config.provision.Zone; import com.yahoo.vespa.curator.Curator; import com.yahoo.vespa.curator.mock.MockCurator; import com.yahoo.vespa.hosted.provision.Node; @@ -25,7 +24,7 @@ public class CuratorDatabaseClientTest { private final Curator curator = new MockCurator(); private final CuratorDatabaseClient zkClient = new CuratorDatabaseClient( - FlavorConfigBuilder.createDummies("default"), curator, Clock.systemUTC(), Zone.defaultZone(), true, 1000); + FlavorConfigBuilder.createDummies("default"), curator, Clock.systemUTC(), true, 1000); @Test public void can_read_stored_host_information() throws Exception { diff --git a/searchlib/src/vespa/searchlib/attribute/postingstore.cpp b/searchlib/src/vespa/searchlib/attribute/postingstore.cpp index 6c62e650345..477917debf0 100644 --- a/searchlib/src/vespa/searchlib/attribute/postingstore.cpp +++ b/searchlib/src/vespa/searchlib/attribute/postingstore.cpp @@ -696,7 +696,10 @@ PostingStore<DataT>::move(EntryRef ref) if (!_store.getCompacting(ref)) { return ref; } - return allocBitVectorCopy(*bve).ref; + auto new_ref = allocBitVectorCopy(*bve).ref; + _bvs.erase(ref.ref()); + _bvs.insert(new_ref.ref()); + return new_ref; } else { if (!_store.getCompacting(ref)) { return ref; diff --git a/searchlib/src/vespa/searchlib/tensor/hnsw_index.cpp b/searchlib/src/vespa/searchlib/tensor/hnsw_index.cpp index 465793739ff..545ee7cfa96 100644 --- a/searchlib/src/vespa/searchlib/tensor/hnsw_index.cpp +++ b/searchlib/src/vespa/searchlib/tensor/hnsw_index.cpp @@ -25,8 +25,8 @@ namespace { // TODO: Move this to MemoryAllocator, with name PAGE_SIZE. constexpr size_t small_page_size = 4_Ki; -constexpr size_t min_num_arrays_for_new_buffer = 64_Ki; -constexpr float alloc_grow_factor = 0.2; +constexpr size_t min_num_arrays_for_new_buffer = 512_Ki; +constexpr float alloc_grow_factor = 0.3; // TODO: Adjust these numbers to what we accept as max in config. constexpr size_t max_level_array_size = 16; constexpr size_t max_link_array_size = 64; diff --git a/storage/src/tests/storageserver/mergethrottlertest.cpp b/storage/src/tests/storageserver/mergethrottlertest.cpp index 12ed9ead1b6..3a153fef9c3 100644 --- a/storage/src/tests/storageserver/mergethrottlertest.cpp +++ b/storage/src/tests/storageserver/mergethrottlertest.cpp @@ -1233,6 +1233,7 @@ TEST_F(MergeThrottlerTest, busy_returned_on_full_queue) { // Wait till we have maxPending replies and maxQueue queued _topLinks[0]->waitForMessages(maxPending, _messageWaitTime); + EXPECT_EQ(19, _throttlers[0]->getMetrics().queueSize.getMaximum()); waitUntilMergeQueueIs(*_throttlers[0], maxQueue, _messageWaitTime); // Clear all forwarded merges diff --git a/storage/src/vespa/storage/storageserver/mergethrottler.cpp b/storage/src/vespa/storage/storageserver/mergethrottler.cpp index dc8457769a2..a9bd7ade270 100644 --- a/storage/src/vespa/storage/storageserver/mergethrottler.cpp +++ b/storage/src/vespa/storage/storageserver/mergethrottler.cpp @@ -71,6 +71,7 @@ MergeThrottler::ChainedMergeState::~ChainedMergeState() = default; MergeThrottler::Metrics::Metrics(metrics::MetricSet* owner) : metrics::MetricSet("mergethrottler", {}, "", owner), averageQueueWaitingTime("averagequeuewaitingtime", {}, "Average time a merge spends in the throttler queue", this), + queueSize("queuesize", {}, "Length of merge queue", this), bounced_due_to_back_pressure("bounced_due_to_back_pressure", {}, "Number of merges bounced due to resource exhaustion back-pressure", this), chaining("mergechains", this), local("locallyexecutedmerges", this) @@ -415,6 +416,7 @@ MergeThrottler::enqueueMerge( if (!validateNewMerge(mergeCmd, nodeSeq, msgGuard)) { return; } + _metrics->queueSize.set(_queue.size()); _queue.insert(MergePriorityQueue::value_type(msg, _queueSequence++)); } diff --git a/storage/src/vespa/storage/storageserver/mergethrottler.h b/storage/src/vespa/storage/storageserver/mergethrottler.h index e8815eee680..0c608f29196 100644 --- a/storage/src/vespa/storage/storageserver/mergethrottler.h +++ b/storage/src/vespa/storage/storageserver/mergethrottler.h @@ -57,12 +57,13 @@ public: MergeFailureMetrics failures; MergeOperationMetrics(const std::string& name, metrics::MetricSet* owner); - ~MergeOperationMetrics(); + ~MergeOperationMetrics() override; }; class Metrics : public metrics::MetricSet { public: metrics::DoubleAverageMetric averageQueueWaitingTime; + metrics::LongValueMetric queueSize; metrics::LongCountMetric bounced_due_to_back_pressure; MergeOperationMetrics chaining; MergeOperationMetrics local; diff --git a/vespa-feed-client/src/main/java/ai/vespa/feed/client/HttpRequestStrategy.java b/vespa-feed-client/src/main/java/ai/vespa/feed/client/HttpRequestStrategy.java index 7a6e2120be6..a805c7eb195 100644 --- a/vespa-feed-client/src/main/java/ai/vespa/feed/client/HttpRequestStrategy.java +++ b/vespa-feed-client/src/main/java/ai/vespa/feed/client/HttpRequestStrategy.java @@ -63,12 +63,15 @@ class HttpRequestStrategy implements RequestStrategy { this.maxInflight = builder.connectionsPerEndpoint * (long) builder.maxStreamsPerConnection; this.minInflight = builder.connectionsPerEndpoint * (long) min(16, builder.maxStreamsPerConnection); this.targetInflightX10 = new AtomicLong(10 * (long) (Math.sqrt(minInflight) * Math.sqrt(maxInflight))); - new Thread(this::dispatch, "feed-client-dispatcher").start(); + + Thread dispatcher = new Thread(this::dispatch, "feed-client-dispatcher"); + dispatcher.setDaemon(true); + dispatcher.start(); } private void dispatch() { try { - while (breaker.state() != OPEN) { + while (breaker.state() != OPEN && ! destroyed.get()) { while ( ! isInExcess() && poll() && breaker.state() == CLOSED); // Sleep when circuit is half-open, nap when queue is empty, or we are throttled. Thread.sleep(breaker.state() == HALF_OPEN ? 1000 : 10); |