diff options
487 files changed, 5450 insertions, 2206 deletions
diff --git a/.github/ISSUE_TEMPLATE/bug_report.md b/.github/ISSUE_TEMPLATE/bug_report.md new file mode 100644 index 00000000000..ec1c19396ff --- /dev/null +++ b/.github/ISSUE_TEMPLATE/bug_report.md @@ -0,0 +1,34 @@ +--- +name: Bug report +about: Create a report to help us improve +title: '' +labels: '' +assignees: '' + +--- + +**Describe the bug** +A clear and concise description of what the bug is. + +**To Reproduce** +Steps to reproduce the behavior: +1. Used sample application '....' +2. In step 4, X failed when '....' +3. See error + +**Expected behavior** +A clear and concise description of what you expected to happen. + +**Screenshots** +If applicable, add screenshots to help explain your problem. + +**Environment (please complete the following information):** + - OS: [e.g. iOS] + - Infrastructure: Kubernetes, Azure, AWS, self-hosted + - Versions [e.g. 22] + +**Vespa version** +Include Vespa version(s) used. + +**Additional context** +Add any other context about the problem here. diff --git a/.github/ISSUE_TEMPLATE/config.yml b/.github/ISSUE_TEMPLATE/config.yml new file mode 100644 index 00000000000..e7852210616 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/config.yml @@ -0,0 +1,4 @@ +contact_links: + - name: Vespa.ai support + url: https://vespa.ai/support + about: Find resources like a FAQ, use cases, Stack Overflow and live chats to help resolve your issue diff --git a/.github/ISSUE_TEMPLATE/feature_request.md b/.github/ISSUE_TEMPLATE/feature_request.md new file mode 100644 index 00000000000..bbcbbe7d615 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/feature_request.md @@ -0,0 +1,20 @@ +--- +name: Feature request +about: Suggest an idea for this project +title: '' +labels: '' +assignees: '' + +--- + +**Is your feature request related to a problem? Please describe.** +A clear and concise description of what the problem is. Ex. I'm always frustrated when [...] + +**Describe the solution you'd like** +A clear and concise description of what you want to happen. + +**Describe alternatives you've considered** +A clear and concise description of any alternative solutions or features you've considered. + +**Additional context** +Add any other context or screenshots about the feature request here. diff --git a/application-preprocessor/src/test/resources/simple/searchdefinitions/music.sd b/application-preprocessor/src/test/resources/simple/schemas/music.sd index c0190a4a7a8..c0190a4a7a8 100644 --- a/application-preprocessor/src/test/resources/simple/searchdefinitions/music.sd +++ b/application-preprocessor/src/test/resources/simple/schemas/music.sd diff --git a/application/src/test/app-packages/withcontent/searchdefinitions/mydoc.sd b/application/src/test/app-packages/withcontent/schemas/mydoc.sd index 3a0b829d60c..3a0b829d60c 100644 --- a/application/src/test/app-packages/withcontent/searchdefinitions/mydoc.sd +++ b/application/src/test/app-packages/withcontent/schemas/mydoc.sd diff --git a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/rpc/SlobrokClient.java b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/rpc/SlobrokClient.java index e180016f286..b4e9a760d8e 100644 --- a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/rpc/SlobrokClient.java +++ b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/rpc/SlobrokClient.java @@ -25,7 +25,7 @@ import java.util.logging.Logger; public class SlobrokClient implements NodeLookup { - public static Logger log = Logger.getLogger(SlobrokClient.class.getName()); + public static final Logger log = Logger.getLogger(SlobrokClient.class.getName()); private final Timer timer; private String[] connectionSpecs; diff --git a/config-application-package/src/main/java/com/yahoo/config/application/Xml.java b/config-application-package/src/main/java/com/yahoo/config/application/Xml.java index c48a41083c7..f2a837026ea 100644 --- a/config-application-package/src/main/java/com/yahoo/config/application/Xml.java +++ b/config-application-package/src/main/java/com/yahoo/config/application/Xml.java @@ -1,4 +1,4 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.config.application; import com.yahoo.config.application.api.ApplicationPackage; @@ -75,10 +75,6 @@ public class Xml { return factory.newDocumentBuilder(); } - static File getServices(File app) { - return new File(app, "services.xml"); // TODO Do not hard-code - } - static Document copyDocument(Document input) throws TransformerException { Transformer transformer = TransformerFactory.newInstance().newTransformer(); DOMSource source = new DOMSource(input); @@ -142,9 +138,7 @@ public class Xml { List<Element> children = XML.getChildren(parent, name); List<Element> allFromFiles = allElemsFromPath(app, pathFromAppRoot); for (Element fromFile : allFromFiles) { - for (Element inThatFile : XML.getChildren(fromFile, name)) { - children.add(inThatFile); - } + children.addAll(XML.getChildren(fromFile, name)); } return children; } diff --git a/config-application-package/src/main/java/com/yahoo/config/model/application/provider/FilesApplicationPackage.java b/config-application-package/src/main/java/com/yahoo/config/model/application/provider/FilesApplicationPackage.java index db875f669fe..81fbc764bb6 100644 --- a/config-application-package/src/main/java/com/yahoo/config/model/application/provider/FilesApplicationPackage.java +++ b/config-application-package/src/main/java/com/yahoo/config/model/application/provider/FilesApplicationPackage.java @@ -301,7 +301,7 @@ public class FilesApplicationPackage implements ApplicationPackage { } @Override - public Collection<NamedReader> searchDefinitionContents() { + public Collection<NamedReader> getSchemas() { Set<NamedReader> ret = new LinkedHashSet<>(); try { for (File f : getSearchDefinitionFiles()) { @@ -575,11 +575,6 @@ public class FilesApplicationPackage implements ApplicationPackage { IOUtils.writeFile(metaFile, metaData.asJsonBytes()); } - @Override - public Collection<NamedReader> getSearchDefinitions() { - return searchDefinitionContents(); - } - private void preprocessXML(File destination, File inputXml, Zone zone) throws IOException { if ( ! inputXml.exists()) return; try { diff --git a/config-application-package/src/test/java/com/yahoo/config/application/IncludeProcessorTest.java b/config-application-package/src/test/java/com/yahoo/config/application/IncludeProcessorTest.java index 562970c266f..f8484a8e455 100644 --- a/config-application-package/src/test/java/com/yahoo/config/application/IncludeProcessorTest.java +++ b/config-application-package/src/test/java/com/yahoo/config/application/IncludeProcessorTest.java @@ -1,15 +1,16 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.config.application; +import com.yahoo.config.application.api.ApplicationPackage; import org.junit.Test; import org.w3c.dom.Document; import org.xml.sax.SAXException; import javax.xml.parsers.DocumentBuilder; import javax.xml.parsers.ParserConfigurationException; -import javax.xml.stream.XMLStreamException; -import javax.xml.transform.*; -import java.io.*; +import javax.xml.transform.TransformerException; +import java.io.File; +import java.io.IOException; import java.nio.file.NoSuchFileException; /** @@ -72,7 +73,7 @@ public class IncludeProcessorTest { " </nodes>\n" + "</container></services>"; - Document doc = new IncludeProcessor(app).process(docBuilder.parse(Xml.getServices(app))); + Document doc = new IncludeProcessor(app).process(docBuilder.parse(getServices(app))); // System.out.println(Xml.documentAsString(doc)); TestBase.assertDocument(expected, doc); } @@ -81,7 +82,11 @@ public class IncludeProcessorTest { public void testRequiredIncludeIsDefault() throws ParserConfigurationException, IOException, SAXException, TransformerException { File app = new File("src/test/resources/multienvapp_failrequired"); DocumentBuilder docBuilder = Xml.getPreprocessDocumentBuilder(); - new IncludeProcessor(app).process(docBuilder.parse(Xml.getServices(app))); + new IncludeProcessor(app).process(docBuilder.parse(getServices(app))); + } + + static File getServices(File app) { + return new File(app, ApplicationPackage.SERVICES); } } diff --git a/config-application-package/src/test/resources/app-pinning-major-version/searchdefinitions/music.sd b/config-application-package/src/test/resources/app-pinning-major-version/schemas/music.sd index c0190a4a7a8..c0190a4a7a8 100644 --- a/config-application-package/src/test/resources/app-pinning-major-version/searchdefinitions/music.sd +++ b/config-application-package/src/test/resources/app-pinning-major-version/schemas/music.sd diff --git a/config-application-package/src/test/resources/app-with-deployment/searchdefinitions/music.sd b/config-application-package/src/test/resources/app-with-deployment/schemas/music.sd index c0190a4a7a8..c0190a4a7a8 100644 --- a/config-application-package/src/test/resources/app-with-deployment/searchdefinitions/music.sd +++ b/config-application-package/src/test/resources/app-with-deployment/schemas/music.sd diff --git a/config-application-package/src/test/resources/multienvapp/searchdefinitions/music.sd b/config-application-package/src/test/resources/multienvapp/schemas/music.sd index c0190a4a7a8..c0190a4a7a8 100644 --- a/config-application-package/src/test/resources/multienvapp/searchdefinitions/music.sd +++ b/config-application-package/src/test/resources/multienvapp/schemas/music.sd diff --git a/config-model-api/abi-spec.json b/config-model-api/abi-spec.json index bdf2b53bc92..58d2693aace 100644 --- a/config-model-api/abi-spec.json +++ b/config-model-api/abi-spec.json @@ -103,7 +103,7 @@ "public abstract java.io.Reader getHosts()", "public java.util.List getUserIncludeDirs()", "public void validateIncludeDir(java.lang.String)", - "public abstract java.util.Collection searchDefinitionContents()", + "public java.util.Collection searchDefinitionContents()", "public abstract java.util.Map getAllExistingConfigDefs()", "public abstract java.util.List getFiles(com.yahoo.path.Path, java.lang.String, boolean)", "public java.util.List getFiles(com.yahoo.path.Path, java.lang.String)", @@ -127,7 +127,8 @@ "public void writeMetaData()", "public java.util.Optional getAllocatedHosts()", "public java.util.Map getFileRegistries()", - "public abstract java.util.Collection getSearchDefinitions()", + "public java.util.Collection getSearchDefinitions()", + "public abstract java.util.Collection getSchemas()", "public com.yahoo.config.application.api.ApplicationPackage preprocess(com.yahoo.config.provision.Zone, com.yahoo.config.application.api.DeployLogger)" ], "fields": [ diff --git a/config-model-api/src/main/java/com/yahoo/config/application/api/ApplicationPackage.java b/config-model-api/src/main/java/com/yahoo/config/application/api/ApplicationPackage.java index d97ff5ca774..2aefc985f4b 100644 --- a/config-model-api/src/main/java/com/yahoo/config/application/api/ApplicationPackage.java +++ b/config-model-api/src/main/java/com/yahoo/config/application/api/ApplicationPackage.java @@ -1,4 +1,4 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.config.application.api; import com.yahoo.component.Version; @@ -79,7 +79,7 @@ public interface ApplicationPackage { * @return the name of the application (i.e the directory where the application package was deployed from) * @deprecated do not use */ - @Deprecated // TODO: Remove on Vespa 8 + @Deprecated // TODO: Remove in Vespa 8 String getApplicationName(); ApplicationId getApplicationId(); @@ -87,14 +87,14 @@ public interface ApplicationPackage { /** * Contents of services.xml. Caller must close reader after use. * - * @return a Reader, or null if no services.xml/vespa-services.xml present + * @return a Reader, or null if no services.xml present */ Reader getServices(); /** * Contents of hosts.xml. Caller must close reader after use. * - * @return a Reader, or null if no hosts.xml/vespa-hosts.xml present + * @return a Reader, or null if no hosts.xml present */ Reader getHosts(); @@ -113,9 +113,12 @@ public interface ApplicationPackage { /** * Readers for all the search definition files for this. + * @deprecated use {@link #getSchemas()} instead * @return a list of readers for search definitions */ - Collection<NamedReader> searchDefinitionContents(); + @Deprecated + // TODO: Remove in Vespa 8 + default Collection<NamedReader> searchDefinitionContents() { return getSchemas(); } /** * Returns all the config definitions available in this package as unparsed data. @@ -143,10 +146,11 @@ public interface ApplicationPackage { /** Returns the major version this application is valid for, or empty if it is valid for all versions */ default Optional<Integer> getMajorVersion() { - if ( ! getDeployment().isPresent()) return Optional.empty(); + if (getDeployment().isEmpty()) return Optional.empty(); Element deployElement = XML.getDocument(getDeployment().get()).getDocumentElement(); if (deployElement == null) return Optional.empty(); + String majorVersionString = deployElement.getAttribute("major-version"); if (majorVersionString == null || majorVersionString.isEmpty()) return Optional.empty(); @@ -178,7 +182,6 @@ public interface ApplicationPackage { /** Returns handle for the file containing client certificate authorities */ default ApplicationFile getClientSecurityFile() { return getFile(SECURITY_DIR.append("clients.pem")); } - //For generating error messages String getHostSource(); String getServicesSource(); @@ -235,7 +238,18 @@ public interface ApplicationPackage { return Collections.emptyMap(); } - Collection<NamedReader> getSearchDefinitions(); + /** + * @deprecated use {@link #getSchemas()} instead + */ + @Deprecated + // TODO: Remove in Vespa 8 + default Collection<NamedReader> getSearchDefinitions() { return getSchemas(); } + + /** + * Readers for all the schema files. + * @return a collection of readers for schemas + */ + Collection<NamedReader> getSchemas(); /** * Preprocess an application for a given zone and return a new application package pointing to the preprocessed diff --git a/config-model-api/src/main/java/com/yahoo/config/application/api/ValidationOverrides.java b/config-model-api/src/main/java/com/yahoo/config/application/api/ValidationOverrides.java index 8845431c71b..ea27b7f70d8 100644 --- a/config-model-api/src/main/java/com/yahoo/config/application/api/ValidationOverrides.java +++ b/config-model-api/src/main/java/com/yahoo/config/application/api/ValidationOverrides.java @@ -1,4 +1,4 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.config.application.api; import com.google.common.collect.ImmutableList; @@ -67,7 +67,7 @@ public class ValidationOverrides { public boolean allows(String validationIdString, Instant now) { Optional<ValidationId> validationId = ValidationId.from(validationIdString); - if ( ! validationId.isPresent()) return false; // unknown id -> not allowed + if (validationId.isEmpty()) return false; // unknown id -> not allowed return allows(validationId.get(), now); } @@ -125,8 +125,8 @@ public class ValidationOverrides { .atStartOfDay().atZone(ZoneOffset.UTC).toInstant() .plus(Duration.ofDays(1)); // Make the override valid *on* the "until" date Optional<ValidationId> validationId = ValidationId.from(XML.getValue(allow)); - if (validationId.isPresent()) // skip unknown ids as they may be valid for other model versions - overrides.add(new ValidationOverrides.Allow(validationId.get(), until)); + // skip unknown ids as they may be valid for other model versions + validationId.ifPresent(id -> overrides.add(new Allow(id, until))); } return new ValidationOverrides(overrides, xmlForm); } diff --git a/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java b/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java index abca424a838..5158f3ec488 100644 --- a/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java +++ b/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java @@ -74,20 +74,18 @@ public interface ModelContext { @ModelFeatureFlag(owners = {"baldersheim"}, comment = "Select sequencer type use while feeding") default String feedSequencerType() { throw new UnsupportedOperationException("TODO specify default value"); } @ModelFeatureFlag(owners = {"baldersheim"}) default String responseSequencerType() { throw new UnsupportedOperationException("TODO specify default value"); } @ModelFeatureFlag(owners = {"baldersheim"}) default int defaultNumResponseThreads() { return 2; } - @ModelFeatureFlag(owners = {"baldersheim"}) default int maxPendingMoveOps() { return 100; } @ModelFeatureFlag(owners = {"baldersheim"}) default boolean skipCommunicationManagerThread() { throw new UnsupportedOperationException("TODO specify default value"); } @ModelFeatureFlag(owners = {"baldersheim"}) default boolean skipMbusRequestThread() { throw new UnsupportedOperationException("TODO specify default value"); } @ModelFeatureFlag(owners = {"baldersheim"}) default boolean skipMbusReplyThread() { throw new UnsupportedOperationException("TODO specify default value"); } @ModelFeatureFlag(owners = {"tokle"}) default boolean useAccessControlTlsHandshakeClientAuth() { return true; } @ModelFeatureFlag(owners = {"baldersheim"}) default boolean useAsyncMessageHandlingOnSchedule() { throw new UnsupportedOperationException("TODO specify default value"); } @ModelFeatureFlag(owners = {"baldersheim"}) default double feedConcurrency() { throw new UnsupportedOperationException("TODO specify default value"); } - @ModelFeatureFlag(owners = {"baldersheim"}) default boolean useBucketExecutorForLidSpaceCompact() { return true; } - @ModelFeatureFlag(owners = {"baldersheim"}) default boolean useBucketExecutorForBucketMove() { return true; } - @ModelFeatureFlag(owners = {"baldersheim"}) default boolean useBucketExecutorForPruneRemoved() { throw new UnsupportedOperationException("TODO specify default value"); } + @ModelFeatureFlag(owners = {"baldersheim"}) default boolean useBucketExecutorForPruneRemoved() { return true; } @ModelFeatureFlag(owners = {"baldersheim"}) default boolean useExternalRankExpressions() { return false; } @ModelFeatureFlag(owners = {"baldersheim"}) default boolean distributeExternalRankExpressions() { return false; } + @ModelFeatureFlag(owners = {"baldersheim"}) default int maxConcurrentMergesPerNode() { throw new UnsupportedOperationException("TODO specify default value"); } + @ModelFeatureFlag(owners = {"baldersheim"}) default int maxMergeQueueSize() { throw new UnsupportedOperationException("TODO specify default value"); } @ModelFeatureFlag(owners = {"geirst"}) default boolean enableFeedBlockInDistributor() { return true; } - @ModelFeatureFlag(owners = {"baldersheim", "geirst", "toregge"}) default double maxDeadBytesRatio() { return 0.05; } @ModelFeatureFlag(owners = {"hmusum"}, removeAfter = "7.406") default int clusterControllerMaxHeapSizeInMb() { return 128; } @ModelFeatureFlag(owners = {"hmusum"}) default int metricsProxyMaxHeapSizeInMb(ClusterSpec.Type type) { return 256; } @ModelFeatureFlag(owners = {"bjorncs", "tokle"}) default List<String> allowedAthenzProxyIdentities() { return List.of(); } @@ -97,6 +95,7 @@ public interface ModelContext { @ModelFeatureFlag(owners = {"bjorncs", "jonmv"}, removeAfter = "7.409") default boolean enableJdiscHttp2() { return true; } @ModelFeatureFlag(owners = {"tokle", "bjorncs"}) default boolean enableCustomAclMapping() { return false; } @ModelFeatureFlag(owners = {"geirst", "vekterli"}) default int numDistributorStripes() { return 0; } + @ModelFeatureFlag(owners = {"arnej"}) default boolean requireConnectivityCheck() { return false; } } /** Warning: As elsewhere in this package, do not make backwards incompatible changes that will break old config models! */ diff --git a/config-model-fat/pom.xml b/config-model-fat/pom.xml index 8a839b228a0..d258623d9c6 100644 --- a/config-model-fat/pom.xml +++ b/config-model-fat/pom.xml @@ -97,11 +97,25 @@ com.yahoo.io.reader, com.yahoo.path, com.google.inject, <!-- must use @Inject exported from container --> - org.xml.sax, + javax.crypto.interfaces, + javax.crypto.spec, + javax.crypto, + javax.naming.directory, + javax.naming.ldap, + javax.naming, + javax.net.ssl, + javax.security.auth.callback, + javax.security.auth.x500, + javax.security.auth, + javax.xml.datatype, + javax.xml.namespace, javax.xml.parsers, javax.xml.transform, javax.xml.xpath, + org.w3c.dom.bootstrap, + org.w3c.dom.ls, org.w3c.dom, + org.xml.sax, <!-- TODO: The fat bundle becomes more brittle for each package added below. Use interfaces in model-api instead. --> com.yahoo.vespa.config, com.yahoo.vespa.config.buildergen, diff --git a/config-model/src/main/java/com/yahoo/config/model/ApplicationConfigProducerRoot.java b/config-model/src/main/java/com/yahoo/config/model/ApplicationConfigProducerRoot.java index 061ad42e028..364dd1742ae 100644 --- a/config-model/src/main/java/com/yahoo/config/model/ApplicationConfigProducerRoot.java +++ b/config-model/src/main/java/com/yahoo/config/model/ApplicationConfigProducerRoot.java @@ -65,8 +65,8 @@ public class ApplicationConfigProducerRoot extends AbstractConfigProducer<Abstra * Creates and initializes a new Vespa from the service config file * in the given application directory. * - * @param parent The parent, usually VespaModel - * @param name The name, used as configId + * @param parent the parent, usually VespaModel + * @param name the name, used as configId * @param documentModel DocumentModel to serve global document config from. */ public ApplicationConfigProducerRoot(AbstractConfigProducer parent, String name, DocumentModel documentModel, Version vespaVersion, ApplicationId applicationId) { diff --git a/config-model/src/main/java/com/yahoo/config/model/ConfigModelRepo.java b/config-model/src/main/java/com/yahoo/config/model/ConfigModelRepo.java index 557df4bd106..100b5ec5a24 100644 --- a/config-model/src/main/java/com/yahoo/config/model/ConfigModelRepo.java +++ b/config-model/src/main/java/com/yahoo/config/model/ConfigModelRepo.java @@ -11,6 +11,16 @@ import com.yahoo.config.model.builder.xml.XmlHelper; import com.yahoo.config.model.graph.ModelGraphBuilder; import com.yahoo.config.model.graph.ModelNode; import com.yahoo.config.model.provision.HostsXmlProvisioner; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.Iterator; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.TreeMap; import java.util.logging.Level; import com.yahoo.path.Path; import com.yahoo.text.XML; @@ -29,7 +39,6 @@ import java.io.IOException; import java.io.Reader; import java.io.Serializable; import java.io.StringReader; -import java.util.*; import java.util.logging.Logger; /** diff --git a/config-model/src/main/java/com/yahoo/config/model/deploy/DeployState.java b/config-model/src/main/java/com/yahoo/config/model/deploy/DeployState.java index 1d0541b67d1..68924dde3e1 100644 --- a/config-model/src/main/java/com/yahoo/config/model/deploy/DeployState.java +++ b/config-model/src/main/java/com/yahoo/config/model/deploy/DeployState.java @@ -22,7 +22,7 @@ import com.yahoo.config.model.api.ValidationParameters; import com.yahoo.config.model.application.provider.BaseDeployLogger; import com.yahoo.config.model.application.provider.MockFileRegistry; import com.yahoo.config.model.provision.HostsXmlProvisioner; -import com.yahoo.config.model.provision.SingleNodeProvisioner; +import com.yahoo.config.model.provision .SingleNodeProvisioner; import com.yahoo.config.model.test.MockApplicationPackage; import com.yahoo.config.provision.DockerImage; import com.yahoo.config.provision.Zone; @@ -460,7 +460,7 @@ public class DeployState implements ConfigDefinitionStore { private SearchDocumentModel createSearchDocumentModel(RankProfileRegistry rankProfileRegistry, QueryProfiles queryProfiles, ValidationParameters validationParameters) { - Collection<NamedReader> readers = applicationPackage.getSearchDefinitions(); + Collection<NamedReader> readers = applicationPackage.getSchemas(); Map<String, String> names = new LinkedHashMap<>(); SearchBuilder builder = new SearchBuilder(applicationPackage, logger, properties, rankProfileRegistry, queryProfiles.getRegistry()); for (NamedReader reader : readers) { @@ -470,14 +470,14 @@ public class DeployState implements ConfigDefinitionStore { String sdName = stripSuffix(readerName, ApplicationPackage.SD_NAME_SUFFIX); names.put(topLevelName, sdName); if ( ! sdName.equals(topLevelName)) { - throw new IllegalArgumentException("Schema definition file name ('" + sdName + "') and name of " + + throw new IllegalArgumentException("Schema file name ('" + sdName + "') and name of " + "top level element ('" + topLevelName + "') are not equal for file '" + readerName + "'"); } } catch (ParseException e) { - throw new IllegalArgumentException("Could not parse sd file '" + reader.getName() + "'", e); + throw new IllegalArgumentException("Could not parse schema file '" + reader.getName() + "'", e); } catch (IOException e) { - throw new IllegalArgumentException("Could not read sd file '" + reader.getName() + "'", e); + throw new IllegalArgumentException("Could not read schema file '" + reader.getName() + "'", e); } finally { closeIgnoreException(reader.getReader()); } diff --git a/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java b/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java index 2a530b78b86..2927df57bc1 100644 --- a/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java +++ b/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java @@ -16,6 +16,7 @@ import com.yahoo.config.provision.HostName; import com.yahoo.config.provision.Zone; import java.net.URI; +import java.security.cert.X509Certificate; import java.util.Collections; import java.util.List; import java.util.Optional; @@ -50,7 +51,6 @@ public class TestProperties implements ModelContext.Properties, ModelContext.Fea private boolean useAccessControlTlsHandshakeClientAuth; private boolean useAsyncMessageHandlingOnSchedule = false; private double feedConcurrency = 0.5; - private boolean useBucketExecutorForPruneRemoved; private boolean enableFeedBlockInDistributor = true; private boolean useExternalRankExpression = false; private int clusterControllerMaxHeapSizeInMb = 128; @@ -59,7 +59,10 @@ public class TestProperties implements ModelContext.Properties, ModelContext.Fea private List<TenantSecretStore> tenantSecretStores = Collections.emptyList(); private String jvmOmitStackTraceInFastThrowOption; private int numDistributorStripes = 0; + private int maxConcurrentMergesPerNode = 16; + private int maxMergeQueueSize = 1024; private boolean allowDisableMtls = true; + private List<X509Certificate> operatorCertificates = Collections.emptyList(); @Override public ModelContext.FeatureFlags featureFlags() { return this; } @Override public boolean multitenant() { return multitenant; } @@ -90,7 +93,6 @@ public class TestProperties implements ModelContext.Properties, ModelContext.Fea @Override public boolean useAccessControlTlsHandshakeClientAuth() { return useAccessControlTlsHandshakeClientAuth; } @Override public boolean useAsyncMessageHandlingOnSchedule() { return useAsyncMessageHandlingOnSchedule; } @Override public double feedConcurrency() { return feedConcurrency; } - @Override public boolean useBucketExecutorForPruneRemoved() { return useBucketExecutorForPruneRemoved; } @Override public boolean enableFeedBlockInDistributor() { return enableFeedBlockInDistributor; } @Override public int clusterControllerMaxHeapSizeInMb() { return clusterControllerMaxHeapSizeInMb; } @Override public int metricsProxyMaxHeapSizeInMb(ClusterSpec.Type type) { return metricsProxyMaxHeapSizeInMb; } @@ -99,8 +101,11 @@ public class TestProperties implements ModelContext.Properties, ModelContext.Fea @Override public String jvmOmitStackTraceInFastThrowOption(ClusterSpec.Type type) { return jvmOmitStackTraceInFastThrowOption; } @Override public int numDistributorStripes() { return numDistributorStripes; } @Override public boolean allowDisableMtls() { return allowDisableMtls; } + @Override public List<X509Certificate> operatorCertificates() { return operatorCertificates; } @Override public boolean useExternalRankExpressions() { return useExternalRankExpression; } @Override public boolean distributeExternalRankExpressions() { return useExternalRankExpression; } + @Override public int maxConcurrentMergesPerNode() { return maxConcurrentMergesPerNode; } + @Override public int maxMergeQueueSize() { return maxMergeQueueSize; } public TestProperties useExternalRankExpression(boolean value) { useExternalRankExpression = value; @@ -133,6 +138,15 @@ public class TestProperties implements ModelContext.Properties, ModelContext.Fea return this; } + public TestProperties setMaxConcurrentMergesPerNode(int maxConcurrentMergesPerNode) { + this.maxConcurrentMergesPerNode = maxConcurrentMergesPerNode; + return this; + } + public TestProperties setMaxMergeQueueSize(int maxMergeQueueSize) { + this.maxMergeQueueSize = maxMergeQueueSize; + return this; + } + public TestProperties setDefaultTermwiseLimit(double limit) { defaultTermwiseLimit = limit; return this; @@ -198,11 +212,6 @@ public class TestProperties implements ModelContext.Properties, ModelContext.Fea return this; } - public TestProperties useBucketExecutorForPruneRemoved(boolean enabled) { - useBucketExecutorForPruneRemoved = enabled; - return this; - } - public TestProperties enableFeedBlockInDistributor(boolean enabled) { enableFeedBlockInDistributor = enabled; return this; @@ -243,6 +252,11 @@ public class TestProperties implements ModelContext.Properties, ModelContext.Fea return this; } + public TestProperties setOperatorCertificates(List<X509Certificate> operatorCertificates) { + this.operatorCertificates = List.copyOf(operatorCertificates); + return this; + } + public static class Spec implements ConfigServerSpec { private final String hostName; diff --git a/config-model/src/main/java/com/yahoo/config/model/test/MockApplicationPackage.java b/config-model/src/main/java/com/yahoo/config/model/test/MockApplicationPackage.java index e9fe0824f30..411a37bb70a 100644 --- a/config-model/src/main/java/com/yahoo/config/model/test/MockApplicationPackage.java +++ b/config-model/src/main/java/com/yahoo/config/model/test/MockApplicationPackage.java @@ -115,7 +115,7 @@ public class MockApplicationPackage implements ApplicationPackage { } @Override - public List<NamedReader> getSearchDefinitions() { + public List<NamedReader> getSchemas() { ArrayList<NamedReader> readers = new ArrayList<>(); SearchBuilder searchBuilder = new SearchBuilder(this, new BaseDeployLogger(), @@ -134,11 +134,6 @@ public class MockApplicationPackage implements ApplicationPackage { } @Override - public List<NamedReader> searchDefinitionContents() { - return new ArrayList<>(); - } - - @Override public Map<ConfigDefinitionKey, UnparsedConfigDefinition> getAllExistingConfigDefs() { return Collections.emptyMap(); } diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/RankProfile.java b/config-model/src/main/java/com/yahoo/searchdefinition/RankProfile.java index 39ab443554c..68e03ecb188 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/RankProfile.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/RankProfile.java @@ -73,12 +73,10 @@ public class RankProfile implements Cloneable { protected Set<RankSetting> rankSettings = new java.util.LinkedHashSet<>(); /** The ranking expression to be used for first phase */ - private RankingExpression firstPhaseRanking = null; + private RankingExpressionFunction firstPhaseRanking = null; /** The ranking expression to be used for second phase */ - private RankingExpression secondPhaseRanking = null; - - private Set<String> externalFileExpressions = new HashSet<>(); + private RankingExpressionFunction secondPhaseRanking = null; /** Number of hits to be reranked in second phase, -1 means use default */ private int rerankCount = -1; @@ -128,8 +126,8 @@ public class RankProfile implements Cloneable { /** * Creates a new rank profile for a particular search definition * - * @param name the name of the new profile - * @param search the search definition owning this profile + * @param name the name of the new profile + * @param search the search definition owning this profile * @param rankProfileRegistry the {@link com.yahoo.searchdefinition.RankProfileRegistry} to use for storing * and looking up rank profiles. */ @@ -169,10 +167,6 @@ public class RankProfile implements Cloneable { return search != null ? search.rankingConstants() : model.rankingConstants(); } - public RankExpressionFiles rankExpressionFiles() { - return search != null ? search.rankExpressionFiles() : model.rankExpressionFiles(); - } - public Map<String, OnnxModel> onnxModels() { return search != null ? search.onnxModels().asMap() : onnxModels.asMap(); } @@ -198,9 +192,7 @@ public class RankProfile implements Cloneable { } /** Returns the name of the profile this one inherits, or null if none is inherited */ - public String getInheritedName() { - return inheritedName; - } + public String getInheritedName() { return inheritedName; } /** Returns the inherited rank profile, or null if there is none */ public RankProfile getInherited() { @@ -243,7 +235,7 @@ public class RankProfile implements Cloneable { public MatchPhaseSettings getMatchPhaseSettings() { MatchPhaseSettings settings = this.matchPhaseSettings; - if (settings != null ) return settings; + if (settings != null) return settings; if (getInherited() != null) return getInherited().getMatchPhaseSettings(); return null; } @@ -264,7 +256,7 @@ public class RankProfile implements Cloneable { * @return the rank setting found, or null. */ RankSetting getDeclaredRankSetting(String field, RankSetting.Type type) { - for (Iterator<RankSetting> i = declaredRankSettingIterator(); i.hasNext();) { + for (Iterator<RankSetting> i = declaredRankSettingIterator(); i.hasNext(); ) { RankSetting setting = i.next(); if (setting.getFieldName().equals(field) && setting.getType().equals(type)) { @@ -369,55 +361,26 @@ public class RankProfile implements Cloneable { * Returns null if no expression is set. */ public RankingExpression getFirstPhaseRanking() { - if (firstPhaseRanking != null) return firstPhaseRanking; - if (getInherited() != null) return getInherited().getFirstPhaseRanking(); - return null; - } - - void setFirstPhaseRanking(RankingExpression rankingExpression) { - this.firstPhaseRanking = rankingExpression; - } - - public String getUniqueExpressionName(String name) { - return getName() + "_" + name; - } - public String getFirstPhaseFile() { - String name = FIRST_PHASE; - if (externalFileExpressions.contains(name)) { - return rankExpressionFiles().get(getUniqueExpressionName(name)).getFileName(); - } - if ((firstPhaseRanking == null) && (getInherited() != null)) { - return getInherited().getFirstPhaseFile(); - } - return null; + RankingExpressionFunction function = getFirstPhase(); + if (function == null) return null; + return function.function.getBody(); } - public String getSecondPhaseFile() { - String name = SECOND_PHASE; - if (externalFileExpressions.contains(name)) { - return rankExpressionFiles().get(getUniqueExpressionName(name)).getFileName(); - } - if ((secondPhaseRanking == null) && (getInherited() != null)) { - return getInherited().getSecondPhaseFile(); - } + public RankingExpressionFunction getFirstPhase() { + if (firstPhaseRanking != null) return firstPhaseRanking; + RankProfile inherited = getInherited(); + if (inherited != null) return inherited.getFirstPhase(); return null; } - public String getExpressionFile(String name) { - if (externalFileExpressions.contains(name)) { - return rankExpressionFiles().get(getUniqueExpressionName(name)).getFileName(); - } - if (getInherited() != null) { - return getInherited().getExpressionFile(name); - } - return null; + void setFirstPhaseRanking(RankingExpression rankingExpression) { + this.firstPhaseRanking = new RankingExpressionFunction(new ExpressionFunction(FIRST_PHASE, Collections.emptyList(), rankingExpression), false); } public void setFirstPhaseRanking(String expression) { try { - this.firstPhaseRanking = parseRankingExpression(FIRST_PHASE, expression); - } - catch (ParseException e) { + firstPhaseRanking = new RankingExpressionFunction(parseRankingExpression(FIRST_PHASE, Collections.emptyList(), expression), false); + } catch (ParseException e) { throw new IllegalArgumentException("Illegal first phase ranking function", e); } } @@ -427,14 +390,21 @@ public class RankProfile implements Cloneable { * Returns null if no expression is set. */ public RankingExpression getSecondPhaseRanking() { + RankingExpressionFunction function = getSecondPhase(); + if (function == null) return null; + return function.function().getBody(); + } + + public RankingExpressionFunction getSecondPhase() { if (secondPhaseRanking != null) return secondPhaseRanking; - if (getInherited() != null) return getInherited().getSecondPhaseRanking(); + RankProfile inherited = getInherited(); + if (inherited != null) return inherited.getSecondPhase(); return null; } public void setSecondPhaseRanking(String expression) { try { - this.secondPhaseRanking = parseRankingExpression(SECOND_PHASE, expression); + secondPhaseRanking = new RankingExpressionFunction(parseRankingExpression(SECOND_PHASE, Collections.emptyList(), expression), false); } catch (ParseException e) { throw new IllegalArgumentException("Illegal second phase ranking function", e); @@ -603,7 +573,7 @@ public class RankProfile implements Cloneable { /** Adds a function */ public void addFunction(String name, List<String> arguments, String expression, boolean inline) { try { - addFunction(new ExpressionFunction(name, arguments, parseRankingExpression(name, expression)), inline); + addFunction(parseRankingExpression(name, arguments, expression), inline); } catch (ParseException e) { throw new IllegalArgumentException("Could not parse function '" + name + "'", e); @@ -691,20 +661,20 @@ public class RankProfile implements Cloneable { return retval; } - private RankingExpression parseRankingExpression(String expressionName, String expression) throws ParseException { + private ExpressionFunction parseRankingExpression(String name, List<String> arguments, String expression) throws ParseException { if (expression.trim().length() == 0) - throw new ParseException("Encountered an empty ranking expression in " + getName()+ ", " + expressionName + "."); + throw new ParseException("Encountered an empty ranking expression in " + getName()+ ", " + name + "."); - try (Reader rankingExpressionReader = openRankingExpressionReader(expressionName, expression.trim())) { - return new RankingExpression(expressionName, rankingExpressionReader); + try (Reader rankingExpressionReader = openRankingExpressionReader(name, expression.trim())) { + return new ExpressionFunction(name, arguments, new RankingExpression(name, rankingExpressionReader)); } catch (com.yahoo.searchlib.rankingexpression.parser.ParseException e) { ParseException exception = new ParseException("Could not parse ranking expression '" + expression.trim() + - "' in " + getName()+ ", " + expressionName + "."); + "' in " + getName()+ ", " + name + "."); throw (ParseException)exception.initCause(e); } catch (IOException e) { - throw new RuntimeException("IOException parsing ranking expression '" + expressionName + "'"); + throw new RuntimeException("IOException parsing ranking expression '" + name + "'"); } } @@ -725,10 +695,6 @@ public class RankProfile implements Cloneable { throw new IllegalArgumentException("In " + getName() + ", " + expName + ", ranking references file '" + file + "' in subdirectory, which is not supported."); - if (search.getDeployProperties().featureFlags().distributeExternalRankExpressions()) { - rankExpressionFiles().add(new RankExpressionFile(getUniqueExpressionName(expName), fileName), search.getDeployLogger()); - externalFileExpressions.add(expName); - } return search.getRankingExpression(fileName); } @@ -777,17 +743,17 @@ public class RankProfile implements Cloneable { Map<String, RankingExpressionFunction> inlineFunctions = compileFunctions(this::getInlineFunctions, queryProfiles, featureTypes, importedModels, Collections.emptyMap(), expressionTransforms); - firstPhaseRanking = compile(this.getFirstPhaseRanking(), queryProfiles, featureTypes, importedModels, getConstants(), inlineFunctions, expressionTransforms); - secondPhaseRanking = compile(this.getSecondPhaseRanking(), queryProfiles, featureTypes, importedModels, getConstants(), inlineFunctions, expressionTransforms); + firstPhaseRanking = compile(this.getFirstPhase(), queryProfiles, featureTypes, importedModels, getConstants(), inlineFunctions, expressionTransforms); + secondPhaseRanking = compile(this.getSecondPhase(), queryProfiles, featureTypes, importedModels, getConstants(), inlineFunctions, expressionTransforms); // Function compiling second pass: compile all functions and insert previously compiled inline functions + // TODO This merges all functions from inherited profiles too and erases inheritance information. Not good. functions = compileFunctions(this::getFunctions, queryProfiles, featureTypes, importedModels, inlineFunctions, expressionTransforms); - } private void checkNameCollisions(Map<String, RankingExpressionFunction> functions, Map<String, Value> constants) { for (Map.Entry<String, RankingExpressionFunction> functionEntry : functions.entrySet()) { - if (constants.get(functionEntry.getKey()) != null) + if (constants.containsKey(functionEntry.getKey())) throw new IllegalArgumentException("Cannot have both a constant and function named '" + functionEntry.getKey() + "'"); } @@ -811,15 +777,15 @@ public class RankProfile implements Cloneable { // A straightforward iteration will either miss those functions, or may cause a ConcurrentModificationException while (null != (entry = findUncompiledFunction(functions.get(), compiledFunctions.keySet()))) { RankingExpressionFunction rankingExpressionFunction = entry.getValue(); - RankingExpression compiled = compile(rankingExpressionFunction.function().getBody(), queryProfiles, featureTypes, + RankingExpressionFunction compiled = compile(rankingExpressionFunction, queryProfiles, featureTypes, importedModels, getConstants(), inlineFunctions, expressionTransforms); - compiledFunctions.put(entry.getKey(), rankingExpressionFunction.withExpression(compiled)); + compiledFunctions.put(entry.getKey(), compiled); } return compiledFunctions; } - private Map.Entry<String, RankingExpressionFunction> findUncompiledFunction(Map<String, RankingExpressionFunction> functions, - Set<String> compiledFunctionNames) { + private static Map.Entry<String, RankingExpressionFunction> findUncompiledFunction(Map<String, RankingExpressionFunction> functions, + Set<String> compiledFunctionNames) { for (Map.Entry<String, RankingExpressionFunction> entry : functions.entrySet()) { if ( ! compiledFunctionNames.contains(entry.getKey())) return entry; @@ -827,25 +793,25 @@ public class RankProfile implements Cloneable { return null; } - private RankingExpression compile(RankingExpression expression, + private RankingExpressionFunction compile(RankingExpressionFunction function, QueryProfileRegistry queryProfiles, Map<Reference, TensorType> featureTypes, ImportedMlModels importedModels, Map<String, Value> constants, Map<String, RankingExpressionFunction> inlineFunctions, ExpressionTransforms expressionTransforms) { - if (expression == null) return null; + if (function == null) return null; RankProfileTransformContext context = new RankProfileTransformContext(this, queryProfiles, featureTypes, importedModels, constants, inlineFunctions); - expression = expressionTransforms.transform(expression, context); + RankingExpression expression = expressionTransforms.transform(function.function().getBody(), context); for (Map.Entry<String, String> rankProperty : context.rankProperties().entrySet()) { addRankProperty(rankProperty.getKey(), rankProperty.getValue()); } - return expression; + return function.withExpression(expression); } /** diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/derived/RawRankProfile.java b/config-model/src/main/java/com/yahoo/searchdefinition/derived/RawRankProfile.java index 41ac1e17d93..6b589a22de5 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/derived/RawRankProfile.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/derived/RawRankProfile.java @@ -55,15 +55,18 @@ public class RawRankProfile implements RankProfilesConfig.Producer { /** * Creates a raw rank profile from the given rank profile */ - public RawRankProfile(RankProfile rankProfile, QueryProfileRegistry queryProfiles, ImportedMlModels importedModels, AttributeFields attributeFields, ModelContext.Properties deployProperties) { + public RawRankProfile(RankProfile rankProfile, QueryProfileRegistry queryProfiles, ImportedMlModels importedModels, + AttributeFields attributeFields, ModelContext.Properties deployProperties) { this.name = rankProfile.getName(); - compressedProperties = compress(new Deriver(rankProfile, queryProfiles, importedModels, attributeFields, deployProperties).derive()); + compressedProperties = compress(new Deriver(rankProfile.compile(queryProfiles, importedModels), + attributeFields, deployProperties).derive()); } /** * Only for testing */ - public RawRankProfile(RankProfile rankProfile, QueryProfileRegistry queryProfiles, ImportedMlModels importedModels, AttributeFields attributeFields) { + public RawRankProfile(RankProfile rankProfile, QueryProfileRegistry queryProfiles, + ImportedMlModels importedModels, AttributeFields attributeFields) { this(rankProfile, queryProfiles, importedModels, attributeFields, new TestProperties()); } @@ -120,61 +123,74 @@ public class RawRankProfile implements RankProfilesConfig.Producer { private static class Deriver { - /** - * The field rank settings of this profile - */ - private Map<String, FieldRankSettings> fieldRankSettings = new java.util.LinkedHashMap<>(); - - private final RankProfile rankProfile; - private RankingExpression firstPhaseRanking = null; - private RankingExpression secondPhaseRanking = null; - - private Set<ReferenceNode> summaryFeatures = new LinkedHashSet<>(); - - private Set<ReferenceNode> rankFeatures = new LinkedHashSet<>(); - - private List<RankProfile.RankProperty> rankProperties = new ArrayList<>(); + private final Map<String, FieldRankSettings> fieldRankSettings = new java.util.LinkedHashMap<>(); + private final Set<ReferenceNode> summaryFeatures; + private final Set<ReferenceNode> rankFeatures; + private final List<RankProfile.RankProperty> rankProperties; /** * Rank properties for weight settings to make these available to feature executors */ - private List<RankProfile.RankProperty> boostAndWeightRankProperties = new ArrayList<>(); - - private boolean ignoreDefaultRankFeatures = false; - - private RankProfile.MatchPhaseSettings matchPhaseSettings = null; - - private int rerankCount = -1; - private int keepRankCount = -1; - private int numThreadsPerSearch = -1; - private int minHitsPerThread = -1; - private int numSearchPartitions = -1; - private double termwiseLimit = 1.0; - private double rankScoreDropLimit = -Double.MAX_VALUE; + private final List<RankProfile.RankProperty> boostAndWeightRankProperties = new ArrayList<>(); + + private final boolean ignoreDefaultRankFeatures; + private final RankProfile.MatchPhaseSettings matchPhaseSettings; + private final int rerankCount; + private final int keepRankCount; + private final int numThreadsPerSearch; + private final int minHitsPerThread; + private final int numSearchPartitions; + private final double termwiseLimit; + private final double rankScoreDropLimit; /** * The rank type definitions used to derive settings for the native rank features */ private final NativeRankTypeDefinitionSet nativeRankTypeDefinitions = new NativeRankTypeDefinitionSet("default"); - private final Map<String, String> attributeTypes; private final Map<String, String> queryFeatureTypes; - private final boolean useExternalExpressionFiles; + private final Set<String> filterFields = new java.util.LinkedHashSet<>(); - private Set<String> filterFields = new java.util.LinkedHashSet<>(); + private RankingExpression firstPhaseRanking; + private RankingExpression secondPhaseRanking; /** * Creates a raw rank profile from the given rank profile */ - Deriver(RankProfile rankProfile, QueryProfileRegistry queryProfiles, ImportedMlModels importedModels, - AttributeFields attributeFields, ModelContext.Properties deployProperties) + Deriver(RankProfile compiled, AttributeFields attributeFields, ModelContext.Properties deployProperties) { - this.rankProfile = rankProfile; - RankProfile compiled = rankProfile.compile(queryProfiles, importedModels); attributeTypes = compiled.getAttributeTypes(); queryFeatureTypes = compiled.getQueryFeatureTypes(); - useExternalExpressionFiles = deployProperties.featureFlags().useExternalRankExpressions(); - deriveRankingFeatures(compiled, deployProperties); + firstPhaseRanking = compiled.getFirstPhaseRanking(); + secondPhaseRanking = compiled.getSecondPhaseRanking(); + summaryFeatures = new LinkedHashSet<>(compiled.getSummaryFeatures()); + rankFeatures = compiled.getRankFeatures(); + rerankCount = compiled.getRerankCount(); + matchPhaseSettings = compiled.getMatchPhaseSettings(); + numThreadsPerSearch = compiled.getNumThreadsPerSearch(); + minHitsPerThread = compiled.getMinHitsPerThread(); + numSearchPartitions = compiled.getNumSearchPartitions(); + termwiseLimit = compiled.getTermwiseLimit().orElse(deployProperties.featureFlags().defaultTermwiseLimit()); + keepRankCount = compiled.getKeepRankCount(); + rankScoreDropLimit = compiled.getRankScoreDropLimit(); + ignoreDefaultRankFeatures = compiled.getIgnoreDefaultRankFeatures(); + rankProperties = new ArrayList<>(compiled.getRankProperties()); + + Map<String, RankProfile.RankingExpressionFunction> functions = compiled.getFunctions(); + List<ExpressionFunction> functionExpressions = functions.values().stream().map(f -> f.function()).collect(Collectors.toList()); + Map<String, String> functionProperties = new LinkedHashMap<>(); + SerializationContext functionSerializationContext = new SerializationContext(functionExpressions); + + if (firstPhaseRanking != null) { + functionProperties.putAll(firstPhaseRanking.getRankProperties(functionSerializationContext)); + } + if (secondPhaseRanking != null) { + functionProperties.putAll(secondPhaseRanking.getRankProperties(functionSerializationContext)); + } + + derivePropertiesAndSummaryFeaturesFromFunctions(functions, functionProperties, functionSerializationContext); + deriveOnnxModelFunctionsAndSummaryFeatures(compiled); + deriveRankTypeSetting(compiled, attributeFields); deriveFilterFields(compiled); deriveWeightProperties(compiled); @@ -184,44 +200,16 @@ public class RawRankProfile implements RankProfilesConfig.Producer { filterFields.addAll(rp.allFilterFields()); } - private void deriveRankingFeatures(RankProfile rankProfile, ModelContext.Properties deployProperties) { - firstPhaseRanking = rankProfile.getFirstPhaseRanking(); - secondPhaseRanking = rankProfile.getSecondPhaseRanking(); - summaryFeatures = new LinkedHashSet<>(rankProfile.getSummaryFeatures()); - rankFeatures = rankProfile.getRankFeatures(); - rerankCount = rankProfile.getRerankCount(); - matchPhaseSettings = rankProfile.getMatchPhaseSettings(); - numThreadsPerSearch = rankProfile.getNumThreadsPerSearch(); - minHitsPerThread = rankProfile.getMinHitsPerThread(); - numSearchPartitions = rankProfile.getNumSearchPartitions(); - termwiseLimit = rankProfile.getTermwiseLimit().orElse(deployProperties.featureFlags().defaultTermwiseLimit()); - keepRankCount = rankProfile.getKeepRankCount(); - rankScoreDropLimit = rankProfile.getRankScoreDropLimit(); - ignoreDefaultRankFeatures = rankProfile.getIgnoreDefaultRankFeatures(); - rankProperties = new ArrayList<>(rankProfile.getRankProperties()); - derivePropertiesAndSummaryFeaturesFromFunctions(rankProfile.getFunctions()); - deriveOnnxModelFunctionsAndSummaryFeatures(rankProfile); - } - - private void derivePropertiesAndSummaryFeaturesFromFunctions(Map<String, RankProfile.RankingExpressionFunction> functions) { + private void derivePropertiesAndSummaryFeaturesFromFunctions(Map<String, RankProfile.RankingExpressionFunction> functions, + Map<String, String> functionProperties, + SerializationContext functionContext) { if (functions.isEmpty()) return; - List<ExpressionFunction> functionExpressions = functions.values().stream().map(f -> f.function()).collect(Collectors.toList()); - Map<String, String> functionProperties = new LinkedHashMap<>(); - - if (firstPhaseRanking != null) { - functionProperties.putAll(firstPhaseRanking.getRankProperties(functionExpressions)); - } - if (secondPhaseRanking != null) { - functionProperties.putAll(secondPhaseRanking.getRankProperties(functionExpressions)); - } - - SerializationContext context = new SerializationContext(functionExpressions, null, functionProperties); - replaceFunctionSummaryFeatures(context); + replaceFunctionSummaryFeatures(functionContext); // First phase, second phase and summary features should add all required functions to the context. // However, we need to add any functions not referenced in those anyway for model-evaluation. - deriveFunctionProperties(functions, functionExpressions, functionProperties); + deriveFunctionProperties(functions, functionProperties, functionContext); for (Map.Entry<String, String> e : functionProperties.entrySet()) { rankProperties.add(new RankProfile.RankProperty(e.getKey(), e.getValue())); @@ -229,15 +217,13 @@ public class RawRankProfile implements RankProfilesConfig.Producer { } private void deriveFunctionProperties(Map<String, RankProfile.RankingExpressionFunction> functions, - List<ExpressionFunction> functionExpressions, - Map<String, String> functionProperties) { - SerializationContext context = new SerializationContext(functionExpressions, null, functionProperties); + Map<String, String> functionProperties, + SerializationContext context) { for (Map.Entry<String, RankProfile.RankingExpressionFunction> e : functions.entrySet()) { - if (useExternalExpressionFiles && rankProfile.getExpressionFile(e.getKey()) != null) continue; String propertyName = RankingExpression.propertyName(e.getKey()); if (context.serializedFunctions().containsKey(propertyName)) continue; - String expressionString = e.getValue().function().getBody().getRoot().toString(new StringBuilder(), context, null, null).toString(); + String expressionString = e.getValue().function().getBody().getRoot().toString(context).toString(); context.addFunctionSerialization(propertyName, expressionString); for (Map.Entry<String, TensorType> argumentType : e.getValue().function().argumentTypes().entrySet()) @@ -259,7 +245,7 @@ public class RawRankProfile implements RankProfilesConfig.Producer { ExpressionFunction function = context.getFunction(referenceNode.getName()); if (function != null) { String propertyName = RankingExpression.propertyName(referenceNode.getName()); - String expressionString = function.getBody().getRoot().toString(new StringBuilder(), context, null, null).toString(); + String expressionString = function.getBody().getRoot().toString(context).toString(); context.addFunctionSerialization(propertyName, expressionString); ReferenceNode newReferenceNode = new ReferenceNode("rankingExpression(" + referenceNode.getName() + ")", referenceNode.getArguments().expressions(), referenceNode.getOutput()); functionSummaryFeatures.put(referenceNode.getName(), newReferenceNode); @@ -355,8 +341,8 @@ public class RawRankProfile implements RankProfilesConfig.Producer { properties.add(new Pair<>(property.getName(), property.getValue())); } } - properties.addAll(deriveRankingPhaseRankProperties(firstPhaseRanking, rankProfile.getFirstPhaseFile(), RankProfile.FIRST_PHASE)); - properties.addAll(deriveRankingPhaseRankProperties(secondPhaseRanking, rankProfile.getSecondPhaseFile(), RankProfile.SECOND_PHASE)); + properties.addAll(deriveRankingPhaseRankProperties(firstPhaseRanking, RankProfile.FIRST_PHASE)); + properties.addAll(deriveRankingPhaseRankProperties(secondPhaseRanking, RankProfile.SECOND_PHASE)); for (FieldRankSettings settings : fieldRankSettings.values()) { properties.addAll(settings.deriveRankProperties()); } @@ -408,9 +394,7 @@ public class RawRankProfile implements RankProfilesConfig.Producer { if (ignoreDefaultRankFeatures) { properties.add(new Pair<>("vespa.dump.ignoredefaultfeatures", String.valueOf(true))); } - Iterator filterFieldsIterator = filterFields.iterator(); - while (filterFieldsIterator.hasNext()) { - String fieldName = (String) filterFieldsIterator.next(); + for (String fieldName : filterFields) { properties.add(new Pair<>("vespa.isfilterfield." + fieldName, String.valueOf(true))); } for (Map.Entry<String, String> attributeType : attributeTypes.entrySet()) { @@ -423,7 +407,7 @@ public class RawRankProfile implements RankProfilesConfig.Producer { return properties; } - private List<Pair<String, String>> deriveRankingPhaseRankProperties(RankingExpression expression, String fileName, String phase) { + private List<Pair<String, String>> deriveRankingPhaseRankProperties(RankingExpression expression, String phase) { List<Pair<String, String>> properties = new ArrayList<>(); if (expression == null) return properties; @@ -431,9 +415,7 @@ public class RawRankProfile implements RankProfilesConfig.Producer { if ("".equals(name)) name = phase; - if (useExternalExpressionFiles && (fileName != null)) { - properties.add(new Pair<>("vespa.rank." + phase, "rankingExpression(" + rankProfile.getUniqueExpressionName(name) + ")")); - } else if (expression.getRoot() instanceof ReferenceNode) { + if (expression.getRoot() instanceof ReferenceNode) { properties.add(new Pair<>("vespa.rank." + phase, expression.getRoot().toString())); } else { properties.add(new Pair<>("vespa.rank." + phase, "rankingExpression(" + name + ")")); diff --git a/config-model/src/main/java/com/yahoo/vespa/model/ConfigSentinel.java b/config-model/src/main/java/com/yahoo/vespa/model/ConfigSentinel.java index d15db6b4a55..800bf73cdbb 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/ConfigSentinel.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/ConfigSentinel.java @@ -1,6 +1,7 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.model; +import com.yahoo.config.model.api.ModelContext; import com.yahoo.cloud.config.SentinelConfig; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.Zone; @@ -18,16 +19,20 @@ public class ConfigSentinel extends AbstractService implements SentinelConfig.Pr private final ApplicationId applicationId; private final Zone zone; + private final boolean requireConnectivityCheck; /** * Constructs a new ConfigSentinel for the given host. * * @param host Physical host on which to run. */ - public ConfigSentinel(Host host, ApplicationId applicationId, Zone zone) { + public ConfigSentinel(Host host, ApplicationId applicationId, Zone zone, + ModelContext.FeatureFlags featureFlags) + { super(host, "sentinel"); this.applicationId = applicationId; this.zone = zone; + this.requireConnectivityCheck = featureFlags.requireConnectivityCheck(); portsMeta.on(0).tag("rpc").tag("admin"); portsMeta.on(1).tag("telnet").tag("interactive").tag("http").tag("state"); setProp("clustertype", "hosts"); @@ -75,6 +80,19 @@ public class ConfigSentinel extends AbstractService implements SentinelConfig.Pr builder.service(getServiceConfig(s)); } } + builder.connectivity(getConnectivityConfig(requireConnectivityCheck)); + } + + private SentinelConfig.Connectivity.Builder getConnectivityConfig(boolean enable) { + var builder = new SentinelConfig.Connectivity.Builder(); + if (enable) { + builder.maxBadOutPercent(60); + builder.maxBadReverseCount(3); + } else { + builder.maxBadOutPercent(100); + builder.maxBadReverseCount(Integer.MAX_VALUE); + } + return builder; } private SentinelConfig.Application.Builder getApplicationConfig() { diff --git a/config-model/src/main/java/com/yahoo/vespa/model/HostSystem.java b/config-model/src/main/java/com/yahoo/vespa/model/HostSystem.java index 09bbd446803..53f42866d8d 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/HostSystem.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/HostSystem.java @@ -47,16 +47,15 @@ public class HostSystem extends AbstractConfigProducer<Host> { void checkName(String hostname) { // Give a warning if the host does not exist try { - @SuppressWarnings("unused") - Object ignore = java.net.InetAddress.getByName(hostname); + var inetAddr = java.net.InetAddress.getByName(hostname); + String canonical = inetAddr.getCanonicalHostName(); + if (! hostname.equals(canonical)) { + deployLogger.logApplicationPackage(Level.WARNING, "Host named '" + hostname + "' may not receive any config " + + "since it differs from its canonical hostname '" + canonical + "' (check DNS and /etc/hosts)."); + } } catch (UnknownHostException e) { deployLogger.logApplicationPackage(Level.WARNING, "Unable to lookup IP address of host: " + hostname); } - if (! hostname.contains(".")) { - deployLogger.logApplicationPackage(Level.WARNING, "Host named '" + hostname + "' may not receive any config " + - "since it is not a canonical hostname. " + - "Disregard this warning when testing in a Docker container."); - } } /** diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/Admin.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/Admin.java index e080ce43730..a2a6ada9093 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/admin/Admin.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/Admin.java @@ -1,6 +1,7 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.model.admin; +import com.yahoo.config.model.api.ModelContext; import com.yahoo.cloud.config.SlobroksConfig; import com.yahoo.cloud.config.ZookeepersConfig; import com.yahoo.cloud.config.log.LogdConfig; @@ -242,7 +243,8 @@ public class Admin extends AbstractConfigProducer<Admin> implements Serializable } private void addCommonServices(HostResource host, DeployState deployState) { - addConfigSentinel(deployState.getDeployLogger(), host, deployState.getProperties().applicationId(), deployState.zone()); + addConfigSentinel(deployState.getDeployLogger(), host, deployState.getProperties().applicationId(), deployState.zone(), + deployState.featureFlags()); addLogd(deployState.getDeployLogger(), host); addConfigProxy(deployState.getDeployLogger(), host); addFileDistribution(host); @@ -262,8 +264,10 @@ public class Admin extends AbstractConfigProducer<Admin> implements Serializable } } - private void addConfigSentinel(DeployLogger deployLogger, HostResource host, ApplicationId applicationId, Zone zone) { - ConfigSentinel configSentinel = new ConfigSentinel(host.getHost(), applicationId, zone); + private void addConfigSentinel(DeployLogger deployLogger, HostResource host, + ApplicationId applicationId, Zone zone, ModelContext.FeatureFlags featureFlags) + { + ConfigSentinel configSentinel = new ConfigSentinel(host.getHost(), applicationId, zone, featureFlags); addAndInitializeService(deployLogger, host, configSentinel); host.getHost().setConfigSentinel(configSentinel); } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/AutoscalingMetrics.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/AutoscalingMetrics.java index 6f467b21535..e2aa325c380 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/AutoscalingMetrics.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/AutoscalingMetrics.java @@ -19,19 +19,31 @@ public class AutoscalingMetrics { private static MetricSet create() { List<String> metrics = new ArrayList<>(); + metrics.add("cpu.util"); - metrics.add("mem.util"); - metrics.add("disk.util"); + + // Memory util + metrics.add("mem.util"); // node level - default + metrics.add("content.proton.resource_usage.memory.average"); // better for content as it is the basis for blocking + + // Disk util + metrics.add("disk.util"); // node level -default + metrics.add("content.proton.resource_usage.disk.average"); // better for content as it is the basis for blocking + metrics.add("application_generation"); + metrics.add("in_service"); + // Query rate metrics.add("queries.rate"); // container metrics.add("content.proton.documentdb.matching.queries.rate"); // content + // Write rate metrics.add("feed.http-requests.rate"); // container metrics.add("vds.filestor.alldisks.allthreads.put.sum.count.rate"); // content metrics.add("vds.filestor.alldisks.allthreads.remove.sum.count.rate"); // content metrics.add("vds.filestor.alldisks.allthreads.update.sum.count.rate"); // content + return new MetricSet("autoscaling", toMetrics(metrics)); } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/VespaMetricSet.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/VespaMetricSet.java index 034bf772ffc..114a3e380ef 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/VespaMetricSet.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/VespaMetricSet.java @@ -256,8 +256,11 @@ public class VespaMetricSet { metrics.add(new Metric("cluster-controller.node-event.count")); metrics.add(new Metric("cluster-controller.resource_usage.nodes_above_limit.last")); + metrics.add(new Metric("cluster-controller.resource_usage.nodes_above_limit.max")); metrics.add(new Metric("cluster-controller.resource_usage.max_memory_utilization.last")); + metrics.add(new Metric("cluster-controller.resource_usage.max_memory_utilization.max")); metrics.add(new Metric("cluster-controller.resource_usage.max_disk_utilization.last")); + metrics.add(new Metric("cluster-controller.resource_usage.max_disk_utilization.max")); metrics.add(new Metric("cluster-controller.resource_usage.disk_limit.last")); metrics.add(new Metric("cluster-controller.resource_usage.memory_limit.last")); @@ -762,6 +765,15 @@ public class VespaMetricSet { metrics.add(new Metric("vds.bouncer.clock_skew_aborts.count")); + metrics.add(new Metric("vds.mergethrottler.averagequeuewaitingtime.max")); + metrics.add(new Metric("vds.mergethrottler.averagequeuewaitingtime.sum")); + metrics.add(new Metric("vds.mergethrottler.averagequeuewaitingtime.count")); + metrics.add(new Metric("vds.mergethrottler.queuesize.max")); + metrics.add(new Metric("vds.mergethrottler.queuesize.sum")); + metrics.add(new Metric("vds.mergethrottler.queuesize.count")); + metrics.add(new Metric("vds.mergethrottler.bounced_due_to_back_pressure.rate")); + metrics.add(new Metric("vds.mergethrottler.locallyexecutedmerges.ok.rate")); + metrics.add(new Metric("vds.mergethrottler.mergechains.ok.rate")); return metrics; } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/SchemasDirValidator.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/SchemasDirValidator.java new file mode 100644 index 00000000000..ffcad391e57 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/SchemasDirValidator.java @@ -0,0 +1,32 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model.application.validation; + +import com.yahoo.config.application.api.ApplicationFile; +import com.yahoo.config.application.api.ApplicationPackage; +import com.yahoo.config.model.deploy.DeployState; +import com.yahoo.vespa.model.VespaModel; + +import java.util.logging.Level; + +/** + * Validates that correct directory is used for schemas + * + * @author hmusum + */ +public class SchemasDirValidator extends Validator { + + public SchemasDirValidator() { + } + + @Override + public void validate(VespaModel model, DeployState deployState) { + ApplicationPackage app = deployState.getApplicationPackage(); + ApplicationFile sdDir = app.getFile(ApplicationPackage.SEARCH_DEFINITIONS_DIR); + if (sdDir.exists() && sdDir.isDirectory()) + deployState.getDeployLogger().logApplicationPackage( + Level.WARNING, + "Directory " + ApplicationPackage.SEARCH_DEFINITIONS_DIR.getRelative() + + "/ should not be used for schemas, use " + ApplicationPackage.SCHEMAS_DIR.getRelative() + "/ instead"); + } + +} diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/Validation.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/Validation.java index d1c93bcd611..55443d4b260 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/Validation.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/Validation.java @@ -9,7 +9,6 @@ import com.yahoo.config.model.api.Model; import com.yahoo.config.model.api.ValidationParameters; import com.yahoo.config.model.deploy.DeployState; import com.yahoo.config.provision.ClusterSpec; -import com.yahoo.container.QrConfig; import com.yahoo.vespa.model.VespaModel; import com.yahoo.vespa.model.application.validation.change.ChangeValidator; import com.yahoo.vespa.model.application.validation.change.ClusterSizeReductionValidator; @@ -25,8 +24,6 @@ import com.yahoo.vespa.model.application.validation.change.ResourcesReductionVal import com.yahoo.vespa.model.application.validation.change.StartupCommandChangeValidator; import com.yahoo.vespa.model.application.validation.change.StreamingSearchClusterChangeValidator; import com.yahoo.vespa.model.application.validation.first.AccessControlOnFirstDeploymentValidator; -import com.yahoo.vespa.model.container.ApplicationContainerCluster; -import com.yahoo.vespa.model.container.Container; import java.time.Instant; import java.util.Arrays; @@ -43,7 +40,6 @@ import static java.util.stream.Collectors.groupingBy; import static java.util.stream.Collectors.mapping; import static java.util.stream.Collectors.toCollection; import static java.util.stream.Collectors.toList; -import static java.util.stream.Collectors.toSet; /** * Executor of validators. This defines the right order of validator execution. @@ -63,6 +59,7 @@ public class Validation { new RoutingValidator().validate(model, deployState); new RoutingSelectorValidator().validate(model, deployState); } + new SchemasDirValidator().validate(model, deployState); new ComponentValidator().validate(model, deployState); new SearchDataTypeValidator().validate(model, deployState); new ComplexAttributeFieldsValidator().validate(model, deployState); diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/component/AccessLogComponent.java b/config-model/src/main/java/com/yahoo/vespa/model/container/component/AccessLogComponent.java index f18f10644ca..fc41da43479 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/component/AccessLogComponent.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/component/AccessLogComponent.java @@ -35,7 +35,9 @@ public final class AccessLogComponent extends SimpleComponent implements AccessL this(cluster, logType, compressionType, String.format("logs/vespa/qrs/%s.%s.%s", capitalize(logType.name()), clusterName, "%Y%m%d%H%M%S"), null, null, isHostedVespa, - capitalize(logType.name()) + "." + clusterName); + capitalize(logType.name()) + "." + clusterName, + queueSize(cluster).orElse(-1), + ((cluster instanceof ApplicationContainerCluster) ? 4*1024*1024 : null)); } private static String capitalize(String name) { @@ -49,7 +51,9 @@ public final class AccessLogComponent extends SimpleComponent implements AccessL String rotationInterval, Boolean compressOnRotation, boolean isHostedVespa, - String symlinkName) + String symlinkName, + Integer queueSize, + Integer bufferSize) { super(new ComponentModel(accessLogClass(logType), null, "container-core", null)); this.fileNamePattern = fileNamePattern; @@ -58,10 +62,8 @@ public final class AccessLogComponent extends SimpleComponent implements AccessL this.isHostedVespa = isHostedVespa; this.symlinkName = symlinkName; this.compressionType = compressionType; - this.queueSize = queueSize(cluster).orElse(-1); - bufferSize = (cluster instanceof ApplicationContainerCluster) - ? 4*1024*1024 - : null; + this.queueSize = (queueSize == null) ? 256 : queueSize; + this.bufferSize = bufferSize; if (fileNamePattern == null) throw new RuntimeException("File name pattern required when configuring access log."); diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/component/ConnectionLogComponent.java b/config-model/src/main/java/com/yahoo/vespa/model/container/component/ConnectionLogComponent.java index 83e06aab703..0b51cd163a2 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/component/ConnectionLogComponent.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/component/ConnectionLogComponent.java @@ -4,7 +4,6 @@ package com.yahoo.vespa.model.container.component; import com.yahoo.container.logging.ConnectionLog; import com.yahoo.container.logging.ConnectionLogConfig; -import com.yahoo.osgi.provider.model.ComponentModel; import com.yahoo.vespa.model.container.ContainerCluster; import java.util.OptionalInt; diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/AccessLogBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/AccessLogBuilder.java index f84d6f0724b..d7812e9b4ff 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/AccessLogBuilder.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/AccessLogBuilder.java @@ -62,7 +62,9 @@ public class AccessLogBuilder { rotationInterval(spec), compressOnRotation(spec), isHostedVespa, - symlinkName(spec)); + symlinkName(spec), + queueSize(spec), + bufferSize(spec)); } private String symlinkName(Element spec) { @@ -74,6 +76,16 @@ public class AccessLogBuilder { return (compress.isEmpty() ? null : Boolean.parseBoolean(compress)); } + private Integer bufferSize(Element spec) { + String value = spec.getAttribute("bufferSize"); + return (value.isEmpty() ? null : Integer.parseInt(value)); + } + + private Integer queueSize(Element spec) { + String value = spec.getAttribute("queueSize"); + return (value.isEmpty() ? null : Integer.parseInt(value)); + } + private String rotationInterval(Element spec) { return nullIfEmpty(spec.getAttribute("rotationInterval")); } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ConfigServerContainerModelBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ConfigServerContainerModelBuilder.java index 7e0be6e448b..a37d0ef416f 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ConfigServerContainerModelBuilder.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ConfigServerContainerModelBuilder.java @@ -48,7 +48,7 @@ public class ConfigServerContainerModelBuilder extends ContainerModelBuilder { cluster.addComponent( new AccessLogComponent( cluster, AccessLogComponent.AccessLogType.jsonAccessLog, AccessLogComponent.CompressionType.ZSTD, - "logs/vespa/configserver/access-json.log.%Y%m%d%H%M%S", null, true, true, "access-json.log")); + "logs/vespa/configserver/access-json.log.%Y%m%d%H%M%S", null, true, true, "access-json.log", 1024,256*1024)); cluster.addComponent(new ConnectionLogComponent(cluster, FileConnectionLog.class, "configserver")); } else { super.addAccessLogs(deployState, cluster, spec); diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java index b477587bcac..08ccfe33cd5 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java @@ -34,6 +34,7 @@ import com.yahoo.container.logging.FileConnectionLog; import com.yahoo.osgi.provider.model.ComponentModel; import com.yahoo.search.rendering.RendererRegistry; import com.yahoo.searchdefinition.derived.RankProfileList; +import com.yahoo.security.X509CertificateUtils; import com.yahoo.text.XML; import com.yahoo.vespa.defaults.Defaults; import com.yahoo.vespa.model.AbstractService; @@ -89,6 +90,7 @@ import org.w3c.dom.Element; import org.w3c.dom.Node; import java.net.URI; +import java.security.cert.X509Certificate; import java.util.ArrayList; import java.util.Collections; import java.util.List; @@ -217,7 +219,7 @@ public class ContainerModelBuilder extends ConfigModelBuilder<ContainerModel> { if(deployState.isHosted()) { cluster.addPlatformBundle(PlatformBundles.absoluteBundlePath("jdisc-cloud-aws")); } - if (deployState.featureFlags().tenantIamRole()) { + if (deployState.zone().system().isPublic()) { BindingPattern bindingPattern = SystemBindingPattern.fromHttpPath("/validate-secret-store"); Handler<AbstractConfigProducer<?>> handler = new Handler<>( new ComponentModel("com.yahoo.jdisc.cloud.aws.AwsParameterStoreValidationHandler", null, "jdisc-cloud-aws", null)); @@ -431,6 +433,7 @@ public class ContainerModelBuilder extends ConfigModelBuilder<ContainerModel> { // If the deployment contains certificate/private key reference, setup TLS port HostedSslConnectorFactory connectorFactory; + boolean enableHttp2 = deployState.featureFlags().enableJdiscHttp2(); if (deployState.endpointCertificateSecrets().isPresent()) { boolean authorizeClient = deployState.zone().system().isPublic(); if (authorizeClient && deployState.tlsClientAuthority().isEmpty()) { @@ -444,7 +447,7 @@ public class ContainerModelBuilder extends ConfigModelBuilder<ContainerModel> { .orElse(false); connectorFactory = authorizeClient - ? HostedSslConnectorFactory.withProvidedCertificateAndTruststore(serverName, endpointCertificateSecrets, deployState.tlsClientAuthority().get()) + ? HostedSslConnectorFactory.withProvidedCertificateAndTruststore(serverName, endpointCertificateSecrets, getTlsClientAuthorities(deployState)) : HostedSslConnectorFactory.withProvidedCertificate(serverName, endpointCertificateSecrets, enforceHandshakeClientAuth); } else { connectorFactory = HostedSslConnectorFactory.withDefaultCertificateAndTruststore(serverName); @@ -453,6 +456,19 @@ public class ContainerModelBuilder extends ConfigModelBuilder<ContainerModel> { server.addConnector(connectorFactory); } + /* + Return trusted certificates as a PEM encoded string containing the concatenation of + trusted certs from the application package and all operator certificates. + */ + String getTlsClientAuthorities(DeployState deployState) { + List<X509Certificate> trustedCertificates = deployState.tlsClientAuthority() + .map(X509CertificateUtils::certificateListFromPem) + .orElse(Collections.emptyList()); + ArrayList<X509Certificate> x509Certificates = new ArrayList<>(trustedCertificates); + x509Certificates.addAll(deployState.getProperties().operatorCertificates()); + return X509CertificateUtils.toPem(x509Certificates); + } + private static boolean isHostedTenantApplication(ConfigModelContext context) { var deployState = context.getDeployState(); boolean isTesterApplication = deployState.getProperties().applicationId().instance().isTester(); diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/Content.java b/config-model/src/main/java/com/yahoo/vespa/model/content/Content.java index 2bd9cb09aa6..ea52f9689ff 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/content/Content.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/content/Content.java @@ -177,7 +177,7 @@ public class Content extends ConfigModel { s.setVespaMallocDebugStackTrace(cluster.getRootGroup().getVespaMallocDebugStackTrace().get()); } } - cluster.prepare(deployState); + cluster.prepare(); } private void setCpuSocketAffinity() { diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/ContentSearchCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/content/ContentSearchCluster.java index 18580249ddc..51949e78838 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/content/ContentSearchCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/content/ContentSearchCluster.java @@ -65,7 +65,6 @@ public class ContentSearchCluster extends AbstractConfigProducer<SearchCluster> private Optional<ResourceLimits> resourceLimits = Optional.empty(); private final ProtonConfig.Indexing.Optimize.Enum feedSequencerType; private final double defaultFeedConcurrency; - private final boolean useBucketExecutorForPruneRemoved; /** Whether the nodes of this cluster also hosts a container cluster in a hosted system */ private final boolean combined; @@ -210,7 +209,6 @@ public class ContentSearchCluster extends AbstractConfigProducer<SearchCluster> this.combined = combined; feedSequencerType = convertFeedSequencerType(featureFlags.feedSequencerType()); defaultFeedConcurrency = featureFlags.feedConcurrency(); - useBucketExecutorForPruneRemoved = featureFlags.useBucketExecutorForPruneRemoved(); } public void setVisibilityDelay(double delay) { @@ -427,7 +425,6 @@ public class ContentSearchCluster extends AbstractConfigProducer<SearchCluster> } else { builder.indexing.optimize(feedSequencerType); } - builder.pruneremoveddocuments.usebucketexecutor(useBucketExecutorForPruneRemoved); } private boolean isGloballyDistributed(NewDocumentType docType) { diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/ContentCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/ContentCluster.java index 830eced56d3..90cca1494b2 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/ContentCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/ContentCluster.java @@ -70,7 +70,7 @@ import static java.util.stream.Collectors.toList; * @author mostly somebody unknown * @author bratseth */ -public class ContentCluster extends AbstractConfigProducer implements +public class ContentCluster extends AbstractConfigProducer<AbstractConfigProducer<?>> implements DistributionConfig.Producer, StorDistributionConfig.Producer, StorDistributormanagerConfig.Producer, @@ -94,14 +94,6 @@ public class ContentCluster extends AbstractConfigProducer implements private Integer maxNodesPerMerge; private final Zone zone; - /** - * If multitenant or a cluster controller was explicitly configured in this cluster: - * The cluster controller cluster of this particular content cluster. - * - * Otherwise: null - the cluster controller is shared by all content clusters and part of Admin. - */ - private ClusterControllerContainerCluster clusterControllers; - public enum DistributionMode { LEGACY, STRICT, LOOSE } private DistributionMode distributionMode; @@ -413,16 +405,10 @@ public class ContentCluster extends AbstractConfigProducer implements public ClusterSpec.Id id() { return ClusterSpec.Id.from(clusterId); } - public void prepare(DeployState deployState) { + public void prepare() { search.prepare(); - - if (clusterControllers != null) - clusterControllers.prepare(deployState); } - /** Returns cluster controllers if this is multitenant, null otherwise */ - public ClusterControllerContainerCluster getClusterControllers() { return clusterControllers; } - public DistributionMode getDistributionMode() { if (distributionMode != null) return distributionMode; return getPersistence().getDefaultDistributionMode(); diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/storagecluster/StorServerProducer.java b/config-model/src/main/java/com/yahoo/vespa/model/content/storagecluster/StorServerProducer.java index 40a634fbfe8..e89d45e8b83 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/content/storagecluster/StorServerProducer.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/content/storagecluster/StorServerProducer.java @@ -6,6 +6,8 @@ import com.yahoo.vespa.config.content.core.StorServerConfig; import com.yahoo.vespa.model.content.cluster.ContentCluster; import com.yahoo.vespa.model.builder.xml.dom.ModelElement; +import java.util.Optional; + /** * Serves config for stor-server for storage clusters (clusters of storage nodes). */ @@ -14,7 +16,7 @@ public class StorServerProducer implements StorServerConfig.Producer { StorServerProducer build(ModelContext.Properties properties, ModelElement element) { ModelElement tuning = element.child("tuning"); - StorServerProducer producer = new StorServerProducer(ContentCluster.getClusterId(element)); + StorServerProducer producer = new StorServerProducer(ContentCluster.getClusterId(element), properties.featureFlags()); if (tuning == null) return producer; ModelElement merges = tuning.child("merges"); @@ -32,11 +34,15 @@ public class StorServerProducer implements StorServerConfig.Producer { private Integer bucketDBStripeBits; private StorServerProducer setMaxMergesPerNode(Integer value) { - maxMergesPerNode = value; + if (value != null) { + maxMergesPerNode = value; + } return this; } private StorServerProducer setMaxQueueSize(Integer value) { - queueSize = value; + if (value != null) { + queueSize = value; + } return this; } private StorServerProducer setBucketDBStripeBits(Integer value) { @@ -44,8 +50,10 @@ public class StorServerProducer implements StorServerConfig.Producer { return this; } - public StorServerProducer(String clusterName) { + StorServerProducer(String clusterName, ModelContext.FeatureFlags featureFlags) { this.clusterName = clusterName; + maxMergesPerNode = featureFlags.maxConcurrentMergesPerNode(); + queueSize = featureFlags.maxMergeQueueSize(); } @Override diff --git a/config-model/src/main/resources/schema/containercluster.rnc b/config-model/src/main/resources/schema/containercluster.rnc index 39df939f78c..3679e53e257 100644 --- a/config-model/src/main/resources/schema/containercluster.rnc +++ b/config-model/src/main/resources/schema/containercluster.rnc @@ -81,7 +81,8 @@ AccessLog = element accesslog { attribute compressOnRotation { xsd:boolean }? & attribute symlinkName { string }? & attribute compressionType { string "gzip" | string "zstd" }? & - + attribute queueSize { xsd:nonNegativeInteger }? & + attribute bufferSize { xsd:nonNegativeInteger }? & attribute rotationInterval { string }? } diff --git a/config-model/src/test/cfg/admin/metricconfig/searchdefinitions/music.sd b/config-model/src/test/cfg/admin/metricconfig/schemas/music.sd index 32e8451d8e2..32e8451d8e2 100644 --- a/config-model/src/test/cfg/admin/metricconfig/searchdefinitions/music.sd +++ b/config-model/src/test/cfg/admin/metricconfig/schemas/music.sd diff --git a/config-model/src/test/cfg/application/app1/searchdefinitions/bar.expression b/config-model/src/test/cfg/application/app1/schemas/bar.expression index eed496e6aeb..eed496e6aeb 100644 --- a/config-model/src/test/cfg/application/app1/searchdefinitions/bar.expression +++ b/config-model/src/test/cfg/application/app1/schemas/bar.expression diff --git a/config-model/src/test/cfg/application/app1/searchdefinitions/foo.expression b/config-model/src/test/cfg/application/app1/schemas/foo.expression index ce26aa75dcb..ce26aa75dcb 100644 --- a/config-model/src/test/cfg/application/app1/searchdefinitions/foo.expression +++ b/config-model/src/test/cfg/application/app1/schemas/foo.expression diff --git a/config-model/src/test/cfg/application/app1/searchdefinitions/laptop.sd b/config-model/src/test/cfg/application/app1/schemas/laptop.sd index 21a76ad605c..21a76ad605c 100644 --- a/config-model/src/test/cfg/application/app1/searchdefinitions/laptop.sd +++ b/config-model/src/test/cfg/application/app1/schemas/laptop.sd diff --git a/config-model/src/test/cfg/application/app1/searchdefinitions/music.sd b/config-model/src/test/cfg/application/app1/schemas/music.sd index 693afbd308d..693afbd308d 100644 --- a/config-model/src/test/cfg/application/app1/searchdefinitions/music.sd +++ b/config-model/src/test/cfg/application/app1/schemas/music.sd diff --git a/config-model/src/test/cfg/application/app1/searchdefinitions/pc.sd b/config-model/src/test/cfg/application/app1/schemas/pc.sd index bdc90328cdb..bdc90328cdb 100644 --- a/config-model/src/test/cfg/application/app1/searchdefinitions/pc.sd +++ b/config-model/src/test/cfg/application/app1/schemas/pc.sd diff --git a/config-model/src/test/cfg/application/app1/searchdefinitions/product.sd b/config-model/src/test/cfg/application/app1/schemas/product.sd index 588a1d544c6..588a1d544c6 100644 --- a/config-model/src/test/cfg/application/app1/searchdefinitions/product.sd +++ b/config-model/src/test/cfg/application/app1/schemas/product.sd diff --git a/config-model/src/test/cfg/application/app1/searchdefinitions/sock.sd b/config-model/src/test/cfg/application/app1/schemas/sock.sd index 9331c0caf54..9331c0caf54 100644 --- a/config-model/src/test/cfg/application/app1/searchdefinitions/sock.sd +++ b/config-model/src/test/cfg/application/app1/schemas/sock.sd diff --git a/config-model/src/test/cfg/application/app_complicated_deployment_spec/searchdefinitions/music.sd b/config-model/src/test/cfg/application/app_complicated_deployment_spec/schemas/music.sd index 693afbd308d..693afbd308d 100644 --- a/config-model/src/test/cfg/application/app_complicated_deployment_spec/searchdefinitions/music.sd +++ b/config-model/src/test/cfg/application/app_complicated_deployment_spec/schemas/music.sd diff --git a/config-model/src/test/cfg/application/app_genericservices/searchdefinitions/music.sd b/config-model/src/test/cfg/application/app_genericservices/schemas/music.sd index 693afbd308d..693afbd308d 100644 --- a/config-model/src/test/cfg/application/app_genericservices/searchdefinitions/music.sd +++ b/config-model/src/test/cfg/application/app_genericservices/schemas/music.sd diff --git a/config-model/src/test/cfg/application/app_nohosts/searchdefinitions/mail.sd b/config-model/src/test/cfg/application/app_nohosts/schemas/mail.sd index 8b6a6c44bf1..8b6a6c44bf1 100644 --- a/config-model/src/test/cfg/application/app_nohosts/searchdefinitions/mail.sd +++ b/config-model/src/test/cfg/application/app_nohosts/schemas/mail.sd diff --git a/config-model/src/test/cfg/application/app_nohosts/searchdefinitions/mailbox.sd b/config-model/src/test/cfg/application/app_nohosts/schemas/mailbox.sd index f98017f401e..f98017f401e 100644 --- a/config-model/src/test/cfg/application/app_nohosts/searchdefinitions/mailbox.sd +++ b/config-model/src/test/cfg/application/app_nohosts/schemas/mailbox.sd diff --git a/config-model/src/test/cfg/application/app_nohosts/searchdefinitions/message.sd b/config-model/src/test/cfg/application/app_nohosts/schemas/message.sd index 6c27917c656..6c27917c656 100644 --- a/config-model/src/test/cfg/application/app_nohosts/searchdefinitions/message.sd +++ b/config-model/src/test/cfg/application/app_nohosts/schemas/message.sd diff --git a/config-model/src/test/cfg/application/app_qrserverandgw/searchdefinitions/message.sd b/config-model/src/test/cfg/application/app_qrserverandgw/schemas/message.sd index c09b2fb2da2..c09b2fb2da2 100644 --- a/config-model/src/test/cfg/application/app_qrserverandgw/searchdefinitions/message.sd +++ b/config-model/src/test/cfg/application/app_qrserverandgw/schemas/message.sd diff --git a/config-model/src/test/cfg/application/deprecated_features_app/hosts.xml b/config-model/src/test/cfg/application/deprecated_features_app/hosts.xml new file mode 100644 index 00000000000..f0c76482b2f --- /dev/null +++ b/config-model/src/test/cfg/application/deprecated_features_app/hosts.xml @@ -0,0 +1,7 @@ +<?xml version="1.0" encoding="utf-8" ?> +<!-- Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. --> +<hosts> + <host name="localhost"> + <alias>node0</alias> + </host> +</hosts> diff --git a/config-model/src/test/cfg/application/deprecated_features_app/searchdefinitions/message.sd b/config-model/src/test/cfg/application/deprecated_features_app/searchdefinitions/message.sd new file mode 100644 index 00000000000..c97c860a129 --- /dev/null +++ b/config-model/src/test/cfg/application/deprecated_features_app/searchdefinitions/message.sd @@ -0,0 +1,9 @@ +# Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +search message { + + document message { + field foo type string { + indexing: summary + } + } +} diff --git a/config-model/src/test/cfg/application/deprecated_features_app/services.xml b/config-model/src/test/cfg/application/deprecated_features_app/services.xml new file mode 100644 index 00000000000..daf03d0c4c3 --- /dev/null +++ b/config-model/src/test/cfg/application/deprecated_features_app/services.xml @@ -0,0 +1,28 @@ +<?xml version="1.0" encoding="utf-8" ?> +<!-- Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. --> + +<services version="1.0"> + + <admin version="2.0"> + <adminserver hostalias="node0"/> + </admin> + + <container version="1.0"> + <nodes> + <node hostalias="node0"/> + </nodes> + <document-api/> + <search/> + </container> + + <content version="1.0" id="message"> + <redundancy>1</redundancy> + <documents> + <document type="message" mode="index"/> + </documents> + <nodes> + <node hostalias="node0" distribution-key="0" /> + </nodes> + </content> + +</services> diff --git a/config-model/src/test/cfg/application/ml_models/searchdefinitions/test.sd b/config-model/src/test/cfg/application/ml_models/schemas/test.sd index cc73f2daff5..cc73f2daff5 100644 --- a/config-model/src/test/cfg/application/ml_models/searchdefinitions/test.sd +++ b/config-model/src/test/cfg/application/ml_models/schemas/test.sd diff --git a/config-model/src/test/cfg/application/onnx/searchdefinitions/test.sd b/config-model/src/test/cfg/application/onnx/schemas/test.sd index d49782ddf39..d49782ddf39 100644 --- a/config-model/src/test/cfg/application/onnx/searchdefinitions/test.sd +++ b/config-model/src/test/cfg/application/onnx/schemas/test.sd diff --git a/config-model/src/test/cfg/application/sdfilenametest/searchdefinitions/notmusic.sd b/config-model/src/test/cfg/application/sdfilenametest/schemas/notmusic.sd index 77307c04843..77307c04843 100644 --- a/config-model/src/test/cfg/application/sdfilenametest/searchdefinitions/notmusic.sd +++ b/config-model/src/test/cfg/application/sdfilenametest/schemas/notmusic.sd diff --git a/config-model/src/test/cfg/application/validation/document_references_validation/searchdefinitions/ad.sd b/config-model/src/test/cfg/application/validation/document_references_validation/schemas/ad.sd index 1e85ab70357..1e85ab70357 100644 --- a/config-model/src/test/cfg/application/validation/document_references_validation/searchdefinitions/ad.sd +++ b/config-model/src/test/cfg/application/validation/document_references_validation/schemas/ad.sd diff --git a/config-model/src/test/cfg/application/validation/document_references_validation/searchdefinitions/campaign.sd b/config-model/src/test/cfg/application/validation/document_references_validation/schemas/campaign.sd index 9e208e397b9..9e208e397b9 100644 --- a/config-model/src/test/cfg/application/validation/document_references_validation/searchdefinitions/campaign.sd +++ b/config-model/src/test/cfg/application/validation/document_references_validation/schemas/campaign.sd diff --git a/config-model/src/test/cfg/application/validation/global_distribution_validation/searchdefinitions/parent.sd b/config-model/src/test/cfg/application/validation/global_distribution_validation/schemas/parent.sd index 8decebc6680..8decebc6680 100644 --- a/config-model/src/test/cfg/application/validation/global_distribution_validation/searchdefinitions/parent.sd +++ b/config-model/src/test/cfg/application/validation/global_distribution_validation/schemas/parent.sd diff --git a/config-model/src/test/cfg/application/validation/global_distribution_validation/searchdefinitions/simple.sd b/config-model/src/test/cfg/application/validation/global_distribution_validation/schemas/simple.sd index b901fa6e82a..b901fa6e82a 100644 --- a/config-model/src/test/cfg/application/validation/global_distribution_validation/searchdefinitions/simple.sd +++ b/config-model/src/test/cfg/application/validation/global_distribution_validation/schemas/simple.sd diff --git a/config-model/src/test/cfg/application/validation/index_struct/searchdefinitions/simple.sd b/config-model/src/test/cfg/application/validation/index_struct/schemas/simple.sd index a27a2d074d2..a27a2d074d2 100644 --- a/config-model/src/test/cfg/application/validation/index_struct/searchdefinitions/simple.sd +++ b/config-model/src/test/cfg/application/validation/index_struct/schemas/simple.sd diff --git a/config-model/src/test/cfg/application/validation/prefix/searchdefinitions/simple.sd b/config-model/src/test/cfg/application/validation/prefix/schemas/simple.sd index 60ce940fd7d..60ce940fd7d 100644 --- a/config-model/src/test/cfg/application/validation/prefix/searchdefinitions/simple.sd +++ b/config-model/src/test/cfg/application/validation/prefix/schemas/simple.sd diff --git a/config-model/src/test/cfg/application/validation/prefix_index/searchdefinitions/simple.sd b/config-model/src/test/cfg/application/validation/prefix_index/schemas/simple.sd index bf70a1662c6..bf70a1662c6 100644 --- a/config-model/src/test/cfg/application/validation/prefix_index/searchdefinitions/simple.sd +++ b/config-model/src/test/cfg/application/validation/prefix_index/schemas/simple.sd diff --git a/config-model/src/test/cfg/application/validation/prefix_index_and_attribute/searchdefinitions/simple.sd b/config-model/src/test/cfg/application/validation/prefix_index_and_attribute/schemas/simple.sd index 41412f4ef4f..41412f4ef4f 100644 --- a/config-model/src/test/cfg/application/validation/prefix_index_and_attribute/searchdefinitions/simple.sd +++ b/config-model/src/test/cfg/application/validation/prefix_index_and_attribute/schemas/simple.sd diff --git a/config-model/src/test/cfg/application/validation/prefix_streaming/searchdefinitions/simple.sd b/config-model/src/test/cfg/application/validation/prefix_streaming/schemas/simple.sd index bf70a1662c6..bf70a1662c6 100644 --- a/config-model/src/test/cfg/application/validation/prefix_streaming/searchdefinitions/simple.sd +++ b/config-model/src/test/cfg/application/validation/prefix_streaming/schemas/simple.sd diff --git a/config-model/src/test/cfg/application/validation/ranking_constants_fail/searchdefinitions/simple.sd b/config-model/src/test/cfg/application/validation/ranking_constants_fail/schemas/simple.sd index 8b782a01946..8b782a01946 100644 --- a/config-model/src/test/cfg/application/validation/ranking_constants_fail/searchdefinitions/simple.sd +++ b/config-model/src/test/cfg/application/validation/ranking_constants_fail/schemas/simple.sd diff --git a/config-model/src/test/cfg/application/validation/ranking_constants_ok/searchdefinitions/simple.sd b/config-model/src/test/cfg/application/validation/ranking_constants_ok/schemas/simple.sd index 8b782a01946..8b782a01946 100644 --- a/config-model/src/test/cfg/application/validation/ranking_constants_ok/searchdefinitions/simple.sd +++ b/config-model/src/test/cfg/application/validation/ranking_constants_ok/schemas/simple.sd diff --git a/config-model/src/test/cfg/application/validation/search_alltypes/searchdefinitions/parent.sd b/config-model/src/test/cfg/application/validation/search_alltypes/schemas/parent.sd index 8decebc6680..8decebc6680 100644 --- a/config-model/src/test/cfg/application/validation/search_alltypes/searchdefinitions/parent.sd +++ b/config-model/src/test/cfg/application/validation/search_alltypes/schemas/parent.sd diff --git a/config-model/src/test/cfg/application/validation/search_alltypes/searchdefinitions/simple.sd b/config-model/src/test/cfg/application/validation/search_alltypes/schemas/simple.sd index 242a7493df3..242a7493df3 100644 --- a/config-model/src/test/cfg/application/validation/search_alltypes/searchdefinitions/simple.sd +++ b/config-model/src/test/cfg/application/validation/search_alltypes/schemas/simple.sd diff --git a/config-model/src/test/cfg/application/validation/search_empty_content/searchdefinitions/simple.sd b/config-model/src/test/cfg/application/validation/search_empty_content/schemas/simple.sd index 51512363fbc..51512363fbc 100644 --- a/config-model/src/test/cfg/application/validation/search_empty_content/searchdefinitions/simple.sd +++ b/config-model/src/test/cfg/application/validation/search_empty_content/schemas/simple.sd diff --git a/config-model/src/test/cfg/application/validation/search_struct/searchdefinitions/simple.sd b/config-model/src/test/cfg/application/validation/search_struct/schemas/simple.sd index 0ab072111c5..0ab072111c5 100644 --- a/config-model/src/test/cfg/application/validation/search_struct/searchdefinitions/simple.sd +++ b/config-model/src/test/cfg/application/validation/search_struct/schemas/simple.sd diff --git a/config-model/src/test/cfg/routing/content_two_clusters/searchdefinitions/mobile.sd b/config-model/src/test/cfg/routing/content_two_clusters/schemas/mobile.sd index 3abc9238e8a..3abc9238e8a 100644 --- a/config-model/src/test/cfg/routing/content_two_clusters/searchdefinitions/mobile.sd +++ b/config-model/src/test/cfg/routing/content_two_clusters/schemas/mobile.sd diff --git a/config-model/src/test/cfg/routing/content_two_clusters/searchdefinitions/music.sd b/config-model/src/test/cfg/routing/content_two_clusters/schemas/music.sd index da9e2d14c55..da9e2d14c55 100644 --- a/config-model/src/test/cfg/routing/content_two_clusters/searchdefinitions/music.sd +++ b/config-model/src/test/cfg/routing/content_two_clusters/schemas/music.sd diff --git a/config-model/src/test/cfg/routing/contentsimpleconfig/searchdefinitions/music.sd b/config-model/src/test/cfg/routing/contentsimpleconfig/schemas/music.sd index da9e2d14c55..da9e2d14c55 100644 --- a/config-model/src/test/cfg/routing/contentsimpleconfig/searchdefinitions/music.sd +++ b/config-model/src/test/cfg/routing/contentsimpleconfig/schemas/music.sd diff --git a/config-model/src/test/cfg/routing/replacehop/searchdefinitions/music.sd b/config-model/src/test/cfg/routing/replacehop/schemas/music.sd index f9816d0d392..f9816d0d392 100755 --- a/config-model/src/test/cfg/routing/replacehop/searchdefinitions/music.sd +++ b/config-model/src/test/cfg/routing/replacehop/schemas/music.sd diff --git a/config-model/src/test/cfg/routing/replaceroute/searchdefinitions/music.sd b/config-model/src/test/cfg/routing/replaceroute/schemas/music.sd index f9816d0d392..f9816d0d392 100755 --- a/config-model/src/test/cfg/routing/replaceroute/searchdefinitions/music.sd +++ b/config-model/src/test/cfg/routing/replaceroute/schemas/music.sd diff --git a/config-model/src/test/cfg/search/data/travel/searchdefinitions/TTData.sd b/config-model/src/test/cfg/search/data/travel/schemas/TTData.sd index 67013239499..67013239499 100644 --- a/config-model/src/test/cfg/search/data/travel/searchdefinitions/TTData.sd +++ b/config-model/src/test/cfg/search/data/travel/schemas/TTData.sd diff --git a/config-model/src/test/cfg/search/data/travel/searchdefinitions/TTEdge.sd b/config-model/src/test/cfg/search/data/travel/schemas/TTEdge.sd index e6707345235..e6707345235 100644 --- a/config-model/src/test/cfg/search/data/travel/searchdefinitions/TTEdge.sd +++ b/config-model/src/test/cfg/search/data/travel/schemas/TTEdge.sd diff --git a/config-model/src/test/cfg/search/data/travel/searchdefinitions/TTPOI.sd b/config-model/src/test/cfg/search/data/travel/schemas/TTPOI.sd index c39ef03add5..c39ef03add5 100644 --- a/config-model/src/test/cfg/search/data/travel/searchdefinitions/TTPOI.sd +++ b/config-model/src/test/cfg/search/data/travel/schemas/TTPOI.sd diff --git a/config-model/src/test/cfg/search/data/v2/inherited_rankprofiles/searchdefinitions/base.sd b/config-model/src/test/cfg/search/data/v2/inherited_rankprofiles/schemas/base.sd index 22944f61997..22944f61997 100644 --- a/config-model/src/test/cfg/search/data/v2/inherited_rankprofiles/searchdefinitions/base.sd +++ b/config-model/src/test/cfg/search/data/v2/inherited_rankprofiles/schemas/base.sd diff --git a/config-model/src/test/cfg/search/data/v2/inherited_rankprofiles/searchdefinitions/left.sd b/config-model/src/test/cfg/search/data/v2/inherited_rankprofiles/schemas/left.sd index c72b349bb4c..c72b349bb4c 100644 --- a/config-model/src/test/cfg/search/data/v2/inherited_rankprofiles/searchdefinitions/left.sd +++ b/config-model/src/test/cfg/search/data/v2/inherited_rankprofiles/schemas/left.sd diff --git a/config-model/src/test/cfg/search/data/v2/inherited_rankprofiles/searchdefinitions/music.sd b/config-model/src/test/cfg/search/data/v2/inherited_rankprofiles/schemas/music.sd index da9e2d14c55..da9e2d14c55 100644 --- a/config-model/src/test/cfg/search/data/v2/inherited_rankprofiles/searchdefinitions/music.sd +++ b/config-model/src/test/cfg/search/data/v2/inherited_rankprofiles/schemas/music.sd diff --git a/config-model/src/test/cfg/search/data/v2/inherited_rankprofiles/searchdefinitions/right.sd b/config-model/src/test/cfg/search/data/v2/inherited_rankprofiles/schemas/right.sd index eb0a4cf8021..eb0a4cf8021 100644 --- a/config-model/src/test/cfg/search/data/v2/inherited_rankprofiles/searchdefinitions/right.sd +++ b/config-model/src/test/cfg/search/data/v2/inherited_rankprofiles/schemas/right.sd diff --git a/config-model/src/test/cfg/storage/app_index_higher_than_num_nodes/searchdefinitions/music.sd b/config-model/src/test/cfg/storage/app_index_higher_than_num_nodes/schemas/music.sd index da9e2d14c55..da9e2d14c55 100644 --- a/config-model/src/test/cfg/storage/app_index_higher_than_num_nodes/searchdefinitions/music.sd +++ b/config-model/src/test/cfg/storage/app_index_higher_than_num_nodes/schemas/music.sd diff --git a/config-model/src/test/cfg/storage/clustercontroller_advanced/searchdefinitions/music.sd b/config-model/src/test/cfg/storage/clustercontroller_advanced/schemas/music.sd index da9e2d14c55..da9e2d14c55 100644 --- a/config-model/src/test/cfg/storage/clustercontroller_advanced/searchdefinitions/music.sd +++ b/config-model/src/test/cfg/storage/clustercontroller_advanced/schemas/music.sd diff --git a/config-model/src/test/derived/rankexpression/rank-profiles.cfg b/config-model/src/test/derived/rankexpression/rank-profiles.cfg index 9629ad863d4..c6263d658c5 100644 --- a/config-model/src/test/derived/rankexpression/rank-profiles.cfg +++ b/config-model/src/test/derived/rankexpression/rank-profiles.cfg @@ -124,12 +124,12 @@ rankprofile[].fef.property[].value "rankingExpression(firstphase)" rankprofile[].fef.property[].name "rankingExpression(firstphase).rankingScript" rankprofile[].fef.property[].value "1" rankprofile[].name "macros" -rankprofile[].fef.property[].name "rankingExpression(fourtimessum).rankingScript" -rankprofile[].fef.property[].value "4 * (var1 + var2)" rankprofile[].fef.property[].name "rankingExpression(myfeature).rankingScript" rankprofile[].fef.property[].value "70 * fieldMatch(title).completeness * pow(0 - fieldMatch(title).earliness,2) + 30 * pow(0 - fieldMatch(description).earliness,2)" rankprofile[].fef.property[].name "rankingExpression(fourtimessum@5cf279212355b980.67f1e87166cfef86).rankingScript" rankprofile[].fef.property[].value "4 * (match + rankBoost)" +rankprofile[].fef.property[].name "rankingExpression(fourtimessum).rankingScript" +rankprofile[].fef.property[].value "4 * (var1 + var2)" rankprofile[].fef.property[].name "vespa.rank.firstphase" rankprofile[].fef.property[].value "rankingExpression(firstphase)" rankprofile[].fef.property[].name "rankingExpression(firstphase).rankingScript" @@ -141,16 +141,16 @@ rankprofile[].fef.property[].value "fieldMatch(title)" rankprofile[].name "macros2" rankprofile[].fef.property[].name "foo" rankprofile[].fef.property[].value "some, list" -rankprofile[].fef.property[].name "rankingExpression(fourtimessum).rankingScript" -rankprofile[].fef.property[].value "4 * (var1 + var2)" -rankprofile[].fef.property[].name "rankingExpression(myfeature).rankingScript" -rankprofile[].fef.property[].value "70 * fieldMatch(title).completeness * pow(0 - fieldMatch(title).earliness,2) + 30 * pow(0 - fieldMatch(description).earliness,2)" +rankprofile[].fef.property[].name "rankingExpression(fourtimessum@2b1138e8965e7ff5.67f1e87166cfef86).rankingScript" +rankprofile[].fef.property[].value "4 * (match + match)" rankprofile[].fef.property[].name "rankingExpression(mysummaryfeature).rankingScript" rankprofile[].fef.property[].value "70 * fieldMatch(title).completeness" +rankprofile[].fef.property[].name "rankingExpression(myfeature).rankingScript" +rankprofile[].fef.property[].value "70 * fieldMatch(title).completeness * pow(0 - fieldMatch(title).earliness,2) + 30 * pow(0 - fieldMatch(description).earliness,2)" +rankprofile[].fef.property[].name "rankingExpression(fourtimessum).rankingScript" +rankprofile[].fef.property[].value "4 * (var1 + var2)" rankprofile[].fef.property[].name "rankingExpression(mysummaryfeature2).rankingScript" rankprofile[].fef.property[].value "71 * fieldMatch(title).completeness" -rankprofile[].fef.property[].name "rankingExpression(fourtimessum@2b1138e8965e7ff5.67f1e87166cfef86).rankingScript" -rankprofile[].fef.property[].value "4 * (match + match)" rankprofile[].fef.property[].name "vespa.rank.firstphase" rankprofile[].fef.property[].value "classicRank" rankprofile[].fef.property[].name "vespa.rank.secondphase" @@ -174,16 +174,16 @@ rankprofile[].fef.property[].value "rankingExpression(matches(title,rankingExpre rankprofile[].name "macros-inherited" rankprofile[].fef.property[].name "foo" rankprofile[].fef.property[].value "some, list" -rankprofile[].fef.property[].name "rankingExpression(fourtimessum).rankingScript" -rankprofile[].fef.property[].value "4 * (var1 + var2)" rankprofile[].fef.property[].name "rankingExpression(myfeature).rankingScript" rankprofile[].fef.property[].value "70 * fieldMatch(title).completeness * pow(0 - fieldMatch(title).earliness,2) + 30 * pow(0 - fieldMatch(description).earliness,2)" rankprofile[].fef.property[].name "rankingExpression(mysummaryfeature).rankingScript" rankprofile[].fef.property[].value "80 * fieldMatch(title).completeness" -rankprofile[].fef.property[].name "rankingExpression(mysummaryfeature2).rankingScript" -rankprofile[].fef.property[].value "71 * fieldMatch(title).completeness" rankprofile[].fef.property[].name "rankingExpression(fourtimessum@2b1138e8965e7ff5.67f1e87166cfef86).rankingScript" rankprofile[].fef.property[].value "4 * (match + match)" +rankprofile[].fef.property[].name "rankingExpression(fourtimessum).rankingScript" +rankprofile[].fef.property[].value "4 * (var1 + var2)" +rankprofile[].fef.property[].name "rankingExpression(mysummaryfeature2).rankingScript" +rankprofile[].fef.property[].value "71 * fieldMatch(title).completeness" rankprofile[].fef.property[].name "vespa.rank.firstphase" rankprofile[].fef.property[].value "rankingExpression(firstphase)" rankprofile[].fef.property[].name "rankingExpression(firstphase).rankingScript" @@ -199,16 +199,16 @@ rankprofile[].fef.property[].value "rankingExpression(mysummaryfeature)" rankprofile[].name "macros-inherited2" rankprofile[].fef.property[].name "foo" rankprofile[].fef.property[].value "some, list" -rankprofile[].fef.property[].name "rankingExpression(fourtimessum).rankingScript" -rankprofile[].fef.property[].value "4 * (var1 + var2)" -rankprofile[].fef.property[].name "rankingExpression(myfeature).rankingScript" -rankprofile[].fef.property[].value "70 * fieldMatch(title).completeness * pow(0 - fieldMatch(title).earliness,2) + 30 * pow(0 - fieldMatch(description).earliness,2)" rankprofile[].fef.property[].name "rankingExpression(mysummaryfeature).rankingScript" rankprofile[].fef.property[].value "80 * fieldMatch(title).completeness" -rankprofile[].fef.property[].name "rankingExpression(mysummaryfeature2).rankingScript" -rankprofile[].fef.property[].value "71 * fieldMatch(title).completeness" +rankprofile[].fef.property[].name "rankingExpression(myfeature).rankingScript" +rankprofile[].fef.property[].value "70 * fieldMatch(title).completeness * pow(0 - fieldMatch(title).earliness,2) + 30 * pow(0 - fieldMatch(description).earliness,2)" rankprofile[].fef.property[].name "rankingExpression(fourtimessum@2b1138e8965e7ff5.67f1e87166cfef86).rankingScript" rankprofile[].fef.property[].value "4 * (match + match)" +rankprofile[].fef.property[].name "rankingExpression(fourtimessum).rankingScript" +rankprofile[].fef.property[].value "4 * (var1 + var2)" +rankprofile[].fef.property[].name "rankingExpression(mysummaryfeature2).rankingScript" +rankprofile[].fef.property[].value "71 * fieldMatch(title).completeness" rankprofile[].fef.property[].name "vespa.rank.firstphase" rankprofile[].fef.property[].value "rankingExpression(firstphase)" rankprofile[].fef.property[].name "rankingExpression(firstphase).rankingScript" @@ -224,12 +224,12 @@ rankprofile[].fef.property[].value "rankingExpression(mysummaryfeature)" rankprofile[].name "macros-inherited3" rankprofile[].fef.property[].name "foo" rankprofile[].fef.property[].value "some, list" -rankprofile[].fef.property[].name "rankingExpression(fourtimessum).rankingScript" -rankprofile[].fef.property[].value "4 * (var1 + var2)" -rankprofile[].fef.property[].name "rankingExpression(myfeature).rankingScript" -rankprofile[].fef.property[].value "700 * fieldMatch(title).completeness" rankprofile[].fef.property[].name "rankingExpression(mysummaryfeature).rankingScript" rankprofile[].fef.property[].value "80 * fieldMatch(title).completeness" +rankprofile[].fef.property[].name "rankingExpression(myfeature).rankingScript" +rankprofile[].fef.property[].value "700 * fieldMatch(title).completeness" +rankprofile[].fef.property[].name "rankingExpression(fourtimessum).rankingScript" +rankprofile[].fef.property[].value "4 * (var1 + var2)" rankprofile[].fef.property[].name "rankingExpression(mysummaryfeature2).rankingScript" rankprofile[].fef.property[].value "71 * fieldMatch(title).completeness" rankprofile[].fef.property[].name "vespa.rank.firstphase" @@ -254,20 +254,20 @@ rankprofile[].fef.property[].value "703 * fieldMatch(fromfile).completeness" rankprofile[].fef.property[].name "vespa.rank.secondphase" rankprofile[].fef.property[].value "rankingExpression(secondphase)" rankprofile[].fef.property[].name "rankingExpression(secondphase).rankingScript" -rankprofile[].fef.property[].value "40000 * rankingExpression(m2)" +rankprofile[].fef.property[].value "40000 * rankingExpression(m2) * rankingExpression(m4)" rankprofile[].name "macros-refering-macros-inherited" rankprofile[].fef.property[].name "rankingExpression(m1).rankingScript" rankprofile[].fef.property[].value "700 * fieldMatch(title).completeness" rankprofile[].fef.property[].name "rankingExpression(m2).rankingScript" rankprofile[].fef.property[].value "rankingExpression(m1) * 67" rankprofile[].fef.property[].name "rankingExpression(m4).rankingScript" -rankprofile[].fef.property[].value "701 * fieldMatch(title).completeness" +rankprofile[].fef.property[].value "703 * fieldMatch(fromfile).completeness" rankprofile[].fef.property[].name "rankingExpression(m3).rankingScript" rankprofile[].fef.property[].value "if (isNan(attribute(nrtgmp)) == 1, 0.0, rankingExpression(m2))" rankprofile[].fef.property[].name "vespa.rank.secondphase" rankprofile[].fef.property[].value "rankingExpression(secondphase)" rankprofile[].fef.property[].name "rankingExpression(secondphase).rankingScript" -rankprofile[].fef.property[].value "3000 * rankingExpression(m2)" +rankprofile[].fef.property[].value "3000 * rankingExpression(m2) * rankingExpression(m4)" rankprofile[].name "macros-refering-macros-inherited2" rankprofile[].fef.property[].name "rankingExpression(m1).rankingScript" rankprofile[].fef.property[].value "700 * fieldMatch(title).completeness" @@ -285,7 +285,7 @@ rankprofile[].fef.property[].value "700 * fieldMatch(title).completeness" rankprofile[].fef.property[].name "rankingExpression(m2).rankingScript" rankprofile[].fef.property[].value "rankingExpression(m1) * 67" rankprofile[].fef.property[].name "rankingExpression(m4).rankingScript" -rankprofile[].fef.property[].value "701 * fieldMatch(title).completeness" +rankprofile[].fef.property[].value "703 * fieldMatch(fromfile).completeness" rankprofile[].fef.property[].name "rankingExpression(m3).rankingScript" rankprofile[].fef.property[].value "if (isNan(attribute(nrtgmp)) == 1, 0.0, rankingExpression(m2))" rankprofile[].fef.property[].name "rankingExpression(m5).rankingScript" @@ -293,4 +293,4 @@ rankprofile[].fef.property[].value "if (isNan(attribute(glmpfw)) == 1, rankingEx rankprofile[].fef.property[].name "vespa.rank.secondphase" rankprofile[].fef.property[].value "rankingExpression(secondphase)" rankprofile[].fef.property[].name "rankingExpression(secondphase).rankingScript" -rankprofile[].fef.property[].value "3000 * rankingExpression(m2)" +rankprofile[].fef.property[].value "3000 * rankingExpression(m2) * rankingExpression(m4)" diff --git a/config-model/src/test/derived/rankexpression/rankexpression.sd b/config-model/src/test/derived/rankexpression/rankexpression.sd index d3e0057cfe1..20f9c7a9160 100644 --- a/config-model/src/test/derived/rankexpression/rankexpression.sd +++ b/config-model/src/test/derived/rankexpression/rankexpression.sd @@ -276,7 +276,7 @@ search rankexpression { second-phase { expression { - 40000 * m2 + 40000 * m2 * m4 } } @@ -291,14 +291,9 @@ search rankexpression { ) } } - macro m4() { - expression { - 701 * fieldMatch(title).completeness - } - } second-phase { expression { - 3000 * m2 + 3000 * m2 * m4 } } } @@ -324,4 +319,3 @@ search rankexpression { } - diff --git a/config-model/src/test/java/com/yahoo/config/model/ApplicationDeployTest.java b/config-model/src/test/java/com/yahoo/config/model/ApplicationDeployTest.java index 75cb41be13f..8f751631fb5 100644 --- a/config-model/src/test/java/com/yahoo/config/model/ApplicationDeployTest.java +++ b/config-model/src/test/java/com/yahoo/config/model/ApplicationDeployTest.java @@ -46,7 +46,7 @@ import static org.junit.Assert.fail; public class ApplicationDeployTest { private static final String TESTDIR = "src/test/cfg/application/"; - private static final String TESTSDDIR = TESTDIR + "app1/searchdefinitions/"; + private static final String TEST_SCHEMAS_DIR = TESTDIR + "app1/schemas/"; @Rule public TemporaryFolder tmpFolder = new TemporaryFolder(); @@ -54,7 +54,7 @@ public class ApplicationDeployTest { @Test public void testVespaModel() throws SAXException, IOException { ApplicationPackageTester tester = ApplicationPackageTester.create(TESTDIR + "app1"); - VespaModel model = new VespaModel(tester.app()); + new VespaModel(tester.app()); List<NamedSchema> schemas = tester.getSchemas(); assertEquals(schemas.size(), 5); for (NamedSchema searchDefinition : schemas) { @@ -73,11 +73,11 @@ public class ApplicationDeployTest { fail(); } } - assertEquals(Set.of(new File(TESTSDDIR + "laptop.sd"), - new File(TESTSDDIR + "music.sd"), - new File(TESTSDDIR + "pc.sd"), - new File(TESTSDDIR + "product.sd"), - new File(TESTSDDIR + "sock.sd")), + assertEquals(Set.of(new File(TEST_SCHEMAS_DIR + "laptop.sd"), + new File(TEST_SCHEMAS_DIR + "music.sd"), + new File(TEST_SCHEMAS_DIR + "pc.sd"), + new File(TEST_SCHEMAS_DIR + "product.sd"), + new File(TEST_SCHEMAS_DIR + "sock.sd")), new HashSet<>(tester.app().getSearchDefinitionFiles())); List<FilesApplicationPackage.Component> components = tester.app().getComponents(); diff --git a/config-model/src/test/java/com/yahoo/config/model/provision/ModelProvisioningTest.java b/config-model/src/test/java/com/yahoo/config/model/provision/ModelProvisioningTest.java index 2b72420614d..b0ddadf11bd 100644 --- a/config-model/src/test/java/com/yahoo/config/model/provision/ModelProvisioningTest.java +++ b/config-model/src/test/java/com/yahoo/config/model/provision/ModelProvisioningTest.java @@ -38,7 +38,6 @@ import com.yahoo.yolean.Exceptions; import org.junit.Test; import java.io.StringReader; -import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Optional; @@ -50,6 +49,7 @@ import static com.yahoo.config.model.test.TestUtil.joinLines; import static com.yahoo.vespa.defaults.Defaults.getDefaults; import static com.yahoo.vespa.model.search.NodeResourcesTuning.GB; import static com.yahoo.vespa.model.search.NodeResourcesTuning.reservedMemoryGb; +import static com.yahoo.vespa.model.test.utils.ApplicationPackageUtils.generateSchemas; import static org.hamcrest.CoreMatchers.is; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; @@ -502,7 +502,6 @@ public class ModelProvisioningTest { // Check content clusters ContentCluster cluster = model.getContentClusters().get("bar"); - assertNull("No own cluster controllers when hosted", cluster.getClusterControllers()); assertEquals(0, cluster.getRootGroup().getNodes().size()); assertEquals(9, cluster.getRootGroup().getSubgroups().size()); assertEquals("0", cluster.getRootGroup().getSubgroups().get(0).getIndex()); @@ -536,7 +535,6 @@ public class ModelProvisioningTest { assertThat(cluster.getRootGroup().getSubgroups().get(8).getNodes().get(2).getConfigId(), is("bar/storage/26")); cluster = model.getContentClusters().get("baz"); - assertNull("No own cluster controllers when hosted", cluster.getClusterControllers()); assertEquals(0, cluster.getRootGroup().getNodes().size()); assertEquals(27, cluster.getRootGroup().getSubgroups().size()); assertThat(cluster.getRootGroup().getSubgroups().get(0).getIndex(), is("0")); @@ -730,7 +728,6 @@ public class ModelProvisioningTest { // Check content cluster ContentCluster cluster = model.getContentClusters().get("bar"); - assertNull(cluster.getClusterControllers()); assertEquals(0, cluster.getRootGroup().getNodes().size()); assertEquals(8, cluster.getRootGroup().getSubgroups().size()); assertEquals(8, cluster.distributionBits()); @@ -865,8 +862,6 @@ public class ModelProvisioningTest { assertEquals(7, model.getRoot().hostSystem().getHosts().size()); // Check cluster controllers - assertNull(model.getContentClusters().get("foo").getClusterControllers()); - assertNull(model.getContentClusters().get("bar").getClusterControllers()); ClusterControllerContainerCluster clusterControllers = model.getAdmin().getClusterControllers(); assertEquals(3, clusterControllers.getContainers().size()); assertEquals("cluster-controllers", clusterControllers.getName()); @@ -2021,7 +2016,7 @@ public class ModelProvisioningTest { } private VespaModel createNonProvisionedModel(boolean multitenant, String hosts, String services) { - VespaModelCreatorWithMockPkg modelCreatorWithMockPkg = new VespaModelCreatorWithMockPkg(hosts, services, ApplicationPackageUtils.generateSearchDefinition("type1")); + VespaModelCreatorWithMockPkg modelCreatorWithMockPkg = new VespaModelCreatorWithMockPkg(hosts, services, generateSchemas("type1")); ApplicationPackage appPkg = modelCreatorWithMockPkg.appPkg; DeployState deployState = new DeployState.Builder().applicationPackage(appPkg). properties((new TestProperties()).setMultitenant(multitenant)). @@ -2029,7 +2024,7 @@ public class ModelProvisioningTest { return modelCreatorWithMockPkg.create(false, deployState); } - private int physicalMemoryPercentage(ContainerCluster cluster) { + private int physicalMemoryPercentage(ContainerCluster<?> cluster) { QrStartConfig.Builder b = new QrStartConfig.Builder(); cluster.getConfig(b); return b.build().jvm().heapSizeAsPercentageOfPhysicalMemory(); diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/derived/ExportingTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/derived/ExportingTestCase.java index 125e25470df..370315ad08b 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/derived/ExportingTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/derived/ExportingTestCase.java @@ -105,7 +105,7 @@ public class ExportingTestCase extends AbstractExportingTestCase { @Test public void testRankExpression() throws IOException, ParseException { assertCorrectDeriving("rankexpression", null, - new TestProperties().useExternalRankExpression(false), new TestableDeployLogger()); + new TestProperties().useExternalRankExpression(true), new TestableDeployLogger()); } @Test diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionsTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionsTestCase.java index ff17a211e17..85ef70132b5 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionsTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionsTestCase.java @@ -52,22 +52,22 @@ public class RankingExpressionsTestCase extends SchemaTestCase { RawRankProfile rawRankProfile = new RawRankProfile(functionsRankProfile, new QueryProfileRegistry(), new ImportedMlModels(), new AttributeFields(search), deployProperties); List<Pair<String, String>> rankProperties = rawRankProfile.configProperties(); - assertEquals(5, rankProperties.size()); + assertEquals(6, rankProperties.size()); - assertEquals("rankingExpression(titlematch$@126063073eb2deb.ab95cd69909927c).rankingScript", rankProperties.get(0).getFirst()); - assertEquals("4 * 5 + 890", rankProperties.get(0).getSecond()); + assertEquals("rankingExpression(titlematch$).rankingScript", rankProperties.get(2).getFirst()); + assertEquals("var1 * var2 + 890", rankProperties.get(2).getSecond()); - assertEquals("rankingExpression(titlematch$@c7e4c2d0e6d9f2a1.1d4ed08e56cce2e6).rankingScript", rankProperties.get(1).getFirst()); - assertEquals("7 * 8 + 890", rankProperties.get(1).getSecond()); + assertEquals("rankingExpression(artistmatch).rankingScript", rankProperties.get(3).getFirst()); + assertEquals("78 + closeness(distance)", rankProperties.get(3).getSecond()); - assertEquals("rankingExpression(artistmatch).rankingScript", rankProperties.get(2).getFirst()); - assertEquals("78 + closeness(distance)", rankProperties.get(2).getSecond()); + assertEquals("rankingExpression(firstphase).rankingScript", rankProperties.get(5).getFirst()); + assertEquals("0.8 + 0.2 * rankingExpression(titlematch$@126063073eb2deb.ab95cd69909927c) + 0.8 * rankingExpression(titlematch$@c7e4c2d0e6d9f2a1.1d4ed08e56cce2e6) * closeness(distance)", rankProperties.get(5).getSecond()); - assertEquals("vespa.rank.firstphase", rankProperties.get(3).getFirst()); - assertEquals("rankingExpression(firstphase)", rankProperties.get(3).getSecond()); + assertEquals("rankingExpression(titlematch$@c7e4c2d0e6d9f2a1.1d4ed08e56cce2e6).rankingScript", rankProperties.get(1).getFirst()); + assertEquals("7 * 8 + 890", rankProperties.get(1).getSecond()); - assertEquals("rankingExpression(firstphase).rankingScript", rankProperties.get(4).getFirst()); - assertEquals("0.8 + 0.2 * rankingExpression(titlematch$@126063073eb2deb.ab95cd69909927c) + 0.8 * rankingExpression(titlematch$@c7e4c2d0e6d9f2a1.1d4ed08e56cce2e6) * closeness(distance)", rankProperties.get(4).getSecond()); + assertEquals("rankingExpression(titlematch$@126063073eb2deb.ab95cd69909927c).rankingScript", rankProperties.get(0).getFirst()); + assertEquals("4 * 5 + 890", rankProperties.get(0).getSecond()); } @Test(expected = IllegalArgumentException.class) diff --git a/config-model/src/test/java/com/yahoo/vespa/documentmodel/DocumentModelBuilderTestCase.java b/config-model/src/test/java/com/yahoo/vespa/documentmodel/DocumentModelBuilderTestCase.java index 91152648b10..8f9aa33ea52 100644 --- a/config-model/src/test/java/com/yahoo/vespa/documentmodel/DocumentModelBuilderTestCase.java +++ b/config-model/src/test/java/com/yahoo/vespa/documentmodel/DocumentModelBuilderTestCase.java @@ -49,9 +49,9 @@ public class DocumentModelBuilderTestCase extends SchemaTestCase { @Test public void testMultipleInheritanceArray() throws IOException, ParseException { SearchBuilder search = new SearchBuilder(); - search.importFile("src/test/cfg/search/data/travel/searchdefinitions/TTData.sd"); - search.importFile("src/test/cfg/search/data/travel/searchdefinitions/TTEdge.sd"); - search.importFile("src/test/cfg/search/data/travel/searchdefinitions/TTPOI.sd"); + search.importFile("src/test/cfg/search/data/travel/schemas/TTData.sd"); + search.importFile("src/test/cfg/search/data/travel/schemas/TTEdge.sd"); + search.importFile("src/test/cfg/search/data/travel/schemas/TTPOI.sd"); search.build(); } diff --git a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/ComplexAttributeFieldsValidatorTestCase.java b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/ComplexAttributeFieldsValidatorTestCase.java index 76f34cf4a81..341a90c6618 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/ComplexAttributeFieldsValidatorTestCase.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/ComplexAttributeFieldsValidatorTestCase.java @@ -15,6 +15,7 @@ import org.junit.rules.ExpectedException; import org.xml.sax.SAXException; import java.io.IOException; +import java.util.List; import static com.yahoo.config.model.test.TestUtil.joinLines; @@ -99,17 +100,17 @@ public class ComplexAttributeFieldsValidatorTestCase { "}")); } - private static void createModelAndValidate(String searchDefinition) throws IOException, SAXException { - DeployState deployState = createDeployState(servicesXml(), searchDefinition); + private static void createModelAndValidate(String schema) throws IOException, SAXException { + DeployState deployState = createDeployState(servicesXml(), schema); VespaModel model = new VespaModel(new NullConfigModelRegistry(), deployState); ValidationParameters validationParameters = new ValidationParameters(CheckRouting.FALSE); Validation.validate(model, validationParameters, deployState); } - private static DeployState createDeployState(String servicesXml, String searchDefinition) { + private static DeployState createDeployState(String servicesXml, String schema) { ApplicationPackage app = new MockApplicationPackage.Builder() .withServices(servicesXml) - .withSearchDefinition(searchDefinition) + .withSchemas(List.of(schema)) .build(); return new DeployState.Builder().applicationPackage(app).build(); } diff --git a/config-model/src/test/java/com/yahoo/vespa/model/builder/xml/dom/ContentBuilderTest.java b/config-model/src/test/java/com/yahoo/vespa/model/builder/xml/dom/ContentBuilderTest.java index ad4603e5c6b..7bff4890b7e 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/builder/xml/dom/ContentBuilderTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/builder/xml/dom/ContentBuilderTest.java @@ -824,24 +824,6 @@ public class ContentBuilderTest extends DomBuilderTest { verifyThatFeatureFlagControlsVisibilityDelayDefault(0.6, 0.6); } - private void verifyThatFeatureFlagControlsUseBucketExecutorForPruneRemoved(boolean flag) { - DeployState.Builder deployStateBuilder = new DeployState.Builder().properties(new TestProperties().useBucketExecutorForPruneRemoved(flag)); - VespaModel model = new VespaModelCreatorWithMockPkg(new MockApplicationPackage.Builder() - .withServices(singleNodeContentXml()) - .withSearchDefinition(MockApplicationPackage.MUSIC_SEARCHDEFINITION) - .build()) - .create(deployStateBuilder); - ProtonConfig config = getProtonConfig(model.getContentClusters().values().iterator().next()); - assertEquals(flag, config.pruneremoveddocuments().usebucketexecutor()); - } - - - @Test - public void verifyUseBucketExecutorForPruneRemoved() { - verifyThatFeatureFlagControlsUseBucketExecutorForPruneRemoved(true); - verifyThatFeatureFlagControlsUseBucketExecutorForPruneRemoved(false); - } - @Test public void failWhenNoDocumentsElementSpecified() { expectedException.expect(IllegalArgumentException.class); diff --git a/config-model/src/test/java/com/yahoo/vespa/model/container/xml/AccessLogTest.java b/config-model/src/test/java/com/yahoo/vespa/model/container/xml/AccessLogTest.java index f3199f6a46f..80aafdd4ec7 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/container/xml/AccessLogTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/container/xml/AccessLogTest.java @@ -69,7 +69,7 @@ public class AccessLogTest extends ContainerModelBuilderTestBase { " <accesslog type='vespa' ", " fileNamePattern='pattern' rotationInterval='interval' />", " <accesslog type='json' ", - " fileNamePattern='pattern' rotationInterval='interval' />", + " fileNamePattern='pattern' rotationInterval='interval' queueSize='17' bufferSize='65536'/>", nodesXml, "</container>" ); @@ -85,8 +85,8 @@ public class AccessLogTest extends ContainerModelBuilderTestBase { AccessLogConfig.FileHandler fileHandlerConfig = config.fileHandler(); assertEquals("pattern", fileHandlerConfig.pattern()); assertEquals("interval", fileHandlerConfig.rotation()); - assertEquals(10000, fileHandlerConfig.queueSize()); - assertEquals(4*1024*1024, fileHandlerConfig.bufferSize()); + assertEquals(256, fileHandlerConfig.queueSize()); + assertEquals(256*1024, fileHandlerConfig.bufferSize()); } { // json @@ -97,8 +97,8 @@ public class AccessLogTest extends ContainerModelBuilderTestBase { AccessLogConfig.FileHandler fileHandlerConfig = config.fileHandler(); assertEquals("pattern", fileHandlerConfig.pattern()); assertEquals("interval", fileHandlerConfig.rotation()); - assertEquals(10000, fileHandlerConfig.queueSize()); - assertEquals(4*1024*1024, fileHandlerConfig.bufferSize()); + assertEquals(17, fileHandlerConfig.queueSize()); + assertEquals(65536, fileHandlerConfig.bufferSize()); } } diff --git a/config-model/src/test/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilderTest.java b/config-model/src/test/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilderTest.java index 7f862afa1b0..543318f9224 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilderTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilderTest.java @@ -40,6 +40,11 @@ import com.yahoo.net.HostName; import com.yahoo.path.Path; import com.yahoo.prelude.cluster.QrMonitorConfig; import com.yahoo.search.config.QrStartConfig; +import com.yahoo.security.KeyAlgorithm; +import com.yahoo.security.KeyUtils; +import com.yahoo.security.SignatureAlgorithm; +import com.yahoo.security.X509CertificateBuilder; +import com.yahoo.security.X509CertificateUtils; import com.yahoo.security.tls.TlsContext; import com.yahoo.vespa.defaults.Defaults; import com.yahoo.vespa.model.AbstractService; @@ -53,6 +58,7 @@ import com.yahoo.vespa.model.container.http.ConnectorFactory; import com.yahoo.vespa.model.content.utils.ContentClusterUtils; import com.yahoo.vespa.model.test.VespaModelTester; import com.yahoo.vespa.model.test.utils.VespaModelCreatorWithFilePkg; +import org.hamcrest.CoreMatchers; import org.hamcrest.Matchers; import org.hamcrest.core.IsEqual; import org.junit.Rule; @@ -61,8 +67,15 @@ import org.junit.rules.TemporaryFolder; import org.w3c.dom.Element; import org.xml.sax.SAXException; +import javax.security.auth.x500.X500Principal; import java.io.IOException; import java.io.StringReader; +import java.math.BigInteger; +import java.security.KeyPair; +import java.security.cert.X509Certificate; +import java.time.Duration; +import java.time.Instant; +import java.time.temporal.ChronoUnit; import java.util.ArrayList; import java.util.HashSet; import java.util.List; @@ -82,6 +95,7 @@ import static org.hamcrest.CoreMatchers.is; import static org.hamcrest.CoreMatchers.not; import static org.hamcrest.CoreMatchers.notNullValue; import static org.hamcrest.CoreMatchers.nullValue; +import static org.hamcrest.Matchers.arrayContainingInAnyOrder; import static org.hamcrest.Matchers.contains; import static org.hamcrest.Matchers.containsInAnyOrder; import static org.hamcrest.Matchers.containsString; @@ -818,6 +832,48 @@ public class ContainerModelBuilderTest extends ContainerModelBuilderTestBase { } @Test + public void operator_certificates_are_joined_with_clients_pem() { + var applicationPackage = new MockApplicationPackage.Builder() + .withRoot(applicationFolder.getRoot()) + .build(); + + var applicationTrustCert = X509CertificateUtils.toPem( + X509CertificateUtils.createSelfSigned("CN=application", Duration.ofDays(1)).certificate()); + var operatorCert = X509CertificateUtils.createSelfSigned("CN=operator", Duration.ofDays(1)).certificate(); + + applicationPackage.getFile(Path.fromString("security")).createDirectory(); + applicationPackage.getFile(Path.fromString("security/clients.pem")).writeFile(new StringReader(applicationTrustCert)); + + var deployState = new DeployState.Builder().properties( + new TestProperties() + .setOperatorCertificates(List.of(operatorCert)) + .setHostedVespa(true) + .setEndpointCertificateSecrets(Optional.of(new EndpointCertificateSecrets("CERT", "KEY")))) + .zone(new Zone(SystemName.PublicCd, Environment.dev, RegionName.defaultName())) + .applicationPackage(applicationPackage) + .build(); + + Element clusterElem = DomBuilderTest.parse("<container version='1.0' />"); + + createModel(root, deployState, null, clusterElem); + + ApplicationContainer container = (ApplicationContainer)root.getProducer("container/container.0"); + List<ConnectorFactory> connectorFactories = container.getHttp().getHttpServer().get().getConnectorFactories(); + ConnectorFactory tlsPort = connectorFactories.stream().filter(connectorFactory -> connectorFactory.getListenPort() == 4443).findFirst().orElseThrow(); + + ConnectorConfig.Builder builder = new ConnectorConfig.Builder(); + tlsPort.getConfig(builder); + + ConnectorConfig connectorConfig = new ConnectorConfig(builder); + var caCerts = X509CertificateUtils.certificateListFromPem(connectorConfig.ssl().caCertificate()); + assertEquals(2, caCerts.size()); + List<String> certnames = caCerts.stream() + .map(cert -> cert.getSubjectX500Principal().getName()) + .collect(Collectors.toList()); + assertThat(certnames, containsInAnyOrder("CN=operator", "CN=application")); + } + + @Test public void environment_vars_are_honoured() { Element clusterElem = DomBuilderTest.parse( "<container version='1.0'>", diff --git a/config-model/src/test/java/com/yahoo/vespa/model/content/ContentClusterTest.java b/config-model/src/test/java/com/yahoo/vespa/model/content/ContentClusterTest.java index 953c42243a6..13d02fc1fb8 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/content/ContentClusterTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/content/ContentClusterTest.java @@ -26,7 +26,6 @@ import com.yahoo.vespa.model.VespaModel; import com.yahoo.vespa.model.admin.clustercontroller.ClusterControllerContainer; import com.yahoo.vespa.model.admin.clustercontroller.ClusterControllerContainerCluster; import com.yahoo.vespa.model.container.ContainerCluster; -import com.yahoo.vespa.model.container.component.Component; import com.yahoo.vespa.model.content.cluster.ContentCluster; import com.yahoo.vespa.model.content.engines.ProtonEngine; import com.yahoo.vespa.model.content.utils.ContentClusterBuilder; @@ -46,7 +45,6 @@ import java.util.List; import java.util.Map; import java.util.Optional; -import static java.util.stream.Collectors.toList; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertNotNull; @@ -533,6 +531,7 @@ public class ContentClusterTest extends ContentBaseTest { ContentCluster stagingNot16Bits = createWithZone(xml, new Zone(Environment.staging, RegionName.from("us-east-3"))); assertDistributionBitsInConfig(stagingNot16Bits, 8); } + @Test public void testGenerateSearchNodes() { @@ -1076,7 +1075,6 @@ public class ContentClusterTest extends ContentBaseTest { " </documents>" + " </content>" + " </services>"); - assertNull("No own cluster controller for content", oneContentModel.getContentClusters().get("storage").getClusterControllers()); assertNotNull("Shared cluster controller with content", oneContentModel.getAdmin().getClusterControllers()); String twoContentServices = "<?xml version='1.0' encoding='UTF-8' ?>" + @@ -1108,8 +1106,6 @@ public class ContentClusterTest extends ContentBaseTest { VespaModel twoContentModel = createEnd2EndOneNode(new TestProperties().setHostedVespa(true) .setMultitenant(true), twoContentServices); - assertNull("No own cluster controller for content", twoContentModel.getContentClusters().get("storage").getClusterControllers()); - assertNull("No own cluster controller for content", twoContentModel.getContentClusters().get("dev-null").getClusterControllers()); assertNotNull("Shared cluster controller with content", twoContentModel.getAdmin().getClusterControllers()); ClusterControllerContainerCluster clusterControllers = twoContentModel.getAdmin().getClusterControllers(); diff --git a/config-model/src/test/java/com/yahoo/vespa/model/content/StorageClusterTest.java b/config-model/src/test/java/com/yahoo/vespa/model/content/StorageClusterTest.java index 5cf57430f91..9a681003293 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/content/StorageClusterTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/content/StorageClusterTest.java @@ -17,6 +17,7 @@ import com.yahoo.vespa.config.content.PersistenceConfig; import com.yahoo.config.model.test.MockRoot; import com.yahoo.documentmodel.NewDocumentType; import static com.yahoo.vespa.defaults.Defaults.getDefaults; +import static com.yahoo.config.model.test.TestUtil.joinLines; import com.yahoo.vespa.model.content.cluster.ContentCluster; import com.yahoo.vespa.model.content.storagecluster.StorageCluster; import com.yahoo.vespa.model.content.utils.ContentClusterUtils; @@ -44,10 +45,17 @@ public class StorageClusterTest { return parse(xml, root); } - StorageCluster parse(String xml) { - MockRoot root = new MockRoot(); + StorageCluster parse(String xml, ModelContext.Properties properties) { + MockRoot root = new MockRoot("", + new DeployState.Builder() + .properties(properties) + .applicationPackage(new MockApplicationPackage.Builder().build()) + .build()); return parse(xml, root); } + StorageCluster parse(String xml) { + return parse(xml, new TestProperties()); + } StorageCluster parse(String xml, MockRoot root) { root.getDeployState().getDocumentModel().getDocumentManager().add( new NewDocumentType(new NewDocumentType.Name("music")) @@ -61,13 +69,23 @@ public class StorageClusterTest { return cluster.getStorageNodes(); } + private static String group() { + return joinLines( + "<group>", + " <node distribution-key=\"0\" hostalias=\"mockhost\"/>", + "</group>"); + } + private static String cluster(String clusterName, String insert) { + return joinLines( + "<content id=\"" + clusterName + "\">", + "<documents/>", + insert, + group(), + "</content>"); + } @Test public void testBasics() { - StorageCluster storage = parse("<content id=\"foofighters\"><documents/>\n" + - " <group>" + - " <node distribution-key=\"0\" hostalias=\"mockhost\"/>" + - " </group>" + - "</content>\n"); + StorageCluster storage = parse(cluster("foofighters", "")); assertEquals(1, storage.getChildren().size()); StorServerConfig.Builder builder = new StorServerConfig.Builder(); @@ -79,11 +97,7 @@ public class StorageClusterTest { } @Test public void testCommunicationManagerDefaults() { - StorageCluster storage = parse("<content id=\"foofighters\"><documents/>\n" + - " <group>" + - " <node distribution-key=\"0\" hostalias=\"mockhost\"/>" + - " </group>" + - "</content>\n"); + StorageCluster storage = parse(cluster("foofighters", "")); StorCommunicationmanagerConfig.Builder builder = new StorCommunicationmanagerConfig.Builder(); storage.getChildren().get("0").getConfig(builder); StorCommunicationmanagerConfig config = new StorCommunicationmanagerConfig(builder); @@ -97,40 +111,49 @@ public class StorageClusterTest { } @Test + public void testMergeDefaults() { + StorServerConfig.Builder builder = new StorServerConfig.Builder(); + parse(cluster("foofighters", "")).getConfig(builder); + + StorServerConfig config = new StorServerConfig(builder); + assertEquals(16, config.max_merges_per_node()); + assertEquals(1024, config.max_merge_queue_size()); + } + + @Test public void testMerges() { StorServerConfig.Builder builder = new StorServerConfig.Builder(); - parse("" + - "<content id=\"foofighters\">\n" + - " <documents/>" + - " <tuning>" + - " <merges max-per-node=\"1K\" max-queue-size=\"10K\"/>\n" + - " </tuning>" + - " <group>" + - " <node distribution-key=\"0\" hostalias=\"mockhost\"/>" + - " </group>" + - "</content>" + parse(cluster("foofighters", joinLines( + "<tuning>", + " <merges max-per-node=\"1K\" max-queue-size=\"10K\"/>", + "</tuning>")), + new TestProperties().setMaxMergeQueueSize(1919).setMaxConcurrentMergesPerNode(37) ).getConfig(builder); StorServerConfig config = new StorServerConfig(builder); assertEquals(1024, config.max_merges_per_node()); assertEquals(1024*10, config.max_merge_queue_size()); } + @Test + public void testMergeFeatureFlags() { + StorServerConfig.Builder builder = new StorServerConfig.Builder(); + parse(cluster("foofighters", ""), new TestProperties().setMaxMergeQueueSize(1919).setMaxConcurrentMergesPerNode(37)).getConfig(builder); + + StorServerConfig config = new StorServerConfig(builder); + assertEquals(37, config.max_merges_per_node()); + assertEquals(1919, config.max_merge_queue_size()); + } @Test public void testVisitors() { StorVisitorConfig.Builder builder = new StorVisitorConfig.Builder(); - parse( - "<cluster id=\"bees\">\n" + - " <documents/>" + - " <tuning>\n" + - " <visitors thread-count=\"7\" max-queue-size=\"1000\">\n" + - " <max-concurrent fixed=\"42\" variable=\"100\"/>\n" + - " </visitors>\n" + - " </tuning>\n" + - " <group>" + - " <node distribution-key=\"0\" hostalias=\"mockhost\"/>" + - " </group>" + - "</cluster>" + parse(cluster("bees", + joinLines( + "<tuning>", + " <visitors thread-count=\"7\" max-queue-size=\"1000\">", + " <max-concurrent fixed=\"42\" variable=\"100\"/>", + " </visitors>", + "</tuning>")) ).getConfig(builder); StorVisitorConfig config = new StorVisitorConfig(builder); @@ -143,16 +166,10 @@ public class StorageClusterTest { @Test public void testPersistenceThreads() { - StorageCluster stc = parse( - "<cluster id=\"bees\">\n" + - " <documents/>" + - " <tuning>\n" + - " <persistence-threads count=\"7\"/>\n" + - " </tuning>\n" + - " <group>" + - " <node distribution-key=\"0\" hostalias=\"mockhost\"/>" + - " </group>" + - "</cluster>", + StorageCluster stc = parse(cluster("bees",joinLines( + "<tuning>", + " <persistence-threads count=\"7\"/>", + "</tuning>")), new Flavor(new FlavorsConfig.Flavor.Builder().name("test-flavor").minCpuCores(9).build()) ); @@ -178,16 +195,10 @@ public class StorageClusterTest { @Test public void testResponseThreads() { - StorageCluster stc = parse( - "<cluster id=\"bees\">\n" + - " <documents/>" + - " <tuning>\n" + - " <persistence-threads count=\"7\"/>\n" + - " </tuning>\n" + - " <group>" + - " <node distribution-key=\"0\" hostalias=\"mockhost\"/>" + - " </group>" + - "</cluster>", + StorageCluster stc = parse(cluster("bees",joinLines( + "<tuning>", + " <persistence-threads count=\"7\"/>", + "</tuning>")), new Flavor(new FlavorsConfig.Flavor.Builder().name("test-flavor").minCpuCores(9).build()) ); StorFilestorConfig.Builder builder = new StorFilestorConfig.Builder(); @@ -201,20 +212,14 @@ public class StorageClusterTest { @Test public void testPersistenceThreadsOld() { - StorageCluster stc = parse( - "<cluster id=\"bees\">\n" + - " <documents/>" + - " <tuning>\n" + - " <persistence-threads>\n" + - " <thread lowest-priority=\"VERY_LOW\" count=\"2\"/>\n" + - " <thread lowest-priority=\"VERY_HIGH\" count=\"1\"/>\n" + - " <thread count=\"1\"/>\n" + - " </persistence-threads>\n" + - " </tuning>\n" + - " <group>" + - " <node distribution-key=\"0\" hostalias=\"mockhost\"/>" + - " </group>" + - "</cluster>", + StorageCluster stc = parse(cluster("bees", joinLines( + "<tuning>", + " <persistence-threads>", + " <thread lowest-priority=\"VERY_LOW\" count=\"2\"/>", + " <thread lowest-priority=\"VERY_HIGH\" count=\"1\"/>", + " <thread count=\"1\"/>", + " </persistence-threads>", + "</tuning>")), new Flavor(new FlavorsConfig.Flavor.Builder().name("test-flavor").minCpuCores(9).build()) ); @@ -238,15 +243,7 @@ public class StorageClusterTest { @Test public void testNoPersistenceThreads() { - StorageCluster stc = parse( - "<cluster id=\"bees\">\n" + - " <documents/>" + - " <tuning>\n" + - " </tuning>\n" + - " <group>" + - " <node distribution-key=\"0\" hostalias=\"mockhost\"/>" + - " </group>" + - "</cluster>", + StorageCluster stc = parse(cluster("bees", ""), new Flavor(new FlavorsConfig.Flavor.Builder().name("test-flavor").minCpuCores(9).build()) ); @@ -267,13 +264,7 @@ public class StorageClusterTest { } private StorageCluster simpleCluster(ModelContext.Properties properties) { - return parse( - "<cluster id=\"bees\">\n" + - " <documents/>" + - " <group>" + - " <node distribution-key=\"0\" hostalias=\"mockhost\"/>" + - " </group>" + - "</cluster>", + return parse(cluster("bees", ""), new Flavor(new FlavorsConfig.Flavor.Builder().name("test-flavor").minCpuCores(9).build()), properties); } @@ -302,14 +293,7 @@ public class StorageClusterTest { @Test public void integrity_checker_explicitly_disabled_when_not_running_with_vds_provider() { StorIntegritycheckerConfig.Builder builder = new StorIntegritycheckerConfig.Builder(); - parse( - "<cluster id=\"bees\">\n" + - " <documents/>" + - " <group>" + - " <node distribution-key=\"0\" hostalias=\"mockhost\"/>" + - " </group>" + - "</cluster>" - ).getConfig(builder); + parse(cluster("bees", "")).getConfig(builder); StorIntegritycheckerConfig config = new StorIntegritycheckerConfig(builder); // '-' --> don't run on the given week day assertEquals("-------", config.weeklycycle()); @@ -317,15 +301,15 @@ public class StorageClusterTest { @Test public void testCapacity() { - String xml = - "<cluster id=\"storage\">\n" + - " <documents/>" + - " <group>\n" + - " <node distribution-key=\"0\" hostalias=\"mockhost\"/>\n" + - " <node distribution-key=\"1\" hostalias=\"mockhost\" capacity=\"1.5\"/>\n" + - " <node distribution-key=\"2\" hostalias=\"mockhost\" capacity=\"2.0\"/>\n" + - " </group>\n" + - "</cluster>"; + String xml = joinLines( + "<cluster id=\"storage\">", + " <documents/>", + " <group>", + " <node distribution-key=\"0\" hostalias=\"mockhost\"/>", + " <node distribution-key=\"1\" hostalias=\"mockhost\" capacity=\"1.5\"/>", + " <node distribution-key=\"2\" hostalias=\"mockhost\" capacity=\"2.0\"/>", + " </group>", + "</cluster>"); ContentCluster cluster = ContentClusterUtils.createCluster(xml, new MockRoot()); @@ -341,15 +325,7 @@ public class StorageClusterTest { @Test public void testRootFolder() { - String xml = - "<cluster id=\"storage\">\n" + - " <documents/>" + - " <group>\n" + - " <node distribution-key=\"0\" hostalias=\"mockhost\"/>\n" + - " </group>\n" + - "</cluster>"; - - ContentCluster cluster = ContentClusterUtils.createCluster(xml, new MockRoot()); + ContentCluster cluster = ContentClusterUtils.createCluster(cluster("storage", ""), new MockRoot()); StorageNode node = cluster.getStorageNodes().getChildren().get("0"); @@ -372,18 +348,18 @@ public class StorageClusterTest { @Test public void testGenericPersistenceTuning() { - String xml = - "<cluster id=\"storage\">\n" + - "<documents/>" + - "<engine>\n" + - " <fail-partition-on-error>true</fail-partition-on-error>\n" + - " <revert-time>34m</revert-time>\n" + - " <recovery-time>5d</recovery-time>\n" + - "</engine>" + - " <group>\n" + - " <node distribution-key=\"0\" hostalias=\"mockhost\"/>\n" + - " </group>\n" + - "</cluster>"; + String xml = joinLines( + "<cluster id=\"storage\">", + " <documents/>", + " <engine>", + " <fail-partition-on-error>true</fail-partition-on-error>", + " <revert-time>34m</revert-time>", + " <recovery-time>5d</recovery-time>", + " </engine>", + " <group>", + " node distribution-key=\"0\" hostalias=\"mockhost\"/>", + " </group>", + "</cluster>"); ContentCluster cluster = ContentClusterUtils.createCluster(xml, new MockRoot()); @@ -398,21 +374,21 @@ public class StorageClusterTest { @Test public void requireThatUserDoesNotSpecifyBothGroupAndNodes() { - String xml = - "<cluster id=\"storage\">\n" + - "<documents/>\n" + - "<engine>\n" + - " <fail-partition-on-error>true</fail-partition-on-error>\n" + - " <revert-time>34m</revert-time>\n" + - " <recovery-time>5d</recovery-time>\n" + - "</engine>" + - " <group>\n" + - " <node distribution-key=\"0\" hostalias=\"mockhost\"/>\n" + - " </group>\n" + - " <nodes>\n" + - " <node distribution-key=\"1\" hostalias=\"mockhost\"/>\n" + - " </nodes>\n" + - "</cluster>"; + String xml = joinLines( + "<cluster id=\"storage\">", + " <documents/>", + " <engine>", + " <fail-partition-on-error>true</fail-partition-on-error>", + " <revert-time>34m</revert-time>", + " <recovery-time>5d</recovery-time>", + " </engine>", + " <group>", + " <node distribution-key=\"0\" hostalias=\"mockhost\"/>", + " </group>", + " <nodes>", + " <node distribution-key=\"1\" hostalias=\"mockhost\"/>", + " </nodes>", + "</cluster>"); try { final MockRoot root = new MockRoot(); @@ -429,20 +405,20 @@ public class StorageClusterTest { @Test public void requireThatGroupNamesMustBeUniqueAmongstSiblings() { - String xml = - "<cluster id=\"storage\">\n" + - " <redundancy>2</redundancy>" + - " <documents/>\n" + - " <group>\n" + - " <distribution partitions=\"*\"/>\n" + - " <group distribution-key=\"0\" name=\"bar\">\n" + - " <node distribution-key=\"0\" hostalias=\"mockhost\"/>\n" + - " </group>\n" + - " <group distribution-key=\"0\" name=\"bar\">\n" + - " <node distribution-key=\"1\" hostalias=\"mockhost\"/>\n" + - " </group>\n" + - " </group>\n" + - "</cluster>"; + String xml = joinLines( + "<cluster id=\"storage\">", + " <redundancy>2</redundancy>", + " <documents/>", + " <group>", + " <distribution partitions=\"*\"/>", + " <group distribution-key=\"0\" name=\"bar\">", + " <node distribution-key=\"0\" hostalias=\"mockhost\"/>", + " </group>", + " <group distribution-key=\"0\" name=\"bar\">", + " <node distribution-key=\"1\" hostalias=\"mockhost\"/>", + " </group>", + " </group>", + "</cluster>"); try { ContentClusterUtils.createCluster(xml, new MockRoot()); @@ -455,24 +431,24 @@ public class StorageClusterTest { @Test public void requireThatGroupNamesCanBeDuplicatedAcrossLevels() { - String xml = - "<cluster id=\"storage\">\n" + - " <redundancy>2</redundancy>" + - "<documents/>\n" + - " <group>\n" + - " <distribution partitions=\"*\"/>\n" + - " <group distribution-key=\"0\" name=\"bar\">\n" + - " <group distribution-key=\"0\" name=\"foo\">\n" + - " <node distribution-key=\"0\" hostalias=\"mockhost\"/>\n" + - " </group>\n" + - " </group>\n" + - " <group distribution-key=\"0\" name=\"foo\">\n" + - " <group distribution-key=\"0\" name=\"bar\">\n" + - " <node distribution-key=\"1\" hostalias=\"mockhost\"/>\n" + - " </group>\n" + - " </group>\n" + - " </group>\n" + - "</cluster>"; + String xml = joinLines( + "<cluster id=\"storage\">", + " <redundancy>2</redundancy>", + " <documents/>", + " <group>", + " <distribution partitions=\"*\"/>", + " <group distribution-key=\"0\" name=\"bar\">", + " <group distribution-key=\"0\" name=\"foo\">", + " <node distribution-key=\"0\" hostalias=\"mockhost\"/>", + " </group>", + " </group>", + " <group distribution-key=\"0\" name=\"foo\">", + " <group distribution-key=\"0\" name=\"bar\">", + " <node distribution-key=\"1\" hostalias=\"mockhost\"/>", + " </group>", + " </group>", + " </group>", + "</cluster>"); // Should not throw. ContentClusterUtils.createCluster(xml, new MockRoot()); @@ -480,18 +456,18 @@ public class StorageClusterTest { @Test public void requireThatNestedGroupsRequireDistribution() { - String xml = - "<cluster id=\"storage\">\n" + - "<documents/>\n" + - " <group>\n" + - " <group distribution-key=\"0\" name=\"bar\">\n" + - " <node distribution-key=\"0\" hostalias=\"mockhost\"/>\n" + - " </group>\n" + - " <group distribution-key=\"0\" name=\"baz\">\n" + - " <node distribution-key=\"1\" hostalias=\"mockhost\"/>\n" + - " </group>\n" + - " </group>\n" + - "</cluster>"; + String xml = joinLines( + "<cluster id=\"storage\">", + " <documents/>", + " <group>", + " <group distribution-key=\"0\" name=\"bar\">", + " <node distribution-key=\"0\" hostalias=\"mockhost\"/>", + " </group>", + " <group distribution-key=\"0\" name=\"baz\">", + " <node distribution-key=\"1\" hostalias=\"mockhost\"/>", + " </group>", + " </group>", + "</cluster>"); try { ContentClusterUtils.createCluster(xml, new MockRoot()); diff --git a/config-model/src/test/java/com/yahoo/vespa/model/content/cluster/ClusterTest.java b/config-model/src/test/java/com/yahoo/vespa/model/content/cluster/ClusterTest.java index 7c93b4ef02b..afeffbbc875 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/content/cluster/ClusterTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/content/cluster/ClusterTest.java @@ -15,9 +15,7 @@ import java.util.List; import static com.yahoo.config.model.test.TestUtil.joinLines; import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertNotNull; -import static org.junit.Assert.assertTrue; /** * @author Simon Thoresen Hult @@ -179,7 +177,7 @@ public class ClusterTest { " </tuning>", " </content>", "</services>")) - .withSchemas(ApplicationPackageUtils.generateSearchDefinition("my_document")) + .withSchemas(ApplicationPackageUtils.generateSchemas("my_document")) .build(); List<Content> contents = new TestDriver().buildModel(app).getConfigModels(Content.class); assertEquals(1, contents.size()); diff --git a/config-model/src/test/java/com/yahoo/vespa/model/ml/MlModelsTest.java b/config-model/src/test/java/com/yahoo/vespa/model/ml/MlModelsTest.java index 854c4d32a64..9e87b5509bc 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/ml/MlModelsTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/ml/MlModelsTest.java @@ -32,8 +32,8 @@ public class MlModelsTest { IOUtils.copy(appDir.append("services.xml").toString(), storedAppDir.append("services.xml").toString()); IOUtils.copyDirectory(appDir.append(ApplicationPackage.MODELS_GENERATED_DIR).toFile(), storedAppDir.append(ApplicationPackage.MODELS_GENERATED_DIR).toFile()); - IOUtils.copyDirectory(appDir.append(ApplicationPackage.SEARCH_DEFINITIONS_DIR).toFile(), - storedAppDir.append(ApplicationPackage.SEARCH_DEFINITIONS_DIR).toFile()); + IOUtils.copyDirectory(appDir.append(ApplicationPackage.SCHEMAS_DIR).toFile(), + storedAppDir.append(ApplicationPackage.SCHEMAS_DIR).toFile()); ImportedModelTester storedTester = new ImportedModelTester("ml_models", storedAppDir); verify(storedTester.createVespaModel()); } @@ -61,7 +61,7 @@ public class MlModelsTest { assertEquals(testProfile, b.toString()); } - private final String testProfile = + private static final String testProfile = "rankingExpression(Placeholder).rankingScript: attribute(argument)\n" + "rankingExpression(Placeholder).type: tensor<float>(d0[1],d1[784])\n" + "rankingExpression(mnist_softmax_onnx).rankingScript: join(reduce(join(rename(rankingExpression(Placeholder), (d0, d1), (d0, d2)), constant(mnist_softmax_Variable), f(a,b)(a * b)), sum, d2), constant(mnist_softmax_Variable_1), f(a,b)(a + b))\n" + diff --git a/config-model/src/test/java/com/yahoo/vespa/model/ml/StatelessOnnxEvaluationTest.java b/config-model/src/test/java/com/yahoo/vespa/model/ml/StatelessOnnxEvaluationTest.java index 5dea4a04229..5c64dd9f8cd 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/ml/StatelessOnnxEvaluationTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/ml/StatelessOnnxEvaluationTest.java @@ -4,17 +4,13 @@ package com.yahoo.vespa.model.ml; import ai.vespa.models.evaluation.FunctionEvaluator; import ai.vespa.models.evaluation.Model; import ai.vespa.models.evaluation.ModelsEvaluator; -import ai.vespa.models.evaluation.RankProfilesConfigImporter; -import ai.vespa.models.handler.ModelsEvaluationHandler; import com.yahoo.component.ComponentId; -import com.yahoo.config.FileReference; import com.yahoo.config.application.api.ApplicationPackage; import com.yahoo.filedistribution.fileacquirer.FileAcquirer; import com.yahoo.filedistribution.fileacquirer.MockFileAcquirer; import com.yahoo.io.IOUtils; import com.yahoo.path.Path; import com.yahoo.tensor.Tensor; -import com.yahoo.tensor.TensorType; import com.yahoo.vespa.config.search.RankProfilesConfig; import com.yahoo.vespa.config.search.core.OnnxModelsConfig; import com.yahoo.vespa.config.search.core.RankingConstantsConfig; @@ -31,7 +27,6 @@ import java.util.stream.Collectors; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNotNull; -import static org.junit.Assert.assertNull; import static org.junit.Assert.assertTrue; /** @@ -55,8 +50,8 @@ public class StatelessOnnxEvaluationTest { IOUtils.copy(appDir.append("services.xml").toString(), storedAppDir.append("services.xml").toString()); IOUtils.copyDirectory(appDir.append(ApplicationPackage.MODELS_GENERATED_DIR).toFile(), storedAppDir.append(ApplicationPackage.MODELS_GENERATED_DIR).toFile()); - IOUtils.copyDirectory(appDir.append(ApplicationPackage.SEARCH_DEFINITIONS_DIR).toFile(), - storedAppDir.append(ApplicationPackage.SEARCH_DEFINITIONS_DIR).toFile()); + IOUtils.copyDirectory(appDir.append(ApplicationPackage.SCHEMAS_DIR).toFile(), + storedAppDir.append(ApplicationPackage.SCHEMAS_DIR).toFile()); ImportedModelTester storedTester = new ImportedModelTester("onnx_rt", storedAppDir); assertModelEvaluation(storedTester.createVespaModel(), appDir); diff --git a/config-model/src/test/java/com/yahoo/vespa/model/test/VespaModelTestCase.java b/config-model/src/test/java/com/yahoo/vespa/model/test/VespaModelTestCase.java index f5250343afe..2b36bfc47b2 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/test/VespaModelTestCase.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/test/VespaModelTestCase.java @@ -192,7 +192,7 @@ public class VespaModelTestCase { " </documents>" + "</content>" + "</services>", - ApplicationPackageUtils.generateSearchDefinition("music")) + ApplicationPackageUtils.generateSchemas("music")) .create(); MessagebusConfig.Builder mBusB = new MessagebusConfig.Builder(); model.getConfig(mBusB, "client"); @@ -216,7 +216,7 @@ public class VespaModelTestCase { "</hosts>"); } - class MyLogger implements DeployLogger { + static class MyLogger implements DeployLogger { List<Pair<Level, String>> msgs = new ArrayList<>(); @Override public void log(Level level, String message) { @@ -305,4 +305,29 @@ public class VespaModelTestCase { assertThat(model.getContainerClusters().size(), is(1)); } + @Test + public void testThatDeployLogContainsWarningWhenUsingSearchdefinitionsDir() throws IOException, SAXException { + ApplicationPackage app = FilesApplicationPackage.fromFile( + new File("src/test/cfg/application/deprecated_features_app/")); + MyLogger logger = new MyLogger(); + DeployState deployState = new DeployState.Builder() + .applicationPackage(app) + .deployLogger(logger) + .build(); + VespaModel model = new VespaModel(new NullConfigModelRegistry(), deployState); + Validation.validate(model, new ValidationParameters(), deployState); + assertContainsWarning(logger.msgs, "Directory searchdefinitions/ should not be used for schemas, use schemas/ instead"); + } + + private void assertContainsWarning(List<Pair<Level,String>> msgs, String text) { + boolean foundCorrectWarning = false; + for (var msg : msgs) + if (msg.getFirst().getName().equals("WARNING") && msg.getSecond().equals(text)) { + foundCorrectWarning = true; + } + if (! foundCorrectWarning) for (var msg : msgs) System.err.println("MSG: "+msg); + assertTrue(msgs.size() > 0); + assertTrue(foundCorrectWarning); + } + } diff --git a/config-model/src/test/java/com/yahoo/vespa/model/test/VespaModelTester.java b/config-model/src/test/java/com/yahoo/vespa/model/test/VespaModelTester.java index ba975e52d1a..7e34e9efbbf 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/test/VespaModelTester.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/test/VespaModelTester.java @@ -21,7 +21,6 @@ import com.yahoo.config.provision.NodeResources; import com.yahoo.config.provision.ProvisionLogger; import com.yahoo.config.provision.Zone; import com.yahoo.vespa.model.VespaModel; -import com.yahoo.vespa.model.test.utils.ApplicationPackageUtils; import com.yahoo.vespa.model.test.utils.VespaModelCreatorWithMockPkg; import java.util.ArrayList; @@ -31,6 +30,8 @@ import java.util.List; import java.util.Map; import java.util.Optional; +import static com.yahoo.vespa.model.test.utils.ApplicationPackageUtils.generateSchemas; + /** * Helper class which sets up a system with multiple hosts. * Usage: @@ -168,7 +169,7 @@ public class VespaModelTester { boolean alwaysReturnOneNode, int startIndexForClusters, Optional<VespaModel> previousModel, DeployState.Builder deployStatebuilder, String ... retiredHostNames) { - VespaModelCreatorWithMockPkg modelCreatorWithMockPkg = new VespaModelCreatorWithMockPkg(null, services, ApplicationPackageUtils.generateSearchDefinition("type1")); + VespaModelCreatorWithMockPkg modelCreatorWithMockPkg = new VespaModelCreatorWithMockPkg(null, services, generateSchemas("type1")); ApplicationPackage appPkg = modelCreatorWithMockPkg.appPkg; provisioner = hosted ? new ProvisionerAdapter(new InMemoryProvisioner(hostsByResources, diff --git a/config-model/src/test/java/com/yahoo/vespa/model/test/utils/ApplicationPackageUtils.java b/config-model/src/test/java/com/yahoo/vespa/model/test/utils/ApplicationPackageUtils.java index df62a3bff07..1f7deaf1991 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/test/utils/ApplicationPackageUtils.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/test/utils/ApplicationPackageUtils.java @@ -1,4 +1,4 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.model.test.utils; import java.util.ArrayList; @@ -43,10 +43,6 @@ public class ApplicationPackageUtils { "}"; } - public static List<String> generateSearchDefinition(String name) { - return generateSchemas(name); - } - public static List<String> generateSchemas(String ... sdNames) { return generateSchemas(Arrays.asList(sdNames)); } diff --git a/config-model/src/test/java/com/yahoo/vespa/model/test/utils/VespaModelCreatorWithFilePkg.java b/config-model/src/test/java/com/yahoo/vespa/model/test/utils/VespaModelCreatorWithFilePkg.java index 8147d2e00ca..eea3da0e096 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/test/utils/VespaModelCreatorWithFilePkg.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/test/utils/VespaModelCreatorWithFilePkg.java @@ -24,9 +24,8 @@ import java.io.IOException; */ public class VespaModelCreatorWithFilePkg { - private FilesApplicationPackage applicationPkg; - - private ConfigModelRegistry configModelRegistry; + private final FilesApplicationPackage applicationPkg; + private final ConfigModelRegistry configModelRegistry; public VespaModelCreatorWithFilePkg(String directoryName) { this(new File(directoryName)); diff --git a/config-model/src/test/schema-test-files/services.xml b/config-model/src/test/schema-test-files/services.xml index db1e6c29586..4ad7865c93b 100644 --- a/config-model/src/test/schema-test-files/services.xml +++ b/config-model/src/test/schema-test-files/services.xml @@ -147,7 +147,7 @@ </http> <accesslog type='json' - fileNamePattern='logs/vespa/qrs/access-json.%Y%m%d%H%M%S' compressOnRotation='true' compressionType='zstd'/> + fileNamePattern='logs/vespa/qrs/access-json.%Y%m%d%H%M%S' compressOnRotation='true' compressionType='zstd' queueSize='13' bufferSize='65536'/> <accesslog type='vespa' fileNamePattern='logs/vespa/qrs/access-vespa.%Y%m%d%H%M%S' /> diff --git a/config-proxy/src/main/sh/vespa-config-ctl.sh b/config-proxy/src/main/sh/vespa-config-ctl.sh index b4127cd337e..03c033e620b 100755 --- a/config-proxy/src/main/sh/vespa-config-ctl.sh +++ b/config-proxy/src/main/sh/vespa-config-ctl.sh @@ -92,7 +92,6 @@ cp="libexec/vespa/patches/configproxy:lib/jars/config-proxy-jar-with-dependencie VESPA_LOG_LEVEL="all -debug -spam" export VESPA_LOG_TARGET VESPA_LOG_LEVEL VESPA_LOG_CONTROL_DIR -export VESPA_SENTINEL_PORT mkdir -p "$LOGDIR" mkdir -p "$VESPA_LOG_CONTROL_DIR" diff --git a/config/src/tests/frtconnectionpool/frtconnectionpool.cpp b/config/src/tests/frtconnectionpool/frtconnectionpool.cpp index 79ec1ee42f4..6320feb1e12 100644 --- a/config/src/tests/frtconnectionpool/frtconnectionpool.cpp +++ b/config/src/tests/frtconnectionpool/frtconnectionpool.cpp @@ -169,14 +169,12 @@ void Test::testSetErrorAllHashBased() { const ServerSpec spec(_sources); FRTConnectionPool sourcePool(spec, timingValues); FRTConnection* firstSource = sourcePool.getNextHashBased(); - std::vector<FRTConnection*> readySources; - sourcePool.getReadySources(readySources); + auto readySources = sourcePool.getReadySources(); for (int i = 0; i < (int)readySources.size(); i++) { readySources[i]->setError(FRTE_RPC_CONNECTION); } - std::vector<FRTConnection*> tmpSources; - EXPECT_EQUAL((int)sourcePool.getReadySources(tmpSources).size(), 0); - EXPECT_EQUAL((int)sourcePool.getSuspendedSources(tmpSources).size(), 3); + EXPECT_EQUAL(sourcePool.getReadySources().size(), 0u); + EXPECT_EQUAL(sourcePool.getSuspendedSources().size(), 3u); // should get the same source now, since all are suspended EXPECT_EQUAL(firstSource->getAddress(), sourcePool.getNextHashBased()->getAddress()); @@ -188,8 +186,8 @@ void Test::testSetErrorAllHashBased() { } } - EXPECT_EQUAL((int)sourcePool.getReadySources(tmpSources).size(), 2); - EXPECT_EQUAL((int)sourcePool.getSuspendedSources(tmpSources).size(), 1); + EXPECT_EQUAL(sourcePool.getReadySources().size(), 2u); + EXPECT_EQUAL(sourcePool.getSuspendedSources().size(), 1u); // should not get the same source now, since original source is // suspended, while the rest are OK diff --git a/config/src/vespa/config/frt/frtconnectionpool.cpp b/config/src/vespa/config/frt/frtconnectionpool.cpp index 1aff50f9041..00ed9334537 100644 --- a/config/src/vespa/config/frt/frtconnectionpool.cpp +++ b/config/src/vespa/config/frt/frtconnectionpool.cpp @@ -66,20 +66,18 @@ FRTConnectionPool::getCurrent() FRTConnection * FRTConnectionPool::getNextRoundRobin() { - std::vector<FRTConnection *> readySources; - getReadySources(readySources); - std::vector<FRTConnection *> suspendedSources; - getSuspendedSources(suspendedSources); + auto ready = getReadySources(); + auto suspended = getSuspendedSources(); FRTConnection* nextFRTConnection = nullptr; - if (!readySources.empty()) { - int sel = _selectIdx % (int)readySources.size(); + if ( ! ready.empty()) { + int sel = _selectIdx % (int)ready.size(); _selectIdx = sel + 1; - nextFRTConnection = readySources[sel]; - } else if (!suspendedSources.empty()) { - int sel = _selectIdx % (int)suspendedSources.size(); + nextFRTConnection = ready[sel]; + } else if ( ! suspended.empty()) { + int sel = _selectIdx % (int)suspended.size(); _selectIdx = sel + 1; - nextFRTConnection = suspendedSources[sel]; + nextFRTConnection = suspended[sel]; } return nextFRTConnection; } @@ -87,28 +85,26 @@ FRTConnectionPool::getNextRoundRobin() FRTConnection * FRTConnectionPool::getNextHashBased() { - std::vector<FRTConnection*> readySources; - getReadySources(readySources); - std::vector<FRTConnection*> suspendedSources; - getSuspendedSources(suspendedSources); + auto ready = getReadySources(); + auto suspended = getSuspendedSources(); FRTConnection* nextFRTConnection = nullptr; - if (!readySources.empty()) { - int sel = std::abs(hashCode(_hostname) % (int)readySources.size()); - nextFRTConnection = readySources[sel]; - } else { - int sel = std::abs(hashCode(_hostname) % (int)suspendedSources.size()); - nextFRTConnection = suspendedSources[sel]; + if ( ! ready.empty()) { + int sel = std::abs(hashCode(_hostname) % (int)ready.size()); + nextFRTConnection = ready[sel]; + } else if ( ! suspended.empty() ){ + int sel = std::abs(hashCode(_hostname) % (int)suspended.size()); + nextFRTConnection = suspended[sel]; } return nextFRTConnection; } -const std::vector<FRTConnection *> & -FRTConnectionPool::getReadySources(std::vector<FRTConnection*> & readySources) const +std::vector<FRTConnection *> +FRTConnectionPool::getReadySources() const { - readySources.clear(); + std::vector<FRTConnection*> readySources; for (const auto & entry : _connections) { FRTConnection* source = entry.second.get(); int64_t tnow = FRTConnection::milliSecsSinceEpoch(); @@ -120,10 +116,10 @@ FRTConnectionPool::getReadySources(std::vector<FRTConnection*> & readySources) c return readySources; } -const std::vector<FRTConnection *> & -FRTConnectionPool::getSuspendedSources(std::vector<FRTConnection*> & suspendedSources) const +std::vector<FRTConnection *> +FRTConnectionPool::getSuspendedSources() const { - suspendedSources.clear(); + std::vector<FRTConnection*> suspendedSources; for (const auto & entry : _connections) { FRTConnection* source = entry.second.get(); int64_t tnow = FRTConnection::milliSecsSinceEpoch(); diff --git a/config/src/vespa/config/frt/frtconnectionpool.h b/config/src/vespa/config/frt/frtconnectionpool.h index 0b30e723272..dc758249771 100644 --- a/config/src/vespa/config/frt/frtconnectionpool.h +++ b/config/src/vespa/config/frt/frtconnectionpool.h @@ -109,14 +109,14 @@ public: * * @return list of FRTConnection pointers */ - const std::vector<FRTConnection*> & getReadySources(std::vector<FRTConnection*> & readySources) const; + std::vector<FRTConnection*> getReadySources() const; /** * Gets list of sources that are suspended. * * @param suspendedSources is list of FRTConnection pointers */ - const std::vector<FRTConnection*> & getSuspendedSources(std::vector<FRTConnection*> & suspendedSources) const; + std::vector<FRTConnection*> getSuspendedSources() const; /** * Implementation of the Java hashCode function for the String class. diff --git a/config/src/vespa/config/frt/frtsource.cpp b/config/src/vespa/config/frt/frtsource.cpp index 94884e19b3f..15aa52c6463 100644 --- a/config/src/vespa/config/frt/frtsource.cpp +++ b/config/src/vespa/config/frt/frtsource.cpp @@ -51,6 +51,10 @@ FRTSource::getConfig() int64_t serverTimeout = _agent->getTimeout(); double clientTimeout = (serverTimeout / 1000.0) + 5.0; // The additional 5 seconds is the time allowed for the server to respond after serverTimeout has elapsed. Connection * connection = _connectionFactory->getCurrent(); + if (connection == nullptr) { + LOG(warning, "No connection available - bad config ?"); + return; + } const ConfigState & state(_agent->getConfigState()); // LOG(debug, "invoking request with md5 %s, gen %" PRId64 ", servertimeout(%" PRId64 "), client(%f)", state.md5.c_str(), state.generation, serverTimeout, clientTimeout); diff --git a/configd/src/apps/sentinel/CMakeLists.txt b/configd/src/apps/sentinel/CMakeLists.txt index b4fca4a3285..43b4f79a0b2 100644 --- a/configd/src/apps/sentinel/CMakeLists.txt +++ b/configd/src/apps/sentinel/CMakeLists.txt @@ -1,16 +1,23 @@ # Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. vespa_add_executable(configd_config-sentinel_app SOURCES - sentinel.cpp - service.cpp - config-handler.cpp + check-completion-handler.cpp + cmdq.cpp + config-owner.cpp + connectivity.cpp + env.cpp line-splitter.cpp - output-connection.cpp + manager.cpp metrics.cpp - state-api.cpp - cmdq.cpp + output-connection.cpp + outward-check.cpp + peer-check.cpp rpchooks.cpp rpcserver.cpp + sentinel.cpp + service.cpp + state-api.cpp + status-callback.cpp OUTPUT_NAME vespa-config-sentinel INSTALL sbin DEPENDS diff --git a/configd/src/apps/sentinel/check-completion-handler.cpp b/configd/src/apps/sentinel/check-completion-handler.cpp new file mode 100644 index 00000000000..3ccda17e617 --- /dev/null +++ b/configd/src/apps/sentinel/check-completion-handler.cpp @@ -0,0 +1,20 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include "check-completion-handler.h" + +namespace config::sentinel { + +CheckCompletionHandler::CheckCompletionHandler(FRT_RPCRequest *parent) + : _parentRequest(parent) +{ +} + +CheckCompletionHandler::~CheckCompletionHandler() = default; + +void CheckCompletionHandler::returnStatus(bool ok) { + FRT_Values *dst = _parentRequest->GetReturn(); + dst->AddString(ok ? "ok" : "bad"); + _parentRequest->Return(); +} + +} diff --git a/configd/src/apps/sentinel/check-completion-handler.h b/configd/src/apps/sentinel/check-completion-handler.h new file mode 100644 index 00000000000..008e456f6d7 --- /dev/null +++ b/configd/src/apps/sentinel/check-completion-handler.h @@ -0,0 +1,24 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#pragma once + +#include "status-callback.h" +#include "peer-check.h" + +namespace config::sentinel { + +/** + * Handles a checkConnectivity request by making an outgoing + * ping request. When the ping finishes, fills an answer + * into the parent request and send the answer back. + **/ +class CheckCompletionHandler : public StatusCallback { +private: + FRT_RPCRequest *_parentRequest; +public: + CheckCompletionHandler(FRT_RPCRequest *parentRequest); + virtual ~CheckCompletionHandler(); + void returnStatus(bool ok) override; +}; + +} diff --git a/configd/src/apps/sentinel/config-owner.cpp b/configd/src/apps/sentinel/config-owner.cpp new file mode 100644 index 00000000000..840c5b1add1 --- /dev/null +++ b/configd/src/apps/sentinel/config-owner.cpp @@ -0,0 +1,70 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include "config-owner.h" +#include <vespa/vespalib/util/exceptions.h> +#include <vespa/config/common/exceptions.h> +#include <string> + +#include <vespa/log/log.h> +LOG_SETUP(".config-owner"); + +namespace config::sentinel { + +ConfigOwner::ConfigOwner() : _subscriber() {} + +ConfigOwner::~ConfigOwner() = default; + +void +ConfigOwner::subscribe(const std::string & configId, std::chrono::milliseconds timeout) { + _sentinelHandle = _subscriber.subscribe<SentinelConfig>(configId, timeout); +} + +void +ConfigOwner::doConfigure() +{ + _currConfig = _sentinelHandle->getConfig(); + LOG_ASSERT(_currConfig); + _currGeneration = _subscriber.getGeneration(); + const SentinelConfig& config(*_currConfig); + const auto & app = config.application; + LOG(config, "Sentinel got %zd service elements [tenant(%s), application(%s), instance(%s)] for config generation %" PRId64, + config.service.size(), app.tenant.c_str(), app.name.c_str(), app.instance.c_str(), _currGeneration); +} + + +// Return true if there was a config generation change +bool +ConfigOwner::checkForConfigUpdate() { + if (_subscriber.nextGenerationNow()) { + doConfigure(); + return true; + } + return false; +} + +std::unique_ptr<ModelConfig> +ConfigOwner::fetchModelConfig(std::chrono::milliseconds timeout) +{ + std::unique_ptr<ModelConfig> modelConfig; + ConfigSubscriber tempSubscriber; + try { + ConfigHandle<ModelConfig>::UP modelHandle = + tempSubscriber.subscribe<ModelConfig>("admin/model", timeout); + if (tempSubscriber.nextGenerationNow()) { + modelConfig = modelHandle->getConfig(); + LOG(config, "Sentinel got model info [version %s] for %zd hosts [config generation %" PRId64 "]", + modelConfig->vespaVersion.c_str(), modelConfig->hosts.size(), + tempSubscriber.getGeneration()); + } + } catch (ConfigTimeoutException & ex) { + LOG(warning, "Timeout getting model config: %s [skipping connectivity checks]", ex.getMessage().c_str()); + } catch (InvalidConfigException& ex) { + LOG(warning, "Invalid model config: %s [skipping connectivity checks]", ex.getMessage().c_str()); + } catch (ConfigRuntimeException& ex) { + LOG(warning, "Runtime exception getting model config: %s [skipping connectivity checks]", ex.getMessage().c_str()); + + } + return modelConfig; +} + +} diff --git a/configd/src/apps/sentinel/config-owner.h b/configd/src/apps/sentinel/config-owner.h new file mode 100644 index 00000000000..2850e6b3904 --- /dev/null +++ b/configd/src/apps/sentinel/config-owner.h @@ -0,0 +1,43 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#pragma once + +#include <vespa/config-sentinel.h> +#include <vespa/config-model.h> +#include <vespa/config/config.h> + +using cloud::config::SentinelConfig; +using cloud::config::ModelConfig; + +using config::ConfigSubscriber; +using config::ConfigHandle; + +namespace config::sentinel { + +/** + * Handles config subscription and has a snapshot of current config. + **/ +class ConfigOwner { +private: + ConfigSubscriber _subscriber; + ConfigHandle<SentinelConfig>::UP _sentinelHandle; + + int64_t _currGeneration = -1; + std::unique_ptr<SentinelConfig> _currConfig; + + ConfigOwner(const ConfigOwner&) = delete; + ConfigOwner& operator =(const ConfigOwner&) = delete; + + void doConfigure(); +public: + ConfigOwner(); + virtual ~ConfigOwner(); + void subscribe(const std::string & configId, std::chrono::milliseconds timeout); + bool checkForConfigUpdate(); + bool hasConfig() const { return _currConfig.get() != nullptr; } + const SentinelConfig& getConfig() const { return *_currConfig; } + int64_t getGeneration() const { return _currGeneration; } + static std::unique_ptr<ModelConfig> fetchModelConfig(std::chrono::milliseconds timeout); +}; + +} diff --git a/configd/src/apps/sentinel/connectivity.cpp b/configd/src/apps/sentinel/connectivity.cpp new file mode 100644 index 00000000000..9cced1d3475 --- /dev/null +++ b/configd/src/apps/sentinel/connectivity.cpp @@ -0,0 +1,112 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include "connectivity.h" +#include "outward-check.h" +#include <vespa/defaults.h> +#include <vespa/log/log.h> +#include <vespa/vespalib/util/exceptions.h> +#include <vespa/vespalib/util/stringfmt.h> +#include <thread> +#include <chrono> + +LOG_SETUP(".connectivity"); + +using vespalib::make_string_short::fmt; +using namespace std::chrono_literals; + +namespace config::sentinel { + +Connectivity::Connectivity(const SentinelConfig::Connectivity & config, RpcServer &rpcServer) + : _config(config), + _rpcServer(rpcServer) +{ + LOG(config, "connectivity.maxBadReverseCount = %d", _config.maxBadReverseCount); + LOG(config, "connectivity.maxBadOutPercent = %d", _config.maxBadOutPercent); +} + +Connectivity::~Connectivity() = default; + +namespace { + +const char *toString(CcResult value) { + switch (value) { + case CcResult::UNKNOWN: return "BAD: missing result"; // very very bad + case CcResult::REVERSE_FAIL: return "connect OK, but reverse check FAILED"; // very bad + case CcResult::CONN_FAIL: return "failed to connect"; // bad + case CcResult::REVERSE_UNAVAIL: return "connect OK (but reverse check unavailable)"; // unfortunate + case CcResult::ALL_OK: return "OK: both ways connectivity verified"; // good + } + LOG(error, "Unknown CcResult enum value: %d", (int)value); + LOG_ABORT("Unknown CcResult enum value"); +} + +std::map<std::string, std::string> specsFrom(const ModelConfig &model) { + std::map<std::string, std::string> checkSpecs; + for (const auto & h : model.hosts) { + bool foundSentinelPort = false; + for (const auto & s : h.services) { + if (s.name == "config-sentinel") { + for (const auto & p : s.ports) { + if (p.tags.find("rpc") != p.tags.npos) { + auto spec = fmt("tcp/%s:%d", h.name.c_str(), p.number); + checkSpecs[h.name] = spec; + foundSentinelPort = true; + } + } + } + } + if (! foundSentinelPort) { + LOG(warning, "Did not find 'config-sentinel' RPC port in model for host %s [%zd services]", + h.name.c_str(), h.services.size()); + } + } + return checkSpecs; +} + +} + +Connectivity::CheckResult +Connectivity::checkConnectivity(const ModelConfig &model) { + const auto checkSpecs = specsFrom(model); + size_t clusterSize = checkSpecs.size(); + OutwardCheckContext checkContext(clusterSize, + vespa::Defaults::vespaHostname(), + _rpcServer.getPort(), + _rpcServer.orb()); + std::map<std::string, OutwardCheck> connectivityMap; + for (const auto & [ hn, spec ] : checkSpecs) { + connectivityMap.try_emplace(hn, spec, checkContext); + } + checkContext.latch.await(); + size_t numFailedConns = 0; + size_t numFailedReverse = 0; + bool allChecksOk = true; + for (const auto & [hostname, check] : connectivityMap) { + LOG_ASSERT(check.result() != CcResult::UNKNOWN); + if (check.result() == CcResult::CONN_FAIL) ++numFailedConns; + if (check.result() == CcResult::REVERSE_FAIL) ++numFailedReverse; + } + if (numFailedReverse > size_t(_config.maxBadReverseCount)) { + LOG(warning, "%zu of %zu nodes report problems connecting to me (max is %d)", + numFailedReverse, clusterSize, _config.maxBadReverseCount); + allChecksOk = false; + } + if (numFailedConns * 100.0 > _config.maxBadOutPercent * clusterSize) { + double pct = numFailedConns * 100.0 / clusterSize; + LOG(warning, "Problems connecting to %zu of %zu nodes, %.2f %% (max is %d)", + numFailedConns, clusterSize, pct, _config.maxBadOutPercent); + allChecksOk = false; + } + std::vector<std::string> details; + for (const auto & [hostname, check] : connectivityMap) { + std::string detail = fmt("%s -> %s", hostname.c_str(), toString(check.result())); + details.push_back(detail); + } + CheckResult result{false, false, {}}; + result.enoughOk = allChecksOk; + result.allOk = (numFailedConns == 0) && (numFailedReverse == 0); + result.details = std::move(details); + return result; +} + +} diff --git a/configd/src/apps/sentinel/connectivity.h b/configd/src/apps/sentinel/connectivity.h new file mode 100644 index 00000000000..0e32b5243e0 --- /dev/null +++ b/configd/src/apps/sentinel/connectivity.h @@ -0,0 +1,36 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#pragma once + +#include "rpcserver.h" +#include <vespa/config-sentinel.h> +#include <vespa/config-model.h> +#include <string> +#include <vector> + +using cloud::config::SentinelConfig; +using cloud::config::ModelConfig; + +namespace config::sentinel { + +/** + * Utility class for running connectivity check. + **/ +class Connectivity { +public: + Connectivity(const SentinelConfig::Connectivity & config, RpcServer &rpcServer); + ~Connectivity(); + + struct CheckResult { + bool enoughOk; + bool allOk; + std::vector<std::string> details; + }; + + CheckResult checkConnectivity(const ModelConfig &model); +private: + const SentinelConfig::Connectivity _config; + RpcServer &_rpcServer; +}; + +} diff --git a/configd/src/apps/sentinel/env.cpp b/configd/src/apps/sentinel/env.cpp new file mode 100644 index 00000000000..e4174ee450d --- /dev/null +++ b/configd/src/apps/sentinel/env.cpp @@ -0,0 +1,152 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include "env.h" +#include "check-completion-handler.h" +#include "outward-check.h" +#include <vespa/defaults.h> +#include <vespa/log/log.h> +#include <vespa/config/common/exceptions.h> +#include <vespa/vespalib/util/exceptions.h> +#include <vespa/vespalib/util/stringfmt.h> +#include <thread> +#include <chrono> + +LOG_SETUP(".env"); + +using vespalib::make_string_short::fmt; +using namespace std::chrono_literals; + +namespace config::sentinel { + +constexpr std::chrono::milliseconds CONFIG_TIMEOUT_MS = 3min; +constexpr std::chrono::milliseconds MODEL_TIMEOUT_MS = 1500ms; + +Env::Env() + : _cfgOwner(), + _rpcCommandQueue(), + _rpcServer(), + _stateApi(), + _startMetrics(), + _stateServer(), + _statePort(0) +{ + _startMetrics.startedTime = vespalib::steady_clock::now(); +} + +Env::~Env() = default; + +void Env::boot(const std::string &configId) { + LOG(debug, "Reading configuration for ID: %s", configId.c_str()); + _cfgOwner.subscribe(configId, CONFIG_TIMEOUT_MS); + bool ok = _cfgOwner.checkForConfigUpdate(); + // subscribe() should throw if something is not OK + LOG_ASSERT(ok && _cfgOwner.hasConfig()); + const auto & cfg = _cfgOwner.getConfig(); + LOG(config, "Booting sentinel '%s' with [stateserver port %d] and [rpc port %d]", + configId.c_str(), cfg.port.telnet, cfg.port.rpc); + rpcPort(cfg.port.rpc); + statePort(cfg.port.telnet); + if (auto up = ConfigOwner::fetchModelConfig(MODEL_TIMEOUT_MS)) { + waitForConnectivity(*up); + } +} + +void Env::rpcPort(int port) { + if (port < 0 || port > 65535) { + throw vespalib::FatalException("Bad port " + std::to_string(port) + ", expected range [1, 65535]", VESPA_STRLOC); + } + if (port == 0) { + port = 19097; // default in config + } + if (_rpcServer && port == _rpcServer->getPort()) { + return; // ok already + } + _rpcServer = std::make_unique<RpcServer>(port, _rpcCommandQueue); +} + +void Env::statePort(int port) { + if (port < 0 || port > 65535) { + throw vespalib::FatalException("Bad port " + std::to_string(port) + ", expected range [1, 65535]", VESPA_STRLOC); + } + if (port == 0) { + port = 19098; // default in config + } + if (_stateServer && port == _statePort) { + return; // ok already + } + LOG(debug, "Config-sentinel accepts state connections on port %d", port); + _stateServer = std::make_unique<vespalib::StateServer>( + port, _stateApi.myHealth, _startMetrics.producer, _stateApi.myComponents); + _statePort = port; +} + +void Env::notifyConfigUpdated() { + vespalib::ComponentConfigProducer::Config current("sentinel", _cfgOwner.getGeneration(), "ok"); + _stateApi.myComponents.addConfig(current); + +} + +void Env::respondAsEmpty() { + auto commands = _rpcCommandQueue.drain(); + for (Cmd::UP &cmd : commands) { + cmd->retError("still booting, not ready for all RPC commands"); + } +} + +namespace { + +const char *toString(CcResult value) { + switch (value) { + case CcResult::UNKNOWN: return "unknown"; + case CcResult::CONN_FAIL: return "failed to connect"; + case CcResult::REVERSE_FAIL: return "connect OK, but reverse check FAILED"; + case CcResult::REVERSE_UNAVAIL: return "connect OK, but reverse check unavailable"; + case CcResult::ALL_OK: return "both ways connectivity OK"; + } + LOG(error, "Unknown CcResult enum value: %d", (int)value); + LOG_ABORT("Unknown CcResult enum value"); +} + +std::map<std::string, std::string> specsFrom(const ModelConfig &model) { + std::map<std::string, std::string> checkSpecs; + for (const auto & h : model.hosts) { + bool foundSentinelPort = false; + for (const auto & s : h.services) { + if (s.name == "config-sentinel") { + for (const auto & p : s.ports) { + if (p.tags.find("rpc") != p.tags.npos) { + auto spec = fmt("tcp/%s:%d", h.name.c_str(), p.number); + checkSpecs[h.name] = spec; + foundSentinelPort = true; + } + } + } + } + if (! foundSentinelPort) { + LOG(warning, "Did not find 'config-sentinel' RPC port in model for host %s [%zd services]", + h.name.c_str(), h.services.size()); + } + } + return checkSpecs; +} + +} + +void Env::waitForConnectivity(const ModelConfig &model) { + auto checkSpecs = specsFrom(model); + OutwardCheckContext checkContext(checkSpecs.size(), + vespa::Defaults::vespaHostname(), + _rpcServer->getPort(), + _rpcServer->orb()); + std::map<std::string, OutwardCheck> connectivityMap; + for (const auto & [ hn, spec ] : checkSpecs) { + connectivityMap.try_emplace(hn, spec, checkContext); + } + checkContext.latch.await(); + for (const auto & [hostname, check] : connectivityMap) { + LOG(info, "outward check status for host %s is: %s", + hostname.c_str(), toString(check.result())); + } +} + +} diff --git a/configd/src/apps/sentinel/env.h b/configd/src/apps/sentinel/env.h new file mode 100644 index 00000000000..f117854f006 --- /dev/null +++ b/configd/src/apps/sentinel/env.h @@ -0,0 +1,45 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#pragma once + +#include "cmdq.h" +#include "config-owner.h" +#include "metrics.h" +#include "rpcserver.h" +#include "state-api.h" +#include <vespa/vespalib/net/state_server.h> + +namespace config::sentinel { + +/** + * Environment for config sentinel, with config + * subscription, rpc server, state server, and + * metrics. + **/ +class Env { +public: + Env(); + ~Env(); + + ConfigOwner &configOwner() { return _cfgOwner; } + CommandQueue &commandQueue() { return _rpcCommandQueue; } + StartMetrics &metrics() { return _startMetrics; } + + void boot(const std::string &configId); + void rpcPort(int portnum); + void statePort(int portnum); + + void notifyConfigUpdated(); +private: + void respondAsEmpty(); + void waitForConnectivity(const ModelConfig &model); + ConfigOwner _cfgOwner; + CommandQueue _rpcCommandQueue; + std::unique_ptr<RpcServer> _rpcServer; + StateApi _stateApi; + StartMetrics _startMetrics; + std::unique_ptr<vespalib::StateServer> _stateServer; + int _statePort; +}; + +} diff --git a/configd/src/apps/sentinel/config-handler.cpp b/configd/src/apps/sentinel/manager.cpp index 9c39b13ba4c..6e0ed78211c 100644 --- a/configd/src/apps/sentinel/config-handler.cpp +++ b/configd/src/apps/sentinel/manager.cpp @@ -1,6 +1,6 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include "config-handler.h" +#include "manager.h" #include "output-connection.h" #include <vespa/vespalib/net/socket_address.h> @@ -11,43 +11,19 @@ #include <sys/wait.h> #include <vespa/log/log.h> -LOG_SETUP(".config-handler"); +LOG_SETUP(".manager"); namespace config::sentinel { -void -ConfigHandler::configure_port(int port) +Manager::Manager(Env &env) + : _env(env), + _services(), + _outputConnections() { - if (port == 0) { - port = 19098; - const char *portString = getenv("VESPA_SENTINEL_PORT"); - if (portString) { - port = strtoul(portString, nullptr, 10); - } - } - if (port <= 0 || port > 65535) { - throw vespalib::FatalException("Bad port " + std::to_string(port) + ", expected range [1, 65535]", VESPA_STRLOC); - } - if (port != _boundPort) { - LOG(debug, "Config-sentinel accepts connections on port %d", port); - _stateServer = std::make_unique<vespalib::StateServer>( - port, _stateApi.myHealth, _startMetrics.producer, _stateApi.myComponents); - _boundPort = port; - } + doConfigure(); } -ConfigHandler::ConfigHandler() - : _subscriber(), - _services(), - _outputConnections(), - _boundPort(0), - _startMetrics(), - _stateApi() -{ - _startMetrics.startedTime = vespalib::steady_clock::now(); -} - -ConfigHandler::~ConfigHandler() +Manager::~Manager() { terminateServices(false); for (OutputConnection * conn : _outputConnections) { @@ -56,7 +32,7 @@ ConfigHandler::~ConfigHandler() } void -ConfigHandler::terminateServices(bool catchable, bool printDebug) +Manager::terminateServices(bool catchable, bool printDebug) { for (const auto & entry : _services) { Service *service = entry.second.get(); @@ -74,7 +50,7 @@ ConfigHandler::terminateServices(bool catchable, bool printDebug) bool -ConfigHandler::terminate() +Manager::terminate() { // Call terminate(true) for all services. // Give them 58 seconds to exit cleanly, then terminate(false) all @@ -98,26 +74,15 @@ ConfigHandler::terminate() } void -ConfigHandler::subscribe(const std::string & configId, std::chrono::milliseconds timeout) +Manager::doConfigure() { - _sentinelHandle = _subscriber.subscribe<SentinelConfig>(configId, timeout); -} + LOG_ASSERT(_env.configOwner().hasConfig()); + const SentinelConfig& config(_env.configOwner().getConfig()); -void -ConfigHandler::doConfigure() -{ - std::unique_ptr<SentinelConfig> cfg(_sentinelHandle->getConfig()); - const SentinelConfig& config(*cfg); - - if (config.port.telnet != _boundPort) { - configure_port(config.port.telnet); - } + _env.rpcPort(config.port.rpc); + _env.statePort(config.port.telnet); - if (!_rpcServer || config.port.rpc != _rpcServer->getPort()) { - _rpcServer = std::make_unique<RpcServer>(config.port.rpc, _cmdQ); - } - - LOG(debug, "ConfigHandler::configure() %d config elements, tenant(%s), application(%s), instance(%s)", + LOG(debug, "Manager::configure() %d config elements, tenant(%s), application(%s), instance(%s)", (int)config.service.size(), config.application.tenant.c_str(), config.application.name.c_str(), config.application.instance.c_str()); ServiceMap services; @@ -126,7 +91,7 @@ ConfigHandler::doConfigure() const vespalib::string name(serviceConfig.name); auto found(_services.find(name)); if (found == _services.end()) { - services[name] = std::make_unique<Service>(serviceConfig, config.application, _outputConnections, _startMetrics); + services[name] = std::make_unique<Service>(serviceConfig, config.application, _outputConnections, _env.metrics()); } else { found->second->reconfigure(serviceConfig); services[name] = std::move(found->second); @@ -140,24 +105,22 @@ ConfigHandler::doConfigure() _orphans[entry.first] = std::move(svc); } } - vespalib::ComponentConfigProducer::Config current("sentinel", _subscriber.getGeneration(), "ok"); - _stateApi.myComponents.addConfig(current); + _env.notifyConfigUpdated(); } -int -ConfigHandler::doWork() +bool +Manager::doWork() { // Return true if there are any running services, false if not. - - if (_subscriber.nextGenerationNow()) { + if (_env.configOwner().checkForConfigUpdate()) { doConfigure(); } handleRestarts(); handleCommands(); handleOutputs(); handleChildDeaths(); - _startMetrics.maybeLog(); + _env.metrics().maybeLog(); // Check for active services. for (const auto & service : _services) { @@ -169,7 +132,7 @@ ConfigHandler::doWork() } void -ConfigHandler::handleRestarts() +Manager::handleRestarts() { for (const auto & entry : _services) { Service & svc = *(entry.second); @@ -180,7 +143,7 @@ ConfigHandler::handleRestarts() } void -ConfigHandler::handleChildDeaths() +Manager::handleChildDeaths() { // See if any of our child processes have exited, and take // the appropriate action. @@ -203,7 +166,7 @@ ConfigHandler::handleChildDeaths() } void -ConfigHandler::updateActiveFdset(fd_set *fds, int *maxNum) +Manager::updateActiveFdset(fd_set *fds, int *maxNum) { // ### _Possibly put an assert here if fd is > 1023??? for (OutputConnection *c : _outputConnections) { @@ -218,7 +181,7 @@ ConfigHandler::updateActiveFdset(fd_set *fds, int *maxNum) } void -ConfigHandler::handleOutputs() +Manager::handleOutputs() { std::list<OutputConnection *>::iterator dst; std::list<OutputConnection *>::const_iterator src; @@ -241,10 +204,10 @@ ConfigHandler::handleOutputs() } void -ConfigHandler::handleCommands() +Manager::handleCommands() { // handle RPC commands - std::vector<Cmd::UP> got = _cmdQ.drain(); + std::vector<Cmd::UP> got = _env.commandQueue().drain(); for (const Cmd::UP & cmd : got) { handleCmd(*cmd); } @@ -252,7 +215,7 @@ ConfigHandler::handleCommands() } Service * -ConfigHandler::serviceByPid(pid_t pid) +Manager::serviceByPid(pid_t pid) { for (const auto & service : _services) { if (service.second->pid() == pid) { @@ -269,7 +232,7 @@ ConfigHandler::serviceByPid(pid_t pid) } Service * -ConfigHandler::serviceByName(const vespalib::string & name) +Manager::serviceByName(const vespalib::string & name) { auto found(_services.find(name)); if (found != _services.end()) { @@ -280,7 +243,7 @@ ConfigHandler::serviceByName(const vespalib::string & name) void -ConfigHandler::handleCmd(const Cmd& cmd) +Manager::handleCmd(const Cmd& cmd) { switch (cmd.type()) { case Cmd::LIST: @@ -353,9 +316,9 @@ ConfigHandler::handleCmd(const Cmd& cmd) } void -ConfigHandler::updateMetrics() +Manager::updateMetrics() { - _startMetrics.maybeLog(); + _env.metrics().maybeLog(); } } diff --git a/configd/src/apps/sentinel/config-handler.h b/configd/src/apps/sentinel/manager.h index 463e9e7ce6a..24bd67cbc49 100644 --- a/configd/src/apps/sentinel/config-handler.h +++ b/configd/src/apps/sentinel/manager.h @@ -1,11 +1,12 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #pragma once -#include "service.h" -#include "metrics.h" -#include "state-api.h" #include "cmdq.h" +#include "env.h" +#include "metrics.h" #include "rpcserver.h" +#include "service.h" +#include "state-api.h" #include <vespa/config-sentinel.h> #include <vespa/config/config.h> #include <vespa/vespalib/net/state_server.h> @@ -22,24 +23,22 @@ namespace config::sentinel { class OutputConnection; -class ConfigHandler { +/** + * Management of services. + * Handles requests from RPC, service events, + * and service configuration updates. + **/ +class Manager { private: typedef std::map<vespalib::string, Service::UP> ServiceMap; - ConfigSubscriber _subscriber; - ConfigHandle<SentinelConfig>::UP _sentinelHandle; + Env &_env; ServiceMap _services; ServiceMap _orphans; std::list<OutputConnection *> _outputConnections; - CommandQueue _cmdQ; - std::unique_ptr<RpcServer> _rpcServer; - int _boundPort; - StartMetrics _startMetrics; - StateApi _stateApi; - std::unique_ptr<vespalib::StateServer> _stateServer; - ConfigHandler(const ConfigHandler&); - ConfigHandler& operator =(const ConfigHandler&); + Manager(const Manager&) = delete; + Manager& operator =(const Manager&) = delete; Service *serviceByPid(pid_t pid); Service *serviceByName(const vespalib::string & name); @@ -49,21 +48,14 @@ private: void handleChildDeaths(); void handleRestarts(); - static int listen(int port); - void configure_port(int port); - void updateMetrics(); - void terminateServices(bool catchable, bool printDebug = false); - void doConfigure(); - public: - ConfigHandler(); - virtual ~ConfigHandler(); - void subscribe(const std::string & configId, std::chrono::milliseconds timeout); + Manager(Env &env); + virtual ~Manager(); bool terminate(); - int doWork(); + bool doWork(); void updateActiveFdset(fd_set *fds, int *maxNum); }; diff --git a/configd/src/apps/sentinel/outward-check.cpp b/configd/src/apps/sentinel/outward-check.cpp new file mode 100644 index 00000000000..5fed69d0b6e --- /dev/null +++ b/configd/src/apps/sentinel/outward-check.cpp @@ -0,0 +1,55 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include "outward-check.h" +#include <vespa/log/log.h> + +LOG_SETUP(".outward-check"); + +namespace config::sentinel { + +OutwardCheck::OutwardCheck(const std::string &spec, OutwardCheckContext &context) + : _spec(spec), + _context(context) +{ + _target = context.orb.GetTarget(spec.c_str()); + _req = context.orb.AllocRPCRequest(); + _req->SetMethodName("sentinel.check.connectivity"); + _req->GetParams()->AddString(context.myHostname); + _req->GetParams()->AddInt32(context.myPortnum); + _req->GetParams()->AddInt32(500); + _target->InvokeAsync(_req, 1.500, this); +} + +OutwardCheck::~OutwardCheck() = default; + +void OutwardCheck::RequestDone(FRT_RPCRequest *req) { + LOG_ASSERT(req == _req); + if (req->CheckReturnTypes("s")) { + std::string answer = _req->GetReturn()->GetValue(0)._string._str; + if (answer == "ok") { + LOG(debug, "ping to %s with reverse connectivity OK", _spec.c_str()); + _result = CcResult::ALL_OK; + } else { + LOG(debug, "connected to %s, but reverse connectivity fails: %s", + _spec.c_str(), answer.c_str()); + _result = CcResult::REVERSE_FAIL; + } + } else if (req->GetErrorCode() == FRTE_RPC_NO_SUCH_METHOD || + req->GetErrorCode() == FRTE_RPC_WRONG_PARAMS || + req->GetErrorCode() == FRTE_RPC_WRONG_RETURN) + { + LOG(debug, "Connected OK to %s but no reverse connectivity check available", _spec.c_str()); + _result = CcResult::REVERSE_UNAVAIL; + } else { + LOG(debug, "error on request to %s : %s (%d)", _spec.c_str(), + req->GetErrorMessage(), req->GetErrorCode()); + _result = CcResult::CONN_FAIL; + } + _req->SubRef(); + _req = nullptr; + _target->SubRef(); + _target = nullptr; + _context.latch.countDown(); +} + +} diff --git a/configd/src/apps/sentinel/outward-check.h b/configd/src/apps/sentinel/outward-check.h new file mode 100644 index 00000000000..01a298aee18 --- /dev/null +++ b/configd/src/apps/sentinel/outward-check.h @@ -0,0 +1,46 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include <string> +#include <vespa/vespalib/util/count_down_latch.h> +#include <vespa/fnet/frt/supervisor.h> +#include <vespa/fnet/frt/invoker.h> +#include <vespa/fnet/frt/rpcrequest.h> +#include <vespa/fnet/frt/supervisor.h> +#include <vespa/fnet/frt/target.h> + +namespace config::sentinel { + +struct OutwardCheckContext { + vespalib::CountDownLatch latch; + const char * myHostname; + int myPortnum; + FRT_Supervisor &orb; + OutwardCheckContext(size_t count, + const char * hostname, + int portnumber, + FRT_Supervisor &supervisor) + : latch(count), + myHostname(hostname), + myPortnum(portnumber), + orb(supervisor) + {} +}; + +enum class CcResult { UNKNOWN, CONN_FAIL, REVERSE_FAIL, REVERSE_UNAVAIL, ALL_OK }; + +class OutwardCheck : public FRT_IRequestWait { +private: + CcResult _result = CcResult::UNKNOWN; + FRT_Target *_target = nullptr; + FRT_RPCRequest *_req = nullptr; + std::string _spec; + OutwardCheckContext &_context; +public: + OutwardCheck(const std::string &spec, OutwardCheckContext &context); + virtual ~OutwardCheck(); + void RequestDone(FRT_RPCRequest *req) override; + bool ok() const { return _result == CcResult::ALL_OK; } + CcResult result() const { return _result; } +}; + +} diff --git a/configd/src/apps/sentinel/peer-check.cpp b/configd/src/apps/sentinel/peer-check.cpp new file mode 100644 index 00000000000..60c3d9c96c9 --- /dev/null +++ b/configd/src/apps/sentinel/peer-check.cpp @@ -0,0 +1,50 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include "peer-check.h" +#include <vespa/vespalib/util/stringfmt.h> +#include <vespa/log/log.h> + +LOG_SETUP(".peer-check"); + +using vespalib::make_string_short::fmt; + +namespace config::sentinel { + +PeerCheck::PeerCheck(StatusCallback &callback, const std::string &host, int port, FRT_Supervisor &orb, int timeout_ms) + : _callback(callback), + _hostname(host), + _portnum(port), + _target(nullptr), + _req(nullptr) +{ + auto spec = fmt("tcp/%s:%d", _hostname.c_str(), _portnum); + _target = orb.GetTarget(spec.c_str()); + _req = orb.AllocRPCRequest(); + _req->SetMethodName("frt.rpc.ping"); + _target->InvokeAsync(_req, timeout_ms * 0.001, this); +} + +PeerCheck::~PeerCheck() { + LOG_ASSERT(_req == nullptr); + LOG_ASSERT(_target == nullptr); +} + +void PeerCheck::RequestDone(FRT_RPCRequest *req) { + LOG_ASSERT(req == _req); + bool statusOk = false; + if (req->IsError()) { + LOG(warning, "error on ping to %s [port %d]: %s (%d)", _hostname.c_str(), _portnum, + req->GetErrorMessage(), req->GetErrorCode()); + } else { + LOG(debug, "OK ping to %s [port %d]", _hostname.c_str(), _portnum); + statusOk = true; + } + _req->SubRef(); + _req = nullptr; + _target->SubRef(); + _target = nullptr; + // Note: will delete this object, so must be called as final step: + _callback.returnStatus(statusOk); +} + +} diff --git a/configd/src/apps/sentinel/peer-check.h b/configd/src/apps/sentinel/peer-check.h new file mode 100644 index 00000000000..096f304467b --- /dev/null +++ b/configd/src/apps/sentinel/peer-check.h @@ -0,0 +1,35 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#pragma once + +#include "status-callback.h" +#include <string> +#include <vespa/fnet/frt/invoker.h> +#include <vespa/fnet/frt/rpcrequest.h> +#include <vespa/fnet/frt/supervisor.h> +#include <vespa/fnet/frt/target.h> + +namespace config::sentinel { + +class PeerCheck : public FRT_IRequestWait +{ +public: + PeerCheck(StatusCallback &callback, const std::string &host, int portnum, FRT_Supervisor &orb, int timeout_ms); + ~PeerCheck(); + + PeerCheck(const PeerCheck &) = delete; + PeerCheck(PeerCheck &&) = delete; + PeerCheck& operator= (const PeerCheck &) = delete; + PeerCheck& operator= (PeerCheck &&) = delete; + + /** from FRT_IRequestWait **/ + void RequestDone(FRT_RPCRequest *req) override; +private: + StatusCallback &_callback; + std::string _hostname; + int _portnum; + FRT_Target *_target; + FRT_RPCRequest *_req; +}; + +} diff --git a/configd/src/apps/sentinel/rpchooks.cpp b/configd/src/apps/sentinel/rpchooks.cpp index aef58b8a1dc..24e3cd53509 100644 --- a/configd/src/apps/sentinel/rpchooks.cpp +++ b/configd/src/apps/sentinel/rpchooks.cpp @@ -2,6 +2,8 @@ #include "rpchooks.h" #include "cmdq.h" +#include "check-completion-handler.h" +#include "peer-check.h" #include <vespa/fnet/frt/supervisor.h> #include <vespa/fnet/frt/rpcrequest.h> @@ -10,6 +12,13 @@ LOG_SETUP(".rpchooks"); namespace config::sentinel { +RPCHooks::RPCHooks(CommandQueue &commands, FRT_Supervisor &supervisor) + : _commands(commands), + _orb(supervisor) +{ + initRPC(&_orb); +} + RPCHooks::~RPCHooks() = default; @@ -36,6 +45,14 @@ RPCHooks::initRPC(FRT_Supervisor *supervisor) FRT_METHOD(RPCHooks::rpc_startService), this); rb.MethodDesc("start a service"); //------------------------------------------------------------------------- + rb.DefineMethod("sentinel.check.connectivity", "sii", "s", + FRT_METHOD(RPCHooks::rpc_checkConnectivity), this); + rb.MethodDesc("check connectivity for peer sentinel"); + rb.ParamDesc("name", "Hostname of peer sentinel"); + rb.ParamDesc("port", "Port number of peer sentinel"); + rb.ParamDesc("timeout", "Timeout for check in milliseconds"); + rb.ReturnDesc("status", "Status (ok, bad, or unknown) for peer"); + //------------------------------------------------------------------------- } void @@ -76,4 +93,17 @@ RPCHooks::rpc_startService(FRT_RPCRequest *req) _commands.enqueue(std::make_unique<Cmd>(req, Cmd::START, srvNM)); } +void +RPCHooks::rpc_checkConnectivity(FRT_RPCRequest *req) +{ + FRT_Values &args = *req->GetParams(); + const char *hostname = args[0]._string._str; + int portnum = args[1]._intval32; + int timeout = args[2]._intval32; + LOG(debug, "got checkConnectivity %s [port %d] timeout %d", hostname, portnum, timeout); + req->Detach(); + auto & completionHandler = req->getStash().create<CheckCompletionHandler>(req); + req->getStash().create<PeerCheck>(completionHandler, hostname, portnum, _orb, timeout); +} + } // namespace slobrok diff --git a/configd/src/apps/sentinel/rpchooks.h b/configd/src/apps/sentinel/rpchooks.h index 05070830491..67f5804dcf7 100644 --- a/configd/src/apps/sentinel/rpchooks.h +++ b/configd/src/apps/sentinel/rpchooks.h @@ -24,17 +24,18 @@ class RPCHooks : public FRT_Invokable { private: CommandQueue &_commands; - + FRT_Supervisor &_orb; public: - RPCHooks(CommandQueue &commands) : _commands(commands) {} + RPCHooks(CommandQueue &commands, FRT_Supervisor &supervisor); ~RPCHooks() override; - - void initRPC(FRT_Supervisor *supervisor); private: + void initRPC(FRT_Supervisor *supervisor); + void rpc_listServices(FRT_RPCRequest *req); void rpc_restartService(FRT_RPCRequest *req); void rpc_stopService(FRT_RPCRequest *req); void rpc_startService(FRT_RPCRequest *req); + void rpc_checkConnectivity(FRT_RPCRequest *req); }; } // namespace config::sentinel diff --git a/configd/src/apps/sentinel/rpcserver.cpp b/configd/src/apps/sentinel/rpcserver.cpp index a49cba50e4d..80c3c81c826 100644 --- a/configd/src/apps/sentinel/rpcserver.cpp +++ b/configd/src/apps/sentinel/rpcserver.cpp @@ -9,10 +9,9 @@ namespace config::sentinel { RpcServer::RpcServer(int portNumber, CommandQueue &cmdQ) : _server(), - _rpcHooks(cmdQ), + _rpcHooks(cmdQ, _server.supervisor()), _port(portNumber) { - _rpcHooks.initRPC(&_server.supervisor()); if (_server.supervisor().Listen(portNumber)) { LOG(config, "listening on port %d", portNumber); } else { diff --git a/configd/src/apps/sentinel/rpcserver.h b/configd/src/apps/sentinel/rpcserver.h index ef4b394fdca..4c6dea00ddf 100644 --- a/configd/src/apps/sentinel/rpcserver.h +++ b/configd/src/apps/sentinel/rpcserver.h @@ -22,6 +22,7 @@ public: ~RpcServer(); int getPort() const { return _port; } + FRT_Supervisor &orb() { return _server.supervisor(); } }; } // namespace config::sentinel diff --git a/configd/src/apps/sentinel/sentinel.cpp b/configd/src/apps/sentinel/sentinel.cpp index 1bca6f72d10..7f3ddcc5882 100644 --- a/configd/src/apps/sentinel/sentinel.cpp +++ b/configd/src/apps/sentinel/sentinel.cpp @@ -1,6 +1,6 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include "config-handler.h" +#include "manager.h" #include <vespa/config/common/exceptions.h> #include <vespa/vespalib/util/signalhandler.h> #include <vespa/vespalib/util/exceptions.h> @@ -15,8 +15,6 @@ LOG_SETUP("config-sentinel"); using namespace config; -constexpr std::chrono::milliseconds CONFIG_TIMEOUT_MS(3 * 60 * 1000); - static bool stop() { return (vespalib::SignalHandler::INT.check() || @@ -61,11 +59,10 @@ main(int argc, char **argv) } setlocale(LC_ALL, "C"); - sentinel::ConfigHandler handler; - + sentinel::Env environment; LOG(debug, "Reading configuration"); try { - handler.subscribe(configId, CONFIG_TIMEOUT_MS); + environment.boot(configId); } catch (ConfigTimeoutException & ex) { LOG(warning, "Timeout getting config, please check your setup. Will exit and restart: %s", ex.getMessage().c_str()); EV_STOPPING("config-sentinel", ex.what()); @@ -80,11 +77,12 @@ main(int argc, char **argv) return EXIT_FAILURE; } + sentinel::Manager manager(environment); vespalib::steady_time lastTime = vespalib::steady_clock::now(); while (!stop()) { try { vespalib::SignalHandler::CHLD.clear(); - handler.doWork(); // Check for child procs & commands + manager.doWork(); // Check for child procs & commands } catch (InvalidConfigException& ex) { LOG(warning, "Configuration problem: (ignoring): %s", ex.what()); } catch (vespalib::PortListenException& ex) { @@ -102,7 +100,7 @@ main(int argc, char **argv) int maxNum = 0; fd_set fds; FD_ZERO(&fds); - handler.updateActiveFdset(&fds, &maxNum); + manager.updateActiveFdset(&fds, &maxNum); struct timeval tv; tv.tv_sec = 0; @@ -118,6 +116,6 @@ main(int argc, char **argv) } EV_STOPPING("config-sentinel", "normal exit"); - int rv = handler.terminate(); - return rv; + bool rv = manager.terminate(); + return rv ? EXIT_SUCCESS : EXIT_FAILURE; } diff --git a/configd/src/apps/sentinel/status-callback.cpp b/configd/src/apps/sentinel/status-callback.cpp new file mode 100644 index 00000000000..0bf11050b8e --- /dev/null +++ b/configd/src/apps/sentinel/status-callback.cpp @@ -0,0 +1,6 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include "status-callback.h" +namespace config::sentinel { + +} diff --git a/configd/src/apps/sentinel/status-callback.h b/configd/src/apps/sentinel/status-callback.h new file mode 100644 index 00000000000..54100277fdc --- /dev/null +++ b/configd/src/apps/sentinel/status-callback.h @@ -0,0 +1,14 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#pragma once + +namespace config::sentinel { + +/** very simple callback API with "ok" or "not ok" status only */ +struct StatusCallback { + virtual void returnStatus(bool ok) = 0; +protected: + ~StatusCallback() = default; +}; + +} diff --git a/configdefinitions/src/vespa/configserver.def b/configdefinitions/src/vespa/configserver.def index f8f4d6b0d92..d489bf2012e 100644 --- a/configdefinitions/src/vespa/configserver.def +++ b/configdefinitions/src/vespa/configserver.def @@ -61,9 +61,6 @@ maxDurationOfBootstrap long default=7200 # Code uses backoff, so wait time will increase for every iteration sleepTimeWhenRedeployingFails long default=15 -# Features (to be overridden in configserver-config.xml if needed) -buildMinimalSetOfConfigModels bool default=true - # Unused, remove in Vespa 8 throwIfActiveSessionCannotBeLoaded bool default=true diff --git a/configdefinitions/src/vespa/sentinel.def b/configdefinitions/src/vespa/sentinel.def index d318a55cccf..45ef9b21cfd 100644 --- a/configdefinitions/src/vespa/sentinel.def +++ b/configdefinitions/src/vespa/sentinel.def @@ -17,6 +17,17 @@ application.environment string default="default" application.instance string default="default" application.region string default="default" +# Connectivity checks run before starting services and measure how +# many nodes in the Vespa cluster we can connect to and how many of +# those that can connect back to us. We delay starting services +# if we have more problems than the following limits allow: + +## Percentage we fail to talk to, maximum +connectivity.maxBadOutPercent int default=100 + +## Absolute number of nodes that fail to talk back to us, maximum +connectivity.maxBadReverseCount int default=999999999 + ## The command to run. This will be run by sh -c, and the following ## environment variables are defined: $ROOT, $VESPA_SERVICE_NAME, ## $VESPA_CONFIG_ID diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java index 195f9dbf8a7..8805c339482 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java @@ -37,6 +37,7 @@ import com.yahoo.vespa.flags.UnboundFlag; import java.io.File; import java.net.URI; +import java.security.cert.X509Certificate; import java.util.List; import java.util.Optional; import java.util.Set; @@ -167,17 +168,18 @@ public class ModelContextImpl implements ModelContext { private final boolean skipMbusReplyThread; private final boolean useAsyncMessageHandlingOnSchedule; private final double feedConcurrency; - private final boolean useBucketExecutorForPruneRemoved; private final boolean enableFeedBlockInDistributor; private final ToIntFunction<ClusterSpec.Type> metricsProxyMaxHeapSizeInMb; private final List<String> allowedAthenzProxyIdentities; - private final boolean tenantIamRole; private final int maxActivationInhibitedOutOfSyncGroups; private final ToIntFunction<ClusterSpec.Type> jvmOmitStackTraceInFastThrow; private final boolean enableCustomAclMapping; private final boolean useExternalRankExpression; private final boolean distributeExternalRankExpressions; private final int numDistributorStripes; + private final boolean requireConnectivityCheck; + private final int maxConcurrentMergesPerContentNode; + private final int maxMergeQueueSize; public FeatureFlags(FlagSource source, ApplicationId appId) { this.dedicatedClusterControllerFlavor = parseDedicatedClusterControllerFlavor(flagValue(source, appId, Flags.DEDICATED_CLUSTER_CONTROLLER_FLAVOR)); @@ -191,17 +193,18 @@ public class ModelContextImpl implements ModelContext { this.skipMbusReplyThread = flagValue(source, appId, Flags.SKIP_MBUS_REPLY_THREAD); this.useAsyncMessageHandlingOnSchedule = flagValue(source, appId, Flags.USE_ASYNC_MESSAGE_HANDLING_ON_SCHEDULE); this.feedConcurrency = flagValue(source, appId, Flags.FEED_CONCURRENCY); - this.useBucketExecutorForPruneRemoved = flagValue(source, appId, Flags.USE_BUCKET_EXECUTOR_FOR_PRUNE_REMOVED); this.enableFeedBlockInDistributor = flagValue(source, appId, Flags.ENABLE_FEED_BLOCK_IN_DISTRIBUTOR); this.metricsProxyMaxHeapSizeInMb = type -> Flags.METRICS_PROXY_MAX_HEAP_SIZE_IN_MB.bindTo(source).with(CLUSTER_TYPE, type.name()).value(); this.allowedAthenzProxyIdentities = flagValue(source, appId, Flags.ALLOWED_ATHENZ_PROXY_IDENTITIES); - this.tenantIamRole = flagValue(source, appId.tenant(), Flags.TENANT_IAM_ROLE); this.maxActivationInhibitedOutOfSyncGroups = flagValue(source, appId, Flags.MAX_ACTIVATION_INHIBITED_OUT_OF_SYNC_GROUPS); this.jvmOmitStackTraceInFastThrow = type -> flagValueAsInt(source, appId, type, PermanentFlags.JVM_OMIT_STACK_TRACE_IN_FAST_THROW); this.enableCustomAclMapping = flagValue(source, appId, Flags.ENABLE_CUSTOM_ACL_MAPPING); this.numDistributorStripes = flagValue(source, appId, Flags.NUM_DISTRIBUTOR_STRIPES); this.useExternalRankExpression = flagValue(source, appId, Flags.USE_EXTERNAL_RANK_EXPRESSION); this.distributeExternalRankExpressions = flagValue(source, appId, Flags.DISTRIBUTE_EXTERNAL_RANK_EXPRESSION); + this.requireConnectivityCheck = flagValue(source, appId, Flags.REQUIRE_CONNECTIVITY_CHECK); + this.maxConcurrentMergesPerContentNode = flagValue(source, appId, Flags.MAX_CONCURRENT_MERGES_PER_NODE); + this.maxMergeQueueSize = flagValue(source, appId, Flags.MAX_MERGE_QUEUE_SIZE); } @Override public Optional<NodeResources> dedicatedClusterControllerFlavor() { return Optional.ofNullable(dedicatedClusterControllerFlavor); } @@ -215,11 +218,9 @@ public class ModelContextImpl implements ModelContext { @Override public boolean skipMbusReplyThread() { return skipMbusReplyThread; } @Override public boolean useAsyncMessageHandlingOnSchedule() { return useAsyncMessageHandlingOnSchedule; } @Override public double feedConcurrency() { return feedConcurrency; } - @Override public boolean useBucketExecutorForPruneRemoved() { return useBucketExecutorForPruneRemoved; } @Override public boolean enableFeedBlockInDistributor() { return enableFeedBlockInDistributor; } @Override public int metricsProxyMaxHeapSizeInMb(ClusterSpec.Type type) { return metricsProxyMaxHeapSizeInMb.applyAsInt(type); } @Override public List<String> allowedAthenzProxyIdentities() { return allowedAthenzProxyIdentities; } - @Override public boolean tenantIamRole() { return tenantIamRole; } @Override public int maxActivationInhibitedOutOfSyncGroups() { return maxActivationInhibitedOutOfSyncGroups; } @Override public String jvmOmitStackTraceInFastThrowOption(ClusterSpec.Type type) { return translateJvmOmitStackTraceInFastThrowIntToString(jvmOmitStackTraceInFastThrow, type); @@ -228,6 +229,9 @@ public class ModelContextImpl implements ModelContext { @Override public int numDistributorStripes() { return numDistributorStripes; } @Override public boolean useExternalRankExpressions() { return useExternalRankExpression; } @Override public boolean distributeExternalRankExpressions() { return distributeExternalRankExpressions; } + @Override public boolean requireConnectivityCheck() { return requireConnectivityCheck; } + @Override public int maxConcurrentMergesPerNode() { return maxConcurrentMergesPerContentNode; } + @Override public int maxMergeQueueSize() { return maxMergeQueueSize; } private static <V> V flagValue(FlagSource source, ApplicationId appId, UnboundFlag<? extends V, ?, ?> flag) { return flag.bindTo(source) @@ -287,6 +291,7 @@ public class ModelContextImpl implements ModelContext { private final SecretStore secretStore; private final StringFlag jvmGCOptionsFlag; private final boolean allowDisableMtls; + private final List<X509Certificate> operatorCertificates; public Properties(ApplicationId applicationId, ConfigserverConfig configserverConfig, @@ -300,7 +305,8 @@ public class ModelContextImpl implements ModelContext { Optional<ApplicationRoles> applicationRoles, Optional<Quota> maybeQuota, List<TenantSecretStore> tenantSecretStores, - SecretStore secretStore) { + SecretStore secretStore, + List<X509Certificate> operatorCertificates) { this.featureFlags = new FeatureFlags(flagSource, applicationId); this.applicationId = applicationId; this.multitenant = configserverConfig.multitenant() || configserverConfig.hostedVespa() || Boolean.getBoolean("multitenant"); @@ -323,6 +329,7 @@ public class ModelContextImpl implements ModelContext { .with(FetchVector.Dimension.APPLICATION_ID, applicationId.serializedForm()); this.allowDisableMtls = PermanentFlags.ALLOW_DISABLE_MTLS.bindTo(flagSource) .with(FetchVector.Dimension.APPLICATION_ID, applicationId.serializedForm()).value(); + this.operatorCertificates = operatorCertificates; } @Override public ModelContext.FeatureFlags featureFlags() { return featureFlags; } @@ -391,6 +398,11 @@ public class ModelContextImpl implements ModelContext { return allowDisableMtls; } + @Override + public List<X509Certificate> operatorCertificates() { + return operatorCertificates; + } + public String flagValueForClusterType(StringFlag flag, Optional<ClusterSpec.Type> clusterType) { return clusterType.map(type -> flag.with(CLUSTER_TYPE, type.name())) .orElse(flag) diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ZooKeeperClient.java b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ZooKeeperClient.java index 1ab99db8985..f0a63757477 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ZooKeeperClient.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ZooKeeperClient.java @@ -35,7 +35,7 @@ import static com.yahoo.vespa.config.server.zookeeper.ConfigCurator.USERAPP_ZK_S import static com.yahoo.vespa.config.server.zookeeper.ConfigCurator.USER_DEFCONFIGS_ZK_SUBPATH; /** - * A class used for reading and writing application data to zookeeper. + * Reads and writes application package to and from ZooKeeper. * * @author hmusum */ @@ -76,13 +76,13 @@ public class ZooKeeperClient { * * @param app the application package to feed to zookeeper */ - void write(ApplicationPackage app) { + void writeApplicationPackage(ApplicationPackage app) { try { writeUserDefs(app); writeSomeOf(app); - writeSearchDefinitions(app); + writeSchemas(app); writeUserIncludeDirs(app, app.getUserIncludeDirs()); - write(app.getMetaData()); + writeMetadata(app.getMetaData()); } catch (Exception e) { throw new IllegalStateException("Unable to write vespa model to config server(s) " + System.getProperty("configsources") + "\n" + "Please ensure that config server is started " + @@ -90,13 +90,11 @@ public class ZooKeeperClient { } } - private void writeSearchDefinitions(ApplicationPackage app) throws IOException { - Collection<NamedReader> sds = app.getSearchDefinitions(); + private void writeSchemas(ApplicationPackage app) throws IOException { + Collection<NamedReader> sds = app.getSchemas(); if (sds.isEmpty()) return; - // TODO: Change to SCHEMAS_DIR after March 2020 - // TODO: When it does also check RankExpressionFile.sendTo - Path zkPath = getZooKeeperAppPath(USERAPP_ZK_SUBPATH).append(ApplicationPackage.SEARCH_DEFINITIONS_DIR); + Path zkPath = getZooKeeperAppPath(USERAPP_ZK_SUBPATH).append(SCHEMAS_DIR); configCurator.createNode(zkPath.getAbsolute()); // Ensures that ranking expressions and other files are also written writeDir(app.getFile(ApplicationPackage.SEARCH_DEFINITIONS_DIR), zkPath, false); @@ -155,7 +153,6 @@ public class ZooKeeperClient { for (ApplicationFile file : listFiles(dir, filenameFilter)) { String name = file.getPath().getName(); if (name.startsWith(".")) continue; //.svn , .git ... - if ("CVS".equals(name)) continue; if (file.isDirectory()) { configCurator.createNode(path.append(name).getAbsolute()); if (recurse) { @@ -200,7 +197,6 @@ public class ZooKeeperClient { } private void writeUserIncludeDirs(ApplicationPackage applicationPackage, List<String> userIncludeDirs) throws IOException { - // User defined include directories for (String userInclude : userIncludeDirs) { ApplicationFile dir = applicationPackage.getFile(Path.fromString(userInclude)); final List<ApplicationFile> files = dir.listFiles(); @@ -240,12 +236,12 @@ public class ZooKeeperClient { } /** - * Feeds application metadata to zookeeper. Used by vespamodel to create config - * for application metadata (used by ApplicationStatusHandler) + * Feeds application metadata to zookeeper. Used by config model to create config + * for application metadata * * @param metaData The application metadata. */ - private void write(ApplicationMetaData metaData) { + private void writeMetadata(ApplicationMetaData metaData) { configCurator.putData(getZooKeeperAppPath(META_ZK_PATH).getAbsolute(), metaData.asJsonBytes()); } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ZooKeeperDeployer.java b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ZooKeeperDeployer.java index 12aa5b7cc35..8c7d6ea28dd 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ZooKeeperDeployer.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ZooKeeperDeployer.java @@ -33,7 +33,7 @@ public class ZooKeeperDeployer { public void deploy(ApplicationPackage applicationPackage, Map<Version, FileRegistry> fileRegistryMap, AllocatedHosts allocatedHosts) throws IOException { zooKeeperClient.initialize(); - zooKeeperClient.write(applicationPackage); + zooKeeperClient.writeApplicationPackage(applicationPackage); zooKeeperClient.write(fileRegistryMap); zooKeeperClient.write(allocatedHosts); } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/ApplicationApiHandler.java b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/ApplicationApiHandler.java index 5b520b10fcf..dfbce72d4ba 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/ApplicationApiHandler.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/ApplicationApiHandler.java @@ -82,7 +82,7 @@ public class ApplicationApiHandler extends SessionHandler { .collect(Collectors.toMap(Part::getName, p -> p)); byte[] params = parts.get(MULTIPART_PARAMS).getInputStream().readAllBytes(); - log.log(Level.FINE, "Deploy parameters: [{}]", new String(params, StandardCharsets.UTF_8)); + log.log(Level.FINE, "Deploy parameters: [{0}]", new String(params, StandardCharsets.UTF_8)); prepareParams = PrepareParams.fromJson(params, tenantName, zookeeperBarrierTimeout); Part appPackagePart = parts.get(MULTIPART_APPLICATION_PACKAGE); compressedStream = createFromCompressedStream(appPackagePart.getInputStream(), appPackagePart.getContentType()); diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/ApplicationPackageMaintainer.java b/configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/ApplicationPackageMaintainer.java index 5519ffc1bdc..003b4fbb345 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/ApplicationPackageMaintainer.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/ApplicationPackageMaintainer.java @@ -48,8 +48,9 @@ public class ApplicationPackageMaintainer extends ConfigServerMaintainer { } @Override - protected boolean maintain() { - boolean success = true; + protected double maintain() { + int attempts = 0; + int failures = 0; try (var fileDownloader = new FileDownloader(connectionPool, downloadDirectory)) { for (var applicationId : applicationRepository.listApplications()) { @@ -62,11 +63,12 @@ public class ApplicationPackageMaintainer extends ConfigServerMaintainer { log.fine(() -> "Verifying application package file reference " + applicationPackage + " for session " + sessionId); if (applicationPackage != null) { + attempts++; if (! fileReferenceExistsOnDisk(downloadDirectory, applicationPackage)) { log.fine(() -> "Downloading missing application package for application " + applicationId + " - session " + sessionId); if (fileDownloader.getFile(applicationPackage).isEmpty()) { - success = false; + failures++; log.warning("Failed to download application package for application " + applicationId + " - session " + sessionId); continue; } @@ -75,7 +77,7 @@ public class ApplicationPackageMaintainer extends ConfigServerMaintainer { } } } - return success; + return asSuccessFactor(attempts, failures); } @Override diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/ConfigServerMaintainer.java b/configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/ConfigServerMaintainer.java index 4938f34131e..e0f0a4b4099 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/ConfigServerMaintainer.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/ConfigServerMaintainer.java @@ -35,14 +35,24 @@ public abstract class ConfigServerMaintainer extends Maintainer { ConfigServerMaintainer(ApplicationRepository applicationRepository, Curator curator, FlagSource flagSource, Instant now, Duration interval) { super(null, interval, now, new JobControl(new JobControlFlags(curator, flagSource)), - jobMetrics(applicationRepository.metric()), cluster(curator), false); + new ConfigServerJobMetrics(applicationRepository.metric()), cluster(curator), false); this.applicationRepository = applicationRepository; } - private static JobMetrics jobMetrics(Metric metric) { - return new JobMetrics((job, consecutiveFailures) -> { + private static class ConfigServerJobMetrics extends JobMetrics { + + private final Metric metric; + + public ConfigServerJobMetrics(Metric metric) { + this.metric = metric; + } + + @Override + protected void recordCompletion(String job, Long consecutiveFailures, double successFactor) { metric.set("maintenance.consecutiveFailures", consecutiveFailures, metric.createContext(Map.of("job", job))); - }); + metric.set("maintenance.successFactor", successFactor, metric.createContext(Map.of("job", job))); + } + } private static class JobControlFlags implements JobControlState { diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/FileDistributionMaintainer.java b/configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/FileDistributionMaintainer.java index b0876fb57e8..ca8db30c21f 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/FileDistributionMaintainer.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/FileDistributionMaintainer.java @@ -33,9 +33,9 @@ public class FileDistributionMaintainer extends ConfigServerMaintainer { } @Override - protected boolean maintain() { + protected double maintain() { applicationRepository.deleteUnusedFiledistributionReferences(fileReferencesDir, maxUnusedFileReferenceAge); - return true; + return 1.0; } } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/ReindexingMaintainer.java b/configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/ReindexingMaintainer.java index 971c2c20ae9..af9ea917aaf 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/ReindexingMaintainer.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/ReindexingMaintainer.java @@ -22,6 +22,7 @@ import java.util.Comparator; import java.util.Map; import java.util.Optional; import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicLong; import java.util.function.Supplier; import java.util.logging.Level; @@ -51,8 +52,9 @@ public class ReindexingMaintainer extends ConfigServerMaintainer { } @Override - protected boolean maintain() { - AtomicBoolean success = new AtomicBoolean(true); + protected double maintain() { + AtomicInteger attempts = new AtomicInteger(0); + AtomicInteger failures = new AtomicInteger(0); for (Tenant tenant : applicationRepository.tenantRepository().getAllTenants()) { ApplicationCuratorDatabase database = tenant.getApplicationRepo().database(); for (ApplicationId id : database.activeApplications()) @@ -60,6 +62,7 @@ public class ReindexingMaintainer extends ConfigServerMaintainer { .map(application -> application.getForVersionOrLatest(Optional.empty(), clock.instant())) .ifPresent(application -> { try { + attempts.incrementAndGet(); applicationRepository.modifyReindexing(id, reindexing -> { reindexing = withNewReady(reindexing, lazyGeneration(application), clock.instant()); reindexing = withOnlyCurrentData(reindexing, application); @@ -68,11 +71,11 @@ public class ReindexingMaintainer extends ConfigServerMaintainer { } catch (RuntimeException e) { log.log(Level.INFO, "Failed to update reindexing status for " + id + ": " + Exceptions.toMessageString(e)); - success.set(false); + failures.incrementAndGet(); } }); } - return success.get(); + return asSuccessFactor(attempts.get(), failures.get()); } private Supplier<Long> lazyGeneration(Application application) { diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/SessionsMaintainer.java b/configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/SessionsMaintainer.java index 7482980e221..1f85dd4579d 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/SessionsMaintainer.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/SessionsMaintainer.java @@ -25,7 +25,7 @@ public class SessionsMaintainer extends ConfigServerMaintainer { } @Override - protected boolean maintain() { + protected double maintain() { if (iteration % 10 == 0) log.log(Level.INFO, () -> "Running " + SessionsMaintainer.class.getSimpleName() + ", iteration " + iteration); @@ -38,7 +38,7 @@ public class SessionsMaintainer extends ConfigServerMaintainer { } iteration++; - return true; + return 1.0; } } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/TenantsMaintainer.java b/configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/TenantsMaintainer.java index 7c01045ee72..0a7df2c9d21 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/TenantsMaintainer.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/TenantsMaintainer.java @@ -31,12 +31,12 @@ public class TenantsMaintainer extends ConfigServerMaintainer { } @Override - protected boolean maintain() { - if ( ! applicationRepository.configserverConfig().hostedVespa()) return true; + protected double maintain() { + if ( ! applicationRepository.configserverConfig().hostedVespa()) return 1.0; Set<TenantName> tenants = applicationRepository.deleteUnusedTenants(ttlForUnusedTenant, clock.instant()); if (tenants.size() > 0) log.log(Level.INFO, "Deleted tenants " + tenants); - return true; + return 1.0; } } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/modelfactory/ActivatedModelsBuilder.java b/configserver/src/main/java/com/yahoo/vespa/config/server/modelfactory/ActivatedModelsBuilder.java index 4e7afa7b3db..2b7ce234777 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/modelfactory/ActivatedModelsBuilder.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/modelfactory/ActivatedModelsBuilder.java @@ -38,7 +38,9 @@ import com.yahoo.vespa.config.server.tenant.TenantRepository; import com.yahoo.vespa.curator.Curator; import com.yahoo.vespa.flags.FlagSource; +import java.security.cert.X509Certificate; import java.util.Comparator; +import java.util.List; import java.util.Map; import java.util.Optional; import java.util.logging.Level; @@ -164,7 +166,8 @@ public class ActivatedModelsBuilder extends ModelsBuilder<Application> { .readApplicationRoles(applicationId), zkClient.readQuota(), zkClient.readTenantSecretStores(), - secretStore); + secretStore, + zkClient.readOperatorCertificates()); } } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/modelfactory/ModelsBuilder.java b/configserver/src/main/java/com/yahoo/vespa/config/server/modelfactory/ModelsBuilder.java index 707a828bdd6..2d4aa78bcf6 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/modelfactory/ModelsBuilder.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/modelfactory/ModelsBuilder.java @@ -1,7 +1,6 @@ // Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.config.server.modelfactory; -import com.google.common.util.concurrent.UncheckedTimeoutException; import com.yahoo.cloud.config.ConfigserverConfig; import com.yahoo.component.Version; import com.yahoo.config.application.api.ApplicationPackage; @@ -32,7 +31,6 @@ import java.util.ArrayList; import java.util.Collections; import java.util.Comparator; import java.util.List; -import java.util.NoSuchElementException; import java.util.Optional; import java.util.Set; import java.util.logging.Level; @@ -224,8 +222,7 @@ public abstract class ModelsBuilder<MODELRESULT extends ModelResult> { } private Set<Version> versionsToBuild(Set<Version> versions, Version wantedVersion, int majorVersion, AllocatedHosts allocatedHosts) { - if (configserverConfig.buildMinimalSetOfConfigModels()) - versions = keepThoseUsedOn(allocatedHosts, versions); + versions = keepThoseUsedOn(allocatedHosts, versions); // Make sure we build wanted version if we are building models for this major version and we are on hosted vespa // If not on hosted vespa, we do not want to try to build this version, since we have only one version (the latest) diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/session/PrepareParams.java b/configserver/src/main/java/com/yahoo/vespa/config/server/session/PrepareParams.java index ea2a525b440..071a0dd8f0c 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/session/PrepareParams.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/session/PrepareParams.java @@ -1,6 +1,7 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.config.server.session; +import com.google.common.collect.ImmutableList; import com.yahoo.component.Version; import com.yahoo.config.model.api.ApplicationRoles; import com.yahoo.config.model.api.ContainerEndpoint; @@ -11,6 +12,8 @@ import com.yahoo.config.provision.AthenzDomain; import com.yahoo.config.provision.DockerImage; import com.yahoo.config.provision.TenantName; import com.yahoo.container.jdisc.HttpRequest; +import com.yahoo.security.X509CertificateUtils; +import com.yahoo.slime.ArrayTraverser; import com.yahoo.slime.Inspector; import com.yahoo.slime.Slime; import com.yahoo.slime.SlimeUtils; @@ -20,14 +23,18 @@ import com.yahoo.vespa.config.server.http.SessionHandler; import com.yahoo.vespa.config.server.tenant.ContainerEndpointSerializer; import com.yahoo.vespa.config.server.tenant.EndpointCertificateMetadataSerializer; import com.yahoo.vespa.config.server.tenant.TenantSecretStoreSerializer; +import org.eclipse.jetty.util.ssl.X509; +import java.security.cert.X509Certificate; import java.time.Clock; import java.time.Duration; +import java.util.ArrayList; import java.util.Collections; import java.util.List; import java.util.Objects; import java.util.Optional; import java.util.function.Function; +import java.util.stream.Collectors; /** * Parameters for preparing an application. Immutable. @@ -52,6 +59,7 @@ public final class PrepareParams { static final String TENANT_SECRET_STORES_PARAM_NAME = "tenantSecretStores"; static final String FORCE_PARAM_NAME = "force"; static final String WAIT_FOR_RESOURCES_IN_PREPARE = "waitForResourcesInPrepare"; + static final String OPERATOR_CERTIFICATES = "operatorCertificates"; private final ApplicationId applicationId; private final TimeoutBudget timeoutBudget; @@ -69,6 +77,7 @@ public final class PrepareParams { private final Optional<ApplicationRoles> applicationRoles; private final Optional<Quota> quota; private final List<TenantSecretStore> tenantSecretStores; + private final List<X509Certificate> operatorCertificates; private PrepareParams(ApplicationId applicationId, TimeoutBudget timeoutBudget, boolean ignoreValidationErrors, boolean dryRun, boolean verbose, boolean isBootstrap, Optional<Version> vespaVersion, @@ -76,7 +85,7 @@ public final class PrepareParams { Optional<EndpointCertificateMetadata> endpointCertificateMetadata, Optional<DockerImage> dockerImageRepository, Optional<AthenzDomain> athenzDomain, Optional<ApplicationRoles> applicationRoles, Optional<Quota> quota, List<TenantSecretStore> tenantSecretStores, - boolean force, boolean waitForResourcesInPrepare) { + boolean force, boolean waitForResourcesInPrepare, List<X509Certificate> operatorCertificates) { this.timeoutBudget = timeoutBudget; this.applicationId = Objects.requireNonNull(applicationId); this.ignoreValidationErrors = ignoreValidationErrors; @@ -93,6 +102,7 @@ public final class PrepareParams { this.tenantSecretStores = tenantSecretStores; this.force = force; this.waitForResourcesInPrepare = waitForResourcesInPrepare; + this.operatorCertificates = operatorCertificates; } public static class Builder { @@ -113,6 +123,7 @@ public final class PrepareParams { private Optional<ApplicationRoles> applicationRoles = Optional.empty(); private Optional<Quota> quota = Optional.empty(); private List<TenantSecretStore> tenantSecretStores = List.of(); + private List<X509Certificate> operatorCertificates = List.of(); public Builder() { } @@ -245,11 +256,17 @@ public final class PrepareParams { return this; } + public Builder withOperatorCertificates(List<X509Certificate> operatorCertificates) { + this.operatorCertificates = List.copyOf(operatorCertificates); + return this; + } + public PrepareParams build() { return new PrepareParams(applicationId, timeoutBudget, ignoreValidationErrors, dryRun, verbose, isBootstrap, vespaVersion, containerEndpoints, endpointCertificateMetadata, dockerImageRepository, athenzDomain, - applicationRoles, quota, tenantSecretStores, force, waitForResourcesInPrepare); + applicationRoles, quota, tenantSecretStores, force, waitForResourcesInPrepare, + operatorCertificates); } } @@ -289,9 +306,10 @@ public final class PrepareParams { .athenzDomain(SlimeUtils.optionalString(params.field(ATHENZ_DOMAIN)).orElse(null)) .applicationRoles(ApplicationRoles.fromString(SlimeUtils.optionalString(params.field(APPLICATION_HOST_ROLE)).orElse(null), SlimeUtils.optionalString(params.field(APPLICATION_CONTAINER_ROLE)).orElse(null))) .quota(deserialize(params.field(QUOTA_PARAM_NAME), Quota::fromSlime)) - .tenantSecretStores(SlimeUtils.optionalString(params.field(TENANT_SECRET_STORES_PARAM_NAME)).orElse(null)) + .tenantSecretStores(deserialize(params.field(TENANT_SECRET_STORES_PARAM_NAME), TenantSecretStoreSerializer::listFromSlime, List.of())) .force(booleanValue(params, FORCE_PARAM_NAME)) .waitForResourcesInPrepare(booleanValue(params, WAIT_FOR_RESOURCES_IN_PREPARE)) + .withOperatorCertificates(deserialize(params.field(OPERATOR_CERTIFICATES), PrepareParams::readOperatorCertificates, Collections.emptyList())) .build(); } @@ -343,6 +361,13 @@ public final class PrepareParams { return Optional.ofNullable(request.getProperty(propertyName)); } + private static List<X509Certificate> readOperatorCertificates(Inspector array) { + return SlimeUtils.entriesStream(array) + .map(Inspector::asString) + .map(X509CertificateUtils::fromPem) + .collect(Collectors.toList()); + } + public String getApplicationName() { return applicationId.application().value(); } @@ -400,4 +425,8 @@ public final class PrepareParams { public List<TenantSecretStore> tenantSecretStores() { return tenantSecretStores; } + + public List<X509Certificate> operatorCertificates() { + return operatorCertificates; + } } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/session/Session.java b/configserver/src/main/java/com/yahoo/vespa/config/server/session/Session.java index f1044b28049..542b54d877e 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/session/Session.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/session/Session.java @@ -17,6 +17,7 @@ import com.yahoo.transaction.Transaction; import com.yahoo.vespa.config.server.application.ApplicationSet; import com.yahoo.vespa.config.server.tenant.TenantRepository; +import java.security.cert.X509Certificate; import java.time.Instant; import java.util.List; import java.util.Optional; @@ -137,6 +138,10 @@ public abstract class Session implements Comparable<Session> { sessionZooKeeperClient.writeTenantSecretStores(tenantSecretStores); } + public void setOperatorCertificates(List<X509Certificate> operatorCertificates) { + sessionZooKeeperClient.writeOperatorCertificates(operatorCertificates); + } + /** Returns application id read from ZooKeeper. Will throw RuntimeException if not found */ public ApplicationId getApplicationId() { return sessionZooKeeperClient.readApplicationId() @@ -172,6 +177,10 @@ public abstract class Session implements Comparable<Session> { return sessionZooKeeperClient.readTenantSecretStores(); } + public List<X509Certificate> getOperatorCertificates() { + return sessionZooKeeperClient.readOperatorCertificates(); + } + private Transaction createSetStatusTransaction(Status status) { return sessionZooKeeperClient.createWriteStatusTransaction(status); } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionPreparer.java b/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionPreparer.java index 50b9ac55bda..dedd9e08655 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionPreparer.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionPreparer.java @@ -49,6 +49,7 @@ import com.yahoo.vespa.flags.FlagSource; import java.io.File; import java.io.IOException; +import java.security.cert.X509Certificate; import java.time.Instant; import java.util.Collection; import java.util.List; @@ -202,7 +203,8 @@ public class SessionPreparer { applicationRoles, params.quota(), params.tenantSecretStores(), - secretStore); + secretStore, + params.operatorCertificates()); this.fileDistributionProvider = fileDistributionFactory.createProvider(serverDbSessionDir); this.preparedModelsBuilder = new PreparedModelsBuilder(modelFactoryRegistry, permanentApplicationPackage, @@ -275,7 +277,8 @@ public class SessionPreparer { prepareResult.allocatedHosts(), athenzDomain, params.quota(), - params.tenantSecretStores()); + params.tenantSecretStores(), + params.operatorCertificates()); checkTimeout("write state to zookeeper"); } @@ -325,7 +328,8 @@ public class SessionPreparer { AllocatedHosts allocatedHosts, Optional<AthenzDomain> athenzDomain, Optional<Quota> quota, - List<TenantSecretStore> tenantSecretStores) { + List<TenantSecretStore> tenantSecretStores, + List<X509Certificate> operatorCertificates) { ZooKeeperDeployer zkDeployer = zooKeeperClient.createDeployer(deployLogger); try { zkDeployer.deploy(applicationPackage, fileRegistryMap, allocatedHosts); @@ -337,6 +341,7 @@ public class SessionPreparer { zooKeeperClient.writeAthenzDomain(athenzDomain); zooKeeperClient.writeQuota(quota); zooKeeperClient.writeTenantSecretStores(tenantSecretStores); + zooKeeperClient.writeOperatorCertificates(operatorCertificates); } catch (RuntimeException | IOException e) { zkDeployer.cleanup(); throw new RuntimeException("Error preparing session", e); diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionRepository.java b/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionRepository.java index 7d74b53fdff..ac350db5c21 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionRepository.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionRepository.java @@ -256,6 +256,7 @@ public class SessionRepository { session.setDockerImageRepository(existingSession.getDockerImageRepository()); session.setAthenzDomain(existingSession.getAthenzDomain()); session.setTenantSecretStores(existingSession.getTenantSecretStores()); + session.setOperatorCertificates(existingSession.getOperatorCertificates()); return session; } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionZooKeeperClient.java b/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionZooKeeperClient.java index c7c4f1926d7..c3d6bba0ac2 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionZooKeeperClient.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionZooKeeperClient.java @@ -21,6 +21,7 @@ import com.yahoo.transaction.Transaction; import com.yahoo.vespa.config.server.UserConfigDefinitionRepo; import com.yahoo.vespa.config.server.deploy.ZooKeeperClient; import com.yahoo.vespa.config.server.deploy.ZooKeeperDeployer; +import com.yahoo.vespa.config.server.tenant.OperatorCertificateSerializer; import com.yahoo.vespa.config.server.tenant.TenantRepository; import com.yahoo.vespa.config.server.tenant.TenantSecretStoreSerializer; import com.yahoo.vespa.config.server.zookeeper.ConfigCurator; @@ -29,6 +30,7 @@ import com.yahoo.vespa.curator.Curator; import com.yahoo.vespa.curator.transaction.CuratorOperations; import com.yahoo.vespa.curator.transaction.CuratorTransaction; +import java.security.cert.X509Certificate; import java.time.Instant; import java.util.List; import java.util.Optional; @@ -57,6 +59,7 @@ public class SessionZooKeeperClient { private static final String ATHENZ_DOMAIN = "athenzDomain"; private static final String QUOTA_PATH = "quota"; private static final String TENANT_SECRET_STORES_PATH = "tenantSecretStores"; + private static final String OPERATOR_CERTIFICATES_PATH = "operatorCertificates"; private final Curator curator; private final ConfigCurator configCurator; @@ -191,6 +194,10 @@ public class SessionZooKeeperClient { return sessionPath.append(TENANT_SECRET_STORES_PATH).getAbsolute(); } + private String operatorCertificatesPath() { + return sessionPath.append(OPERATOR_CERTIFICATES_PATH).getAbsolute(); + } + public void writeVespaVersion(Version version) { configCurator.putData(versionPath(), version.toString()); } @@ -282,6 +289,21 @@ public class SessionZooKeeperClient { .orElse(List.of()); } + public void writeOperatorCertificates(List<X509Certificate> certificates) { + if( ! certificates.isEmpty()) { + var bytes = uncheck(() -> SlimeUtils.toJsonBytes(OperatorCertificateSerializer.toSlime(certificates))); + configCurator.putData(operatorCertificatesPath(), bytes); + } + } + + public List<X509Certificate> readOperatorCertificates() { + if ( ! configCurator.exists(operatorCertificatesPath())) return List.of(); + return Optional.ofNullable(configCurator.getData(operatorCertificatesPath())) + .map(SlimeUtils::jsonToSlime) + .map(slime -> OperatorCertificateSerializer.fromSlime(slime.get())) + .orElse(List.of()); + } + /** * Create necessary paths atomically for a new session. * diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/OperatorCertificateSerializer.java b/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/OperatorCertificateSerializer.java new file mode 100644 index 00000000000..3dbdf1380f1 --- /dev/null +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/OperatorCertificateSerializer.java @@ -0,0 +1,37 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +package com.yahoo.vespa.config.server.tenant; + +import com.yahoo.config.model.api.ApplicationRoles; +import com.yahoo.security.X509CertificateUtils; +import com.yahoo.slime.Cursor; +import com.yahoo.slime.Inspector; +import com.yahoo.slime.Slime; +import com.yahoo.slime.SlimeUtils; + +import java.security.cert.X509Certificate; +import java.util.List; +import java.util.stream.Collectors; + +public class OperatorCertificateSerializer { + + private final static String certificateField = "certificates"; + + + public static Slime toSlime(List<X509Certificate> certificateList) { + Slime slime = new Slime(); + var root = slime.setObject(); + Cursor array = root.setArray(certificateField); + certificateList.stream() + .map(X509CertificateUtils::toPem) + .forEach(array::addString); + return slime; + } + + public static List<X509Certificate> fromSlime(Inspector object) { + return SlimeUtils.entriesStream(object.field(certificateField)) + .map(Inspector::asString) + .map(X509CertificateUtils::fromPem) + .collect(Collectors.toList()); + } +} diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/zookeeper/ZKApplicationPackage.java b/configserver/src/main/java/com/yahoo/vespa/config/server/zookeeper/ZKApplicationPackage.java index f01bf1fe6ef..9f1bfa0b4e4 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/zookeeper/ZKApplicationPackage.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/zookeeper/ZKApplicationPackage.java @@ -132,16 +132,17 @@ public class ZKApplicationPackage implements ApplicationPackage { } @Override - public List<NamedReader> searchDefinitionContents() { + public List<NamedReader> getSchemas() { List<NamedReader> schemas = new ArrayList<>(); - for (String sd : zkApplication.getChildren(ConfigCurator.USERAPP_ZK_SUBPATH + "/" + SEARCH_DEFINITIONS_DIR)) { - if (sd.endsWith(SD_NAME_SUFFIX)) - schemas.add(new NamedReader(sd, new StringReader(zkApplication.getData(ConfigCurator.USERAPP_ZK_SUBPATH + "/" + SEARCH_DEFINITIONS_DIR, sd)))); - } for (String sd : zkApplication.getChildren(ConfigCurator.USERAPP_ZK_SUBPATH + "/" + SCHEMAS_DIR)) { if (sd.endsWith(SD_NAME_SUFFIX)) schemas.add(new NamedReader(sd, new StringReader(zkApplication.getData(ConfigCurator.USERAPP_ZK_SUBPATH + "/" + SCHEMAS_DIR, sd)))); } + // TODO: Remove when 7.414.19 is oldest version in use + for (String sd : zkApplication.getChildren(ConfigCurator.USERAPP_ZK_SUBPATH + "/" + SEARCH_DEFINITIONS_DIR)) { + if (sd.endsWith(SD_NAME_SUFFIX)) + schemas.add(new NamedReader(sd, new StringReader(zkApplication.getData(ConfigCurator.USERAPP_ZK_SUBPATH + "/" + SEARCH_DEFINITIONS_DIR, sd)))); + } return schemas; } @@ -164,11 +165,6 @@ public class ZKApplicationPackage implements ApplicationPackage { return fileRegistry; } - @Override - public List<NamedReader> getSearchDefinitions() { - return searchDefinitionContents(); - } - private Reader retrieveConfigDefReader(String def) { try { return zkApplication.getDataReader(ConfigCurator.DEFCONFIGS_ZK_SUBPATH, def); @@ -262,6 +258,7 @@ public class ZKApplicationPackage implements ApplicationPackage { @Override public Reader getRankingExpression(String name) { Optional<Reader> reader = zkApplication.getOptionalDataReader(ConfigCurator.USERAPP_ZK_SUBPATH + "/" + SCHEMAS_DIR, name); + // TODO: Remove when 7.414.19 is oldest version in use return reader.orElseGet(() -> zkApplication.getDataReader(ConfigCurator.USERAPP_ZK_SUBPATH + "/" + SEARCH_DEFINITIONS_DIR, name)); } diff --git a/configserver/src/test/apps/app-jdisc-only-restart/searchdefinitions/music.sd b/configserver/src/test/apps/app-jdisc-only-restart/schemas/music.sd index a2d4614c657..a2d4614c657 100644 --- a/configserver/src/test/apps/app-jdisc-only-restart/searchdefinitions/music.sd +++ b/configserver/src/test/apps/app-jdisc-only-restart/schemas/music.sd diff --git a/configserver/src/test/apps/app-jdisc-only/searchdefinitions/music.sd b/configserver/src/test/apps/app-jdisc-only/schemas/music.sd index a2d4614c657..a2d4614c657 100644 --- a/configserver/src/test/apps/app-jdisc-only/searchdefinitions/music.sd +++ b/configserver/src/test/apps/app-jdisc-only/schemas/music.sd diff --git a/configserver/src/test/apps/app-major-version-2/searchdefinitions/music.sd b/configserver/src/test/apps/app-major-version-2/schemas/music.sd index 7670e78f22b..7670e78f22b 100644 --- a/configserver/src/test/apps/app-major-version-2/searchdefinitions/music.sd +++ b/configserver/src/test/apps/app-major-version-2/schemas/music.sd diff --git a/configserver/src/test/apps/app/searchdefinitions/music.sd b/configserver/src/test/apps/app/schemas/music.sd index 7670e78f22b..7670e78f22b 100644 --- a/configserver/src/test/apps/app/searchdefinitions/music.sd +++ b/configserver/src/test/apps/app/schemas/music.sd diff --git a/configserver/src/test/apps/hosted-no-write-access-control/searchdefinitions/music.sd b/configserver/src/test/apps/hosted-no-write-access-control/schemas/music.sd index 78d58b27d4a..78d58b27d4a 100644 --- a/configserver/src/test/apps/hosted-no-write-access-control/searchdefinitions/music.sd +++ b/configserver/src/test/apps/hosted-no-write-access-control/schemas/music.sd diff --git a/configserver/src/test/apps/hosted/searchdefinitions/music.sd b/configserver/src/test/apps/hosted/schemas/music.sd index 78d58b27d4a..78d58b27d4a 100644 --- a/configserver/src/test/apps/hosted/searchdefinitions/music.sd +++ b/configserver/src/test/apps/hosted/schemas/music.sd diff --git a/configserver/src/test/apps/zkapp/searchdefinitions/bar.expression b/configserver/src/test/apps/zkapp/schemas/bar.expression index eed496e6aeb..eed496e6aeb 100644 --- a/configserver/src/test/apps/zkapp/searchdefinitions/bar.expression +++ b/configserver/src/test/apps/zkapp/schemas/bar.expression diff --git a/configserver/src/test/apps/zkapp/searchdefinitions/foo.expression b/configserver/src/test/apps/zkapp/schemas/foo.expression index ce26aa75dcb..ce26aa75dcb 100644 --- a/configserver/src/test/apps/zkapp/searchdefinitions/foo.expression +++ b/configserver/src/test/apps/zkapp/schemas/foo.expression diff --git a/configserver/src/test/apps/zkapp/searchdefinitions/laptop.sd b/configserver/src/test/apps/zkapp/schemas/laptop.sd index 21a76ad605c..21a76ad605c 100644 --- a/configserver/src/test/apps/zkapp/searchdefinitions/laptop.sd +++ b/configserver/src/test/apps/zkapp/schemas/laptop.sd diff --git a/configserver/src/test/apps/zkapp/searchdefinitions/music.sd b/configserver/src/test/apps/zkapp/schemas/music.sd index bdf4c060cf7..bdf4c060cf7 100644 --- a/configserver/src/test/apps/zkapp/searchdefinitions/music.sd +++ b/configserver/src/test/apps/zkapp/schemas/music.sd diff --git a/configserver/src/test/apps/zkapp/searchdefinitions/pc.sd b/configserver/src/test/apps/zkapp/schemas/pc.sd index bdc90328cdb..bdc90328cdb 100644 --- a/configserver/src/test/apps/zkapp/searchdefinitions/pc.sd +++ b/configserver/src/test/apps/zkapp/schemas/pc.sd diff --git a/configserver/src/test/apps/zkapp/searchdefinitions/product.sd b/configserver/src/test/apps/zkapp/schemas/product.sd index 588a1d544c6..588a1d544c6 100644 --- a/configserver/src/test/apps/zkapp/searchdefinitions/product.sd +++ b/configserver/src/test/apps/zkapp/schemas/product.sd diff --git a/configserver/src/test/apps/zkapp/searchdefinitions/sock.sd b/configserver/src/test/apps/zkapp/schemas/sock.sd index 9331c0caf54..9331c0caf54 100644 --- a/configserver/src/test/apps/zkapp/searchdefinitions/sock.sd +++ b/configserver/src/test/apps/zkapp/schemas/sock.sd diff --git a/configserver/src/test/apps/zkfeed/searchdefinitions/bar.expression b/configserver/src/test/apps/zkfeed/schemas/bar.expression index eed496e6aeb..eed496e6aeb 100644 --- a/configserver/src/test/apps/zkfeed/searchdefinitions/bar.expression +++ b/configserver/src/test/apps/zkfeed/schemas/bar.expression diff --git a/configserver/src/test/apps/zkfeed/searchdefinitions/foo.expression b/configserver/src/test/apps/zkfeed/schemas/foo.expression index ce26aa75dcb..ce26aa75dcb 100644 --- a/configserver/src/test/apps/zkfeed/searchdefinitions/foo.expression +++ b/configserver/src/test/apps/zkfeed/schemas/foo.expression diff --git a/configserver/src/test/apps/zkfeed/searchdefinitions/laptop.sd b/configserver/src/test/apps/zkfeed/schemas/laptop.sd index 21a76ad605c..21a76ad605c 100644 --- a/configserver/src/test/apps/zkfeed/searchdefinitions/laptop.sd +++ b/configserver/src/test/apps/zkfeed/schemas/laptop.sd diff --git a/configserver/src/test/apps/zkfeed/searchdefinitions/pc.sd b/configserver/src/test/apps/zkfeed/schemas/pc.sd index bdc90328cdb..bdc90328cdb 100644 --- a/configserver/src/test/apps/zkfeed/searchdefinitions/pc.sd +++ b/configserver/src/test/apps/zkfeed/schemas/pc.sd diff --git a/configserver/src/test/apps/zkfeed/searchdefinitions/product.sd b/configserver/src/test/apps/zkfeed/schemas/product.sd index 588a1d544c6..588a1d544c6 100644 --- a/configserver/src/test/apps/zkfeed/searchdefinitions/product.sd +++ b/configserver/src/test/apps/zkfeed/schemas/product.sd diff --git a/configserver/src/test/apps/zkfeed/searchdefinitions/sock.sd b/configserver/src/test/apps/zkfeed/schemas/sock.sd index 9331c0caf54..9331c0caf54 100644 --- a/configserver/src/test/apps/zkfeed/searchdefinitions/sock.sd +++ b/configserver/src/test/apps/zkfeed/schemas/sock.sd diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/ModelContextImplTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/ModelContextImplTest.java index 7b9420b6b9e..0acf4404326 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/ModelContextImplTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/ModelContextImplTest.java @@ -74,7 +74,8 @@ public class ModelContextImplTest { Optional.empty(), Optional.empty(), List.of(), - new SecretStoreProvider().get()), + new SecretStoreProvider().get(), + List.of()), Optional.empty(), Optional.empty(), new Version(7), diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/application/CompressedApplicationInputStreamTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/application/CompressedApplicationInputStreamTest.java index 016192e3281..2a18a1b2221 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/application/CompressedApplicationInputStreamTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/application/CompressedApplicationInputStreamTest.java @@ -141,9 +141,9 @@ public class CompressedApplicationInputStreamTest { assertTrue(files.contains(new File(outApp, "services.xml"))); assertTrue(files.contains(new File(outApp, "hosts.xml"))); assertTrue(files.contains(new File(outApp, "deployment.xml"))); - assertTrue(files.contains(new File(outApp, "searchdefinitions"))); + assertTrue(files.contains(new File(outApp, "schemas"))); assertTrue(files.contains(new File(outApp, "external"))); - File sd = files.get(files.indexOf(new File(outApp, "searchdefinitions"))); + File sd = files.get(files.indexOf(new File(outApp, "schemas"))); assertTrue(sd.isDirectory()); assertThat(sd.listFiles().length, is(1)); assertThat(sd.listFiles()[0].getAbsolutePath(), is(new File(sd, "keyvalue.sd").getAbsolutePath())); diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/deploy/ZooKeeperClientTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/deploy/ZooKeeperClientTest.java index 7d14b1996b0..e20363af4e9 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/deploy/ZooKeeperClientTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/deploy/ZooKeeperClientTest.java @@ -64,7 +64,7 @@ public class ZooKeeperClientTest { Map<Version, FileRegistry> fileRegistries = createFileRegistries(); app.writeMetaData(); zkc.initialize(); - zkc.write(app); + zkc.writeApplicationPackage(app); zkc.write(fileRegistries); } diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/session/PrepareParamsTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/session/PrepareParamsTest.java index f50238f2b85..f68e79ae266 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/session/PrepareParamsTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/session/PrepareParamsTest.java @@ -4,9 +4,16 @@ package com.yahoo.vespa.config.server.session; import com.yahoo.config.model.api.ApplicationRoles; import com.yahoo.config.model.api.ContainerEndpoint; import com.yahoo.config.model.api.EndpointCertificateMetadata; +import com.yahoo.config.model.api.TenantSecretStore; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.TenantName; import com.yahoo.container.jdisc.HttpRequest; +import com.yahoo.security.KeyAlgorithm; +import com.yahoo.security.KeyUtils; +import com.yahoo.security.SignatureAlgorithm; +import com.yahoo.security.X509CertificateBuilder; +import com.yahoo.security.X509CertificateUtils; +import com.yahoo.security.X509CertificateWithKey; import com.yahoo.slime.ArrayInserter; import com.yahoo.slime.Cursor; import com.yahoo.slime.Injector; @@ -18,12 +25,19 @@ import com.yahoo.slime.SlimeInserter; import com.yahoo.slime.SlimeUtils; import com.yahoo.vespa.config.server.tenant.ContainerEndpointSerializer; import com.yahoo.vespa.config.server.tenant.EndpointCertificateMetadataSerializer; +import com.yahoo.vespa.config.server.tenant.TenantSecretStoreSerializer; import org.junit.Test; +import javax.security.auth.x500.X500Principal; import java.io.IOException; +import java.math.BigInteger; import java.net.URLEncoder; import java.nio.charset.StandardCharsets; +import java.security.KeyPair; +import java.security.cert.X509Certificate; import java.time.Duration; +import java.time.Instant; +import java.time.temporal.ChronoUnit; import java.util.List; import java.util.Map; import java.util.Objects; @@ -179,6 +193,38 @@ public class PrepareParamsTest { assertPrepareParamsEqual(urlPrepareParams, jsonPrepareParams); } + @Test + public void testOperatorCertificates() throws IOException { + Slime slime = SlimeUtils.jsonToSlime(json); + Cursor cursor = slime.get(); + Cursor array = cursor.setArray(PrepareParams.OPERATOR_CERTIFICATES); + X509Certificate certificate = X509CertificateUtils.createSelfSigned("cn=myservice", Duration.ofDays(1)).certificate(); + array.addString(X509CertificateUtils.toPem(certificate)); + PrepareParams prepareParams = PrepareParams.fromJson(SlimeUtils.toJsonBytes(slime), TenantName.from("foo"), Duration.ofSeconds(60)); + assertEquals(1, prepareParams.operatorCertificates().size()); + assertEquals(certificate, prepareParams.operatorCertificates().get(0)); + } + + @Test + public void testSecretStores() throws IOException { + List<TenantSecretStore> secretStores = List.of(new TenantSecretStore("name", "awsId", "role")); + Slime secretStoreSlime = TenantSecretStoreSerializer.toSlime(secretStores); + String secretStoreParam = new String(SlimeUtils.toJsonBytes(secretStoreSlime), StandardCharsets.UTF_8); + + var prepareParams = createParams(request + "&" + PrepareParams.TENANT_SECRET_STORES_PARAM_NAME + "=" + URLEncoder.encode(secretStoreParam, StandardCharsets.UTF_8), TenantName.from("foo")); + assertEquals(1, prepareParams.tenantSecretStores().size()); + TenantSecretStore tenantSecretStore = prepareParams.tenantSecretStores().get(0); + assertEquals("name", tenantSecretStore.getName()); + assertEquals("awsId", tenantSecretStore.getAwsId()); + assertEquals("role", tenantSecretStore.getRole()); + + // Verify using json object + var root = SlimeUtils.jsonToSlime(json); + new Injector().inject(secretStoreSlime.get(), new ObjectInserter(root.get(), PrepareParams.TENANT_SECRET_STORES_PARAM_NAME)); + PrepareParams prepareParamsJson = PrepareParams.fromJson(SlimeUtils.toJsonBytes(root), TenantName.from("foo"), Duration.ofSeconds(60)); + assertPrepareParamsEqual(prepareParams, prepareParamsJson); + } + private void assertPrepareParamsEqual(PrepareParams urlParams, PrepareParams jsonParams) { assertEquals(urlParams.ignoreValidationErrors(), jsonParams.ignoreValidationErrors()); assertEquals(urlParams.isDryRun(), jsonParams.isDryRun()); diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/tenant/OperatorCertificateSerializerTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/tenant/OperatorCertificateSerializerTest.java new file mode 100644 index 00000000000..b77248f0840 --- /dev/null +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/tenant/OperatorCertificateSerializerTest.java @@ -0,0 +1,30 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +package com.yahoo.vespa.config.server.tenant; + +import com.yahoo.security.X509CertificateUtils; +import com.yahoo.security.X509CertificateWithKey; +import com.yahoo.slime.Slime; +import com.yahoo.slime.SlimeUtils; +import org.junit.Assert; +import org.junit.Test; + +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.security.cert.X509Certificate; +import java.time.Duration; +import java.util.List; + +import static org.junit.Assert.assertEquals; + +public class OperatorCertificateSerializerTest { + + @Test + public void testSerialization() { + X509Certificate certificate = X509CertificateUtils.createSelfSigned("cn=mycn", Duration.ofDays(1)).certificate(); + Slime slime = OperatorCertificateSerializer.toSlime(List.of(certificate)); + List<X509Certificate> deserialized = OperatorCertificateSerializer.fromSlime(slime.get()); + assertEquals(1, deserialized.size()); + assertEquals(certificate, deserialized.get(0)); + } +} diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/zookeeper/ZKApplicationPackageTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/zookeeper/ZKApplicationPackageTest.java index 80d01fa4d36..458cdb82066 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/zookeeper/ZKApplicationPackageTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/zookeeper/ZKApplicationPackageTest.java @@ -78,7 +78,7 @@ public class ZKApplicationPackageTest { assertTrue(Pattern.compile(".*<slobroks>.*",Pattern.MULTILINE+Pattern.DOTALL).matcher(IOUtils.readAll(zkApp.getFile(Path.fromString("services.xml")).createReader())).matches()); DeployState deployState = new DeployState.Builder().applicationPackage(zkApp).build(); assertEquals(deployState.getSchemas().size(), 5); - assertEquals(zkApp.searchDefinitionContents().size(), 5); + assertEquals(zkApp.getSchemas().size(), 5); assertEquals(IOUtils.readAll(zkApp.getRankingExpression("foo.expression")), "foo()+1\n"); assertEquals(zkApp.getFiles(Path.fromString(""), "xml").size(), 3); assertEquals(zkApp.getFileReference(Path.fromString("components/file.txt")).getAbsolutePath(), "/home/vespa/test/file.txt"); @@ -124,8 +124,7 @@ public class ZKApplicationPackageTest { } /** - * Takes for instance the dir /app and puts the contents into the given ZK path. Ignores files starting with dot, - * and dirs called CVS. + * Takes for instance the dir /app and puts the contents into the given ZK path. Ignores files starting with dot. * * @param dir directory which holds the summary class part files * @param path zookeeper path @@ -142,7 +141,6 @@ public class ZKApplicationPackageTest { } for (File file : listFiles(dir, filenameFilter)) { if (file.getName().startsWith(".")) continue; //.svn , .git ... - if ("CVS".equals(file.getName())) continue; if (file.isFile()) { String contents = IOUtils.readFile(file); zk.putData(path, file.getName(), contents); diff --git a/configserver/src/test/resources/deploy/advancedapp/searchdefinitions/keyvalue.sd b/configserver/src/test/resources/deploy/advancedapp/schemas/keyvalue.sd index 2d0f1ef0b75..2d0f1ef0b75 100644 --- a/configserver/src/test/resources/deploy/advancedapp/searchdefinitions/keyvalue.sd +++ b/configserver/src/test/resources/deploy/advancedapp/schemas/keyvalue.sd diff --git a/container-core/abi-spec.json b/container-core/abi-spec.json index efe6701342f..02d43104a3f 100644 --- a/container-core/abi-spec.json +++ b/container-core/abi-spec.json @@ -1039,6 +1039,7 @@ "public com.yahoo.jdisc.http.ConnectorConfig$Builder maxRequestsPerConnection(int)", "public com.yahoo.jdisc.http.ConnectorConfig$Builder maxConnectionLife(double)", "public com.yahoo.jdisc.http.ConnectorConfig$Builder http2Enabled(boolean)", + "public com.yahoo.jdisc.http.ConnectorConfig$Builder http2(com.yahoo.jdisc.http.ConnectorConfig$Http2$Builder)", "public final boolean dispatchGetConfig(com.yahoo.config.ConfigInstance$Producer)", "public final java.lang.String getDefMd5()", "public final java.lang.String getDefName()", @@ -1053,7 +1054,8 @@ "public com.yahoo.jdisc.http.ConnectorConfig$TlsClientAuthEnforcer$Builder tlsClientAuthEnforcer", "public com.yahoo.jdisc.http.ConnectorConfig$HealthCheckProxy$Builder healthCheckProxy", "public com.yahoo.jdisc.http.ConnectorConfig$ProxyProtocol$Builder proxyProtocol", - "public com.yahoo.jdisc.http.ConnectorConfig$SecureRedirect$Builder secureRedirect" + "public com.yahoo.jdisc.http.ConnectorConfig$SecureRedirect$Builder secureRedirect", + "public com.yahoo.jdisc.http.ConnectorConfig$Http2$Builder http2" ] }, "com.yahoo.jdisc.http.ConnectorConfig$HealthCheckProxy$Builder": { @@ -1089,6 +1091,37 @@ ], "fields": [] }, + "com.yahoo.jdisc.http.ConnectorConfig$Http2$Builder": { + "superClass": "java.lang.Object", + "interfaces": [ + "com.yahoo.config.ConfigBuilder" + ], + "attributes": [ + "public" + ], + "methods": [ + "public void <init>()", + "public void <init>(com.yahoo.jdisc.http.ConnectorConfig$Http2)", + "public com.yahoo.jdisc.http.ConnectorConfig$Http2$Builder streamIdleTimeout(double)", + "public com.yahoo.jdisc.http.ConnectorConfig$Http2$Builder maxConcurrentStreams(int)", + "public com.yahoo.jdisc.http.ConnectorConfig$Http2 build()" + ], + "fields": [] + }, + "com.yahoo.jdisc.http.ConnectorConfig$Http2": { + "superClass": "com.yahoo.config.InnerNode", + "interfaces": [], + "attributes": [ + "public", + "final" + ], + "methods": [ + "public void <init>(com.yahoo.jdisc.http.ConnectorConfig$Http2$Builder)", + "public double streamIdleTimeout()", + "public int maxConcurrentStreams()" + ], + "fields": [] + }, "com.yahoo.jdisc.http.ConnectorConfig$Producer": { "superClass": "java.lang.Object", "interfaces": [ @@ -1361,7 +1394,8 @@ "public com.yahoo.jdisc.http.ConnectorConfig$SecureRedirect secureRedirect()", "public int maxRequestsPerConnection()", "public double maxConnectionLife()", - "public boolean http2Enabled()" + "public boolean http2Enabled()", + "public com.yahoo.jdisc.http.ConnectorConfig$Http2 http2()" ], "fields": [ "public static final java.lang.String CONFIG_DEF_MD5", diff --git a/container-core/src/main/java/com/yahoo/jdisc/http/server/jetty/ConnectorFactory.java b/container-core/src/main/java/com/yahoo/jdisc/http/server/jetty/ConnectorFactory.java index 0d3ab1ce32d..4ad39f91a83 100644 --- a/container-core/src/main/java/com/yahoo/jdisc/http/server/jetty/ConnectorFactory.java +++ b/container-core/src/main/java/com/yahoo/jdisc/http/server/jetty/ConnectorFactory.java @@ -75,7 +75,7 @@ public class ConnectorFactory { connector.setName(connectorConfig.name()); connector.setAcceptQueueSize(connectorConfig.acceptQueueSize()); connector.setReuseAddress(connectorConfig.reuseAddress()); - connector.setIdleTimeout((long)(connectorConfig.idleTimeout() * 1000.0)); + connector.setIdleTimeout(toMillis(connector.getIdleTimeout())); return connector; } @@ -162,7 +162,8 @@ public class ConnectorFactory { private HTTP2ServerConnectionFactory newHttp2ConnectionFactory() { HTTP2ServerConnectionFactory factory = new HTTP2ServerConnectionFactory(newHttpConfiguration()); - factory.setMaxConcurrentStreams(4096); + factory.setStreamIdleTimeout(toMillis(connectorConfig.http2().streamIdleTimeout())); + factory.setMaxConcurrentStreams(connectorConfig.http2().maxConcurrentStreams()); return factory; } @@ -193,4 +194,6 @@ public class ConnectorFactory { || (config.implicitTlsEnabled() && TransportSecurityUtils.isTransportSecurityEnabled()); } + private static long toMillis(double seconds) { return (long)(seconds * 1000); } + } diff --git a/container-core/src/main/java/com/yahoo/jdisc/http/server/jetty/HttpRequestDispatch.java b/container-core/src/main/java/com/yahoo/jdisc/http/server/jetty/HttpRequestDispatch.java index 7828751df5a..ba292062197 100644 --- a/container-core/src/main/java/com/yahoo/jdisc/http/server/jetty/HttpRequestDispatch.java +++ b/container-core/src/main/java/com/yahoo/jdisc/http/server/jetty/HttpRequestDispatch.java @@ -26,6 +26,7 @@ import java.time.Instant; import java.util.Arrays; import java.util.concurrent.CompletableFuture; import java.util.concurrent.CompletionException; +import java.util.concurrent.TimeoutException; import java.util.concurrent.atomic.AtomicBoolean; import java.util.function.BiConsumer; import java.util.function.Consumer; @@ -119,6 +120,10 @@ class HttpRequestDispatch { error, () -> "Network connection was unexpectedly terminated: " + parent.jettyRequest.getRequestURI()); parent.metricReporter.prematurelyClosed(); + } else if (isErrorOfType(error, TimeoutException.class)) { + log.log(Level.FINE, + error, + () -> "Request/stream was timed out by Jetty: " + parent.jettyRequest.getRequestURI()); } else if (!isErrorOfType(error, OverloadException.class, BindingNotFoundException.class, RequestException.class)) { log.log(Level.WARNING, "Request failed: " + parent.jettyRequest.getRequestURI(), error); } diff --git a/container-core/src/main/java/com/yahoo/jdisc/http/server/jetty/ServletRequestReader.java b/container-core/src/main/java/com/yahoo/jdisc/http/server/jetty/ServletRequestReader.java index 26d74bdccb3..e2bf5711e15 100644 --- a/container-core/src/main/java/com/yahoo/jdisc/http/server/jetty/ServletRequestReader.java +++ b/container-core/src/main/java/com/yahoo/jdisc/http/server/jetty/ServletRequestReader.java @@ -251,6 +251,7 @@ class ServletRequestReader implements ReadListener { @Override public void onError(final Throwable t) { finishedFuture.completeExceptionally(t); + requestContentChannel.onError(t); doneReading(); } diff --git a/container-core/src/main/java/com/yahoo/jdisc/http/server/jetty/ServletResponseController.java b/container-core/src/main/java/com/yahoo/jdisc/http/server/jetty/ServletResponseController.java index 31fa9e9ebaa..d61a3745653 100644 --- a/container-core/src/main/java/com/yahoo/jdisc/http/server/jetty/ServletResponseController.java +++ b/container-core/src/main/java/com/yahoo/jdisc/http/server/jetty/ServletResponseController.java @@ -20,6 +20,7 @@ import java.nio.ByteBuffer; import java.util.Map; import java.util.Optional; import java.util.concurrent.CompletableFuture; +import java.util.concurrent.TimeoutException; import java.util.logging.Level; import java.util.logging.Logger; @@ -75,6 +76,9 @@ public class ServletResponseController { return HttpServletResponse.SC_NOT_FOUND; } else if (t instanceof RequestException) { return ((RequestException)t).getResponseStatus(); + } else if (t instanceof TimeoutException) { + // E.g stream idle timeout for HTTP/2 + return HttpServletResponse.SC_SERVICE_UNAVAILABLE; } else { return HttpServletResponse.SC_INTERNAL_SERVER_ERROR; } diff --git a/container-core/src/main/resources/configdefinitions/jdisc.http.jdisc.http.connector.def b/container-core/src/main/resources/configdefinitions/jdisc.http.jdisc.http.connector.def index 09b883a620e..0b01f690aea 100644 --- a/container-core/src/main/resources/configdefinitions/jdisc.http.jdisc.http.connector.def +++ b/container-core/src/main/resources/configdefinitions/jdisc.http.jdisc.http.connector.def @@ -128,3 +128,7 @@ maxConnectionLife double default=0.0 # Enable HTTP/2 (in addition to HTTP/1.1 using ALPN) http2Enabled bool default=true + +http2.streamIdleTimeout double default=600 + +http2.maxConcurrentStreams int default=4096 diff --git a/container-search/src/main/java/com/yahoo/search/handler/SearchHandler.java b/container-search/src/main/java/com/yahoo/search/handler/SearchHandler.java index ba9f1804d34..e2c15b6e35b 100644 --- a/container-search/src/main/java/com/yahoo/search/handler/SearchHandler.java +++ b/container-search/src/main/java/com/yahoo/search/handler/SearchHandler.java @@ -532,12 +532,13 @@ public class SearchHandler extends LoggingRequestHandler { if (query.getHits() > maxHits) { return new Result(query, ErrorMessage.createIllegalQuery(query.getHits() + - " hits requested, configured limit: " + maxHits + ".")); + " hits requested, configured limit: " + maxHits + + ". See https://docs.vespa.ai/en/reference/query-api-reference.html#native-execution-parameters")); } else if (query.getOffset() > maxOffset) { - return new Result(query, - ErrorMessage.createIllegalQuery("Offset of " + query.getOffset() + - " requested, configured limit: " + maxOffset + ".")); + return new Result(query, ErrorMessage.createIllegalQuery("Offset of " + query.getOffset() + + " requested, configured limit: " + maxOffset + + ". See https://docs.vespa.ai/en/reference/query-api-reference.html#native-execution-parameters")); } return null; } diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/application/v4/model/DeploymentData.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/application/v4/model/DeploymentData.java index d0b9653bbf3..55e1e879ef7 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/application/v4/model/DeploymentData.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/application/v4/model/DeploymentData.java @@ -12,6 +12,7 @@ import com.yahoo.vespa.hosted.controller.api.integration.certificates.EndpointCe import com.yahoo.vespa.hosted.controller.api.integration.configserver.ContainerEndpoint; import com.yahoo.vespa.hosted.controller.api.integration.secrets.TenantSecretStore; +import java.security.cert.X509Certificate; import java.util.List; import java.util.Optional; import java.util.Set; @@ -36,6 +37,7 @@ public class DeploymentData { private final Optional<TenantRoles> tenantRoles; private final Quota quota; private final List<TenantSecretStore> tenantSecretStores; + private final List<X509Certificate> operatorCertificates; public DeploymentData(ApplicationId instance, ZoneId zone, byte[] applicationPackage, Version platform, Set<ContainerEndpoint> containerEndpoints, @@ -44,7 +46,8 @@ public class DeploymentData { Optional<AthenzDomain> athenzDomain, Optional<TenantRoles> tenantRoles, Quota quota, - List<TenantSecretStore> tenantSecretStores) { + List<TenantSecretStore> tenantSecretStores, + List<X509Certificate> operatorCertificates) { this.instance = requireNonNull(instance); this.zone = requireNonNull(zone); this.applicationPackage = requireNonNull(applicationPackage); @@ -56,6 +59,7 @@ public class DeploymentData { this.tenantRoles = tenantRoles; this.quota = quota; this.tenantSecretStores = tenantSecretStores; + this.operatorCertificates = operatorCertificates; } public ApplicationId instance() { @@ -102,4 +106,7 @@ public class DeploymentData { return tenantSecretStores; } + public List<X509Certificate> operatorCertificates() { + return operatorCertificates; + } } diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/ServiceRegistry.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/ServiceRegistry.java index 98591ba41e2..0b17428296c 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/ServiceRegistry.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/ServiceRegistry.java @@ -2,6 +2,7 @@ package com.yahoo.vespa.hosted.controller.api.integration; import com.yahoo.vespa.hosted.controller.api.integration.archive.ArchiveService; +import com.yahoo.vespa.hosted.controller.api.integration.athenz.AccessControlService; import com.yahoo.vespa.hosted.controller.api.integration.aws.RoleService; import com.yahoo.vespa.hosted.controller.api.integration.aws.AwsEventFetcher; import com.yahoo.vespa.hosted.controller.api.integration.aws.ResourceTagger; @@ -93,4 +94,6 @@ public interface ServiceRegistry { ArchiveService archiveService(); ChangeRequestClient changeRequestClient(); + + AccessControlService accessControlService(); } diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/athenz/AccessControlService.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/athenz/AccessControlService.java new file mode 100644 index 00000000000..78c67236f78 --- /dev/null +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/athenz/AccessControlService.java @@ -0,0 +1,18 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +package com.yahoo.vespa.hosted.controller.api.integration.athenz; + +import com.yahoo.vespa.athenz.api.AthenzUser; + +import java.time.Instant; +import java.util.Collection; + +/** + * Manage operator data plane access control + * + * @author mortent + */ +public interface AccessControlService { + boolean approveDataPlaneAccess(AthenzUser user, Instant expiry); + Collection<AthenzUser> listMembers(); +} diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/athenz/AthenzAccessControlService.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/athenz/AthenzAccessControlService.java new file mode 100644 index 00000000000..233759f47a7 --- /dev/null +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/athenz/AthenzAccessControlService.java @@ -0,0 +1,45 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +package com.yahoo.vespa.hosted.controller.api.integration.athenz; + +import com.yahoo.vespa.athenz.api.AthenzDomain; +import com.yahoo.vespa.athenz.api.AthenzRole; +import com.yahoo.vespa.athenz.api.AthenzUser; +import com.yahoo.vespa.athenz.client.zms.ZmsClient; + +import java.time.Instant; +import java.util.Collection; +import java.util.List; +import java.util.stream.Collectors; + +public class AthenzAccessControlService implements AccessControlService { + + private static final String DATAPLANE_ACCESS_ROLENAME = "operator-data-plane"; + private final ZmsClient zmsClient; + private final AthenzRole dataPlaneAccessRole; + + + public AthenzAccessControlService(ZmsClient zmsClient, AthenzDomain domain) { + this.zmsClient = zmsClient; + this.dataPlaneAccessRole = new AthenzRole(domain, DATAPLANE_ACCESS_ROLENAME); + } + + @Override + public boolean approveDataPlaneAccess(AthenzUser user, Instant expiry) { + List<AthenzUser> users = zmsClient.listPendingRoleApprovals(dataPlaneAccessRole); + if (users.contains(user)) { + zmsClient.approvePendingRoleMembership(dataPlaneAccessRole, user, expiry); + return true; + } + return false; + } + + @Override + // Return list of approved members (users, excluding services) of data plane role + public Collection<AthenzUser> listMembers() { + return zmsClient.listMembers(dataPlaneAccessRole) + .stream().filter(AthenzUser.class::isInstance) + .map(AthenzUser.class::cast) + .collect(Collectors.toList()); + } +} diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/athenz/MockAccessControlService.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/athenz/MockAccessControlService.java new file mode 100644 index 00000000000..81bc7725c7a --- /dev/null +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/athenz/MockAccessControlService.java @@ -0,0 +1,34 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +package com.yahoo.vespa.hosted.controller.api.integration.athenz; + +import com.yahoo.vespa.athenz.api.AthenzUser; + +import java.time.Instant; +import java.util.Collection; +import java.util.HashSet; +import java.util.Set; + +public class MockAccessControlService implements AccessControlService { + + private final Set<AthenzUser> pendingMembers = new HashSet<>(); + private final Set<AthenzUser> members = new HashSet<>(); + + @Override + public boolean approveDataPlaneAccess(AthenzUser user, Instant expiry) { + if (pendingMembers.remove(user)) { + return members.add(user); + } else { + return false; + } + } + + @Override + public Collection<AthenzUser> listMembers() { + return Set.copyOf(members); + } + + public void addPendingMember(AthenzUser user) { + pendingMembers.add(user); + } +} diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/athenz/ZmsClientMock.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/athenz/ZmsClientMock.java index 942f0f35f58..deeecf217e7 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/athenz/ZmsClientMock.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/athenz/ZmsClientMock.java @@ -5,6 +5,7 @@ import com.yahoo.vespa.athenz.api.AthenzDomain; import com.yahoo.vespa.athenz.api.AthenzIdentity; import com.yahoo.vespa.athenz.api.AthenzResourceName; import com.yahoo.vespa.athenz.api.AthenzRole; +import com.yahoo.vespa.athenz.api.AthenzUser; import com.yahoo.vespa.athenz.api.OktaAccessToken; import com.yahoo.vespa.athenz.api.OktaIdentityToken; import com.yahoo.vespa.athenz.client.zms.RoleAction; @@ -12,6 +13,7 @@ import com.yahoo.vespa.athenz.client.zms.ZmsClient; import com.yahoo.vespa.athenz.client.zms.ZmsClientException; import com.yahoo.vespa.hosted.controller.api.identifiers.ApplicationId; +import java.time.Instant; import java.util.ArrayList; import java.util.List; import java.util.Optional; @@ -145,6 +147,19 @@ public class ZmsClientMock implements ZmsClient { return false; } + @Override + public List<AthenzUser> listPendingRoleApprovals(AthenzRole athenzRole) { + return List.of(); + } + + @Override + public void approvePendingRoleMembership(AthenzRole athenzRole, AthenzUser athenzUser, Instant expiry) { + } + + @Override + public List<AthenzIdentity> listMembers(AthenzRole athenzRole) { + return List.of(); + } @Override public void close() {} diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/Cluster.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/Cluster.java index b3317c7f268..ccfd3241810 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/Cluster.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/Cluster.java @@ -23,6 +23,7 @@ public class Cluster { private final Optional<ClusterResources> suggested; private final Utilization utilization; private final List<ScalingEvent> scalingEvents; + private final String autoscalingStatusCode; private final String autoscalingStatus; private final Duration scalingDuration; private final double maxQueryGrowthRate; @@ -37,6 +38,7 @@ public class Cluster { Optional<ClusterResources> suggested, Utilization utilization, List<ScalingEvent> scalingEvents, + String autoscalingStatusCode, String autoscalingStatus, Duration scalingDuration, double maxQueryGrowthRate, @@ -50,6 +52,7 @@ public class Cluster { this.suggested = suggested; this.utilization = utilization; this.scalingEvents = scalingEvents; + this.autoscalingStatusCode = autoscalingStatusCode; this.autoscalingStatus = autoscalingStatus; this.scalingDuration = scalingDuration; this.maxQueryGrowthRate = maxQueryGrowthRate; @@ -65,6 +68,7 @@ public class Cluster { public Optional<ClusterResources> suggested() { return suggested; } public Utilization utilization() { return utilization; } public List<ScalingEvent> scalingEvents() { return scalingEvents; } + public String autoscalingStatusCode() { return autoscalingStatusCode; } public String autoscalingStatus() { return autoscalingStatus; } public Duration scalingDuration() { return scalingDuration; } public double maxQueryGrowthRate() { return maxQueryGrowthRate; } diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/noderepository/ClusterData.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/noderepository/ClusterData.java index d62ab781a50..6f9b2b496bf 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/noderepository/ClusterData.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/noderepository/ClusterData.java @@ -35,6 +35,8 @@ public class ClusterData { public ClusterUtilizationData utilization; @JsonProperty("scalingEvents") public List<ScalingEventData> scalingEvents; + @JsonProperty("autoscalingStatusCode") + public String autoscalingStatusCode; @JsonProperty("autoscalingStatus") public String autoscalingStatus; @JsonProperty("scalingDuration") @@ -55,10 +57,11 @@ public class ClusterData { utilization == null ? Cluster.Utilization.empty() : utilization.toClusterUtilization(), scalingEvents == null ? List.of() : scalingEvents.stream().map(data -> data.toScalingEvent()).collect(Collectors.toList()), + autoscalingStatusCode, autoscalingStatus, - Duration.ofMillis(scalingDuration), - maxQueryGrowthRate, - currentQueryFractionOfMax); + scalingDuration == null ? Duration.ofMillis(0) : Duration.ofMillis(scalingDuration), + maxQueryGrowthRate == null ? -1 : maxQueryGrowthRate, + currentQueryFractionOfMax == null ? -1 : currentQueryFractionOfMax); } } diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/organization/DeploymentFailureMails.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/organization/DeploymentFailureMails.java index 2adc83ff415..96321ceaf73 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/organization/DeploymentFailureMails.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/organization/DeploymentFailureMails.java @@ -35,8 +35,8 @@ public class DeploymentFailureMails { public Mail installationFailure(RunId id, Collection<String> recipients) { return mail(id, recipients, "installation", - "as nodes were not able to start the new Java containers. " + - "This is very often due to a misconfiguration of the components of an " + + "as nodes were not able to upgrade to the new configuration. " + + "This is often due to a misconfiguration of the components of an " + "application, where one or more of these can not be instantiated."); } diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/resource/ResourceSnapshot.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/resource/ResourceSnapshot.java index 5c6effced93..a8d9c4b1f8a 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/resource/ResourceSnapshot.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/resource/ResourceSnapshot.java @@ -53,6 +53,10 @@ public class ResourceSnapshot { return applicationId; } + public ResourceAllocation allocation() { + return resourceAllocation; + } + public double getCpuCores() { return resourceAllocation.getCpuCores(); } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java index 0f9188d1f65..cb3c84f5bd1 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java @@ -63,6 +63,7 @@ import com.yahoo.vespa.hosted.controller.notification.NotificationSource; import com.yahoo.vespa.hosted.controller.persistence.CuratorDb; import com.yahoo.vespa.hosted.controller.security.AccessControl; import com.yahoo.vespa.hosted.controller.security.Credentials; +import com.yahoo.vespa.hosted.controller.support.access.SupportAccessGrant; import com.yahoo.vespa.hosted.controller.tenant.AthenzTenant; import com.yahoo.vespa.hosted.controller.tenant.CloudTenant; import com.yahoo.vespa.hosted.controller.tenant.Tenant; @@ -70,6 +71,7 @@ import com.yahoo.vespa.hosted.controller.versions.VespaVersion; import com.yahoo.yolean.Exceptions; import java.security.Principal; +import java.security.cert.X509Certificate; import java.time.Clock; import java.time.Duration; import java.time.Instant; @@ -88,6 +90,7 @@ import java.util.function.Consumer; import java.util.logging.Level; import java.util.logging.Logger; import java.util.stream.Collectors; +import java.util.stream.Stream; import static com.yahoo.vespa.hosted.controller.api.integration.configserver.Node.State.active; import static com.yahoo.vespa.hosted.controller.api.integration.configserver.Node.State.reserved; @@ -501,11 +504,14 @@ public class ApplicationController { .filter(tenant-> tenant instanceof CloudTenant) .map(tenant -> ((CloudTenant) tenant).tenantSecretStores()) .orElse(List.of()); + List<X509Certificate> operatorCertificates = controller.supportAccess().activeGrantsFor(new DeploymentId(application, zone)).stream() + .map(SupportAccessGrant::certificate) + .collect(toList()); ConfigServer.PreparedApplication preparedApplication = configServer.deploy(new DeploymentData(application, zone, applicationPackage.zippedContent(), platform, endpoints, endpointCertificateMetadata, dockerImageRepo, domain, - tenantRoles, deploymentQuota, tenantSecretStores)); + tenantRoles, deploymentQuota, tenantSecretStores, operatorCertificates)); return new ActivateResult(new RevisionId(applicationPackage.hash()), preparedApplication.prepareResponse(), applicationPackage.zippedContent().length); diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Instance.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Instance.java index 0fcb3cd9be4..72d3cf7723b 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Instance.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Instance.java @@ -24,6 +24,7 @@ import java.util.List; import java.util.Map; import java.util.Objects; import java.util.Optional; +import java.util.OptionalDouble; import java.util.OptionalLong; import java.util.Set; import java.util.function.Function; @@ -69,11 +70,13 @@ public class Instance { version, instant, DeploymentMetrics.none, DeploymentActivity.none, - QuotaUsage.none)); + QuotaUsage.none, + OptionalDouble.empty())); Deployment newDeployment = new Deployment(zone, applicationVersion, version, instant, previousDeployment.metrics().with(warnings), previousDeployment.activity(), - quotaUsage); + quotaUsage, + previousDeployment.cost()); return with(newDeployment); } @@ -99,6 +102,15 @@ public class Instance { return with(deployment.withMetrics(deploymentMetrics)); } + public Instance withDeploymentCosts(Map<ZoneId, Double> costByZone) { + Map<ZoneId, Deployment> deployments = this.deployments.entrySet().stream() + .map(entry -> Optional.ofNullable(costByZone.get(entry.getKey())) + .map(entry.getValue()::withCost) + .orElseGet(entry.getValue()::withoutCost)) + .collect(Collectors.toUnmodifiableMap(Deployment::zone, deployment -> deployment)); + return with(deployments); + } + public Instance withoutDeploymentIn(ZoneId zone) { Map<ZoneId, Deployment> deployments = new LinkedHashMap<>(this.deployments); deployments.remove(zone); diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/RoutingController.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/RoutingController.java index abdb394c278..1b1df28c201 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/RoutingController.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/RoutingController.java @@ -180,11 +180,11 @@ public class RoutingController { builder = builder.routingMethod(RoutingMethod.exclusive) .on(Port.tls()); Endpoint endpoint = builder.in(controller.system()); - endpointDnsNames.add(endpoint.dnsName()); if (controller.system().isPublic() && vespaAppDomainInCertificate.with(FetchVector.Dimension.APPLICATION_ID, deployment.applicationId().serializedForm()).value()) { Endpoint legacyEndpoint = builder.legacy().in(controller.system()); endpointDnsNames.add(legacyEndpoint.dnsName()); } + endpointDnsNames.add(endpoint.dnsName()); } return Collections.unmodifiableList(endpointDnsNames); } @@ -393,7 +393,7 @@ public class RoutingController { private static String commonNameHashOf(ApplicationId application, SystemName system) { HashCode sha1 = Hashing.sha1().hashString(application.serializedForm(), StandardCharsets.UTF_8); String base32 = BaseEncoding.base32().omitPadding().lowerCase().encode(sha1.asBytes()); - return 'v' + base32 + Endpoint.dnsSuffix(system); + return 'v' + base32 + Endpoint.dnsSuffix(system, system.isPublic()); } /** Returns direct routing endpoints if any exist and feature flag is set for given application */ diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/TenantController.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/TenantController.java index 4b102ef3077..1ff68ae641a 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/TenantController.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/TenantController.java @@ -42,14 +42,11 @@ public class TenantController { private final Controller controller; private final CuratorDb curator; private final AccessControl accessControl; - private final BooleanFlag provisionTenantRoles; - public TenantController(Controller controller, CuratorDb curator, AccessControl accessControl, FlagSource flagSource) { this.controller = Objects.requireNonNull(controller, "controller must be non-null"); this.curator = Objects.requireNonNull(curator, "curator must be non-null"); this.accessControl = accessControl; - this.provisionTenantRoles = Flags.PROVISION_TENANT_ROLES.bindTo(flagSource); // Update serialization format of all tenants @@ -116,15 +113,11 @@ public class TenantController { TenantId.validate(tenantSpec.tenant().value()); curator.writeTenant(accessControl.createTenant(tenantSpec, controller.clock().instant(), credentials, asList())); - // Provision tenant role if enabled - if (provisionTenantRoles.with(FetchVector.Dimension.TENANT_ID, tenantSpec.tenant().value()).value()) { - try { - controller.serviceRegistry().roleService().createTenantRole(tenantSpec.tenant()); - } catch (Exception e) { - throw new RuntimeException("Unable to create tenant role for tenant: " + tenantSpec.tenant()); - } + try { + controller.serviceRegistry().roleService().createTenantRole(tenantSpec.tenant()); + } catch (Exception e) { + throw new RuntimeException("Unable to create tenant role for tenant: " + tenantSpec.tenant()); } - } } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/Deployment.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/Deployment.java index 3d17a7f8681..43ce466e8e6 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/Deployment.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/Deployment.java @@ -7,6 +7,7 @@ import com.yahoo.config.provision.zone.ZoneId; import java.time.Instant; import java.util.Objects; +import java.util.OptionalDouble; /** * A deployment of an application in a particular zone. @@ -23,9 +24,10 @@ public class Deployment { private final DeploymentMetrics metrics; private final DeploymentActivity activity; private final QuotaUsage quota; + private final OptionalDouble cost; public Deployment(ZoneId zone, ApplicationVersion applicationVersion, Version version, Instant deployTime, - DeploymentMetrics metrics, DeploymentActivity activity, QuotaUsage quota) { + DeploymentMetrics metrics, DeploymentActivity activity, QuotaUsage quota, OptionalDouble cost) { this.zone = Objects.requireNonNull(zone, "zone cannot be null"); this.applicationVersion = Objects.requireNonNull(applicationVersion, "applicationVersion cannot be null"); this.version = Objects.requireNonNull(version, "version cannot be null"); @@ -33,6 +35,7 @@ public class Deployment { this.metrics = Objects.requireNonNull(metrics, "deploymentMetrics cannot be null"); this.activity = Objects.requireNonNull(activity, "activity cannot be null"); this.quota = Objects.requireNonNull(quota, "usage cannot be null"); + this.cost = Objects.requireNonNull(cost, "cost cannot be null"); } /** Returns the zone this was deployed to */ @@ -58,17 +61,30 @@ public class Deployment { /** Returns quota usage for this */ public QuotaUsage quota() { return quota; } + /** Returns cost, in dollars per hour, for this */ + public OptionalDouble cost() { return cost; } + public Deployment recordActivityAt(Instant instant) { return new Deployment(zone, applicationVersion, version, deployTime, metrics, - activity.recordAt(instant, metrics), quota); + activity.recordAt(instant, metrics), quota, cost); } public Deployment withMetrics(DeploymentMetrics metrics) { - return new Deployment(zone, applicationVersion, version, deployTime, metrics, activity, quota); + return new Deployment(zone, applicationVersion, version, deployTime, metrics, activity, quota, cost); } public Deployment withQuota(QuotaUsage quota) { - return new Deployment(zone, applicationVersion, version, deployTime, metrics, activity, quota); + return new Deployment(zone, applicationVersion, version, deployTime, metrics, activity, quota, cost); + } + + public Deployment withCost(double cost) { + if (this.cost.isPresent() && Double.compare(this.cost.getAsDouble(), cost) == 0) return this; + return new Deployment(zone, applicationVersion, version, deployTime, metrics, activity, quota, OptionalDouble.of(cost)); + } + + public Deployment withoutCost() { + if (cost.isEmpty()) return this; + return new Deployment(zone, applicationVersion, version, deployTime, metrics, activity, quota, OptionalDouble.empty()); } @Override @@ -82,12 +98,13 @@ public class Deployment { deployTime.equals(that.deployTime) && metrics.equals(that.metrics) && activity.equals(that.activity) && - quota.equals(that.quota); + quota.equals(that.quota) && + cost.equals(that.cost); } @Override public int hashCode() { - return Objects.hash(zone, applicationVersion, version, deployTime, metrics, activity, quota); + return Objects.hash(zone, applicationVersion, version, deployTime, metrics, activity, quota, cost); } @Override diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/Endpoint.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/Endpoint.java index 3f079a5fb9b..2132f731280 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/Endpoint.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/Endpoint.java @@ -26,12 +26,10 @@ public class Endpoint { private static final String YAHOO_DNS_SUFFIX = ".vespa.yahooapis.com"; private static final String OATH_DNS_SUFFIX = ".vespa.oath.cloud"; - private static final String PUBLIC_DNS_SUFFIX = ".public.vespa.oath.cloud"; - private static final String PUBLIC_CD_DNS_SUFFIX = ".public-cd.vespa.oath.cloud"; - // TODO(mpolden): New domain is considered "legacy" for the time being, until it's ready for use. Once it's ready - // we'll make the vespa.oath.cloud variant legacy and this non-legacy. - private static final String PUBLIC_DNS_LEGACY_SUFFIX = ".vespa-app.cloud"; - private static final String PUBLIC_CD_LEGACY_DNS_SUFFIX = ".cd.vespa-app.cloud"; + private static final String PUBLIC_DNS_SUFFIX = ".vespa-app.cloud"; + private static final String PUBLIC_CD_DNS_SUFFIX = ".cd.vespa-app.cloud"; + private static final String PUBLIC_DNS_LEGACY_SUFFIX = ".public.vespa.oath.cloud"; + private static final String PUBLIC_CD_LEGACY_DNS_SUFFIX = ".public-cd.vespa.oath.cloud"; private final EndpointId id; private final ClusterSpec.Id cluster; @@ -161,11 +159,6 @@ public class Endpoint { return String.format("endpoint %s [scope=%s, legacy=%s, routingMethod=%s]", url, scope, legacy, routingMethod); } - /** Returns the DNS suffix used for endpoints in given system */ - public static String dnsSuffix(SystemName system) { - return dnsSuffix(system, false); - } - private static String endpointOrClusterAsString(EndpointId id, ClusterSpec.Id cluster) { return id == null ? cluster.value() : id.id(); } @@ -206,7 +199,7 @@ public class Endpoint { } private static String scopePart(Scope scope, List<ZoneId> zones, boolean legacy, SystemName system) { - if (system.isPublic() && legacy) { + if (system.isPublic() && !legacy) { if (scope == Scope.global) return "g"; var zone = zones.get(0); var region = zone.region().value(); @@ -218,7 +211,7 @@ public class Endpoint { var zone = zones.get(0); var region = zone.region().value(); if (scope == Scope.region) region += "-w"; - if (!legacy && zone.environment().isProduction()) return region; // Skip prod environment for non-legacy endpoints + if ((system.isPublic() || !legacy) && zone.environment().isProduction()) return region; return region + "." + zone.environment().value(); } @@ -229,11 +222,12 @@ public class Endpoint { private static String systemPart(SystemName system, String separator, boolean legacy) { if (!system.isCd()) return ""; - if (system.isPublic() && legacy) return ""; + if (system.isPublic() && !legacy) return ""; return system.value() + separator; } - private static String dnsSuffix(SystemName system, boolean legacy) { + /** Returns the DNS suffix used for endpoints in given system */ + public static String dnsSuffix(SystemName system, boolean legacy) { switch (system) { case cd: case main: diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentStatus.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentStatus.java index 5b873f11618..8981ad4e6db 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentStatus.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentStatus.java @@ -246,7 +246,8 @@ public class DeploymentStatus { existing.at(), existing.metrics(), existing.activity(), - existing.quota()) + existing.quota(), + existing.cost()) : existing); if ( job.application().instance().equals(instance) && job.type().isProduction() diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java index e9755b7fd8b..c7270b6c426 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java @@ -182,7 +182,7 @@ public class InternalStepRunner implements StepRunner { private Optional<RunStatus> deployReal(RunId id, DualLogger logger) { Versions versions = controller.jobController().run(id).get().versions(); logger.log("Deploying platform version " + versions.targetPlatform() + - " and application version " + versions.targetApplication().id() + " ..."); + " and application version " + versions.targetApplication().id() + " ..."); return deployReal(id, false, logger); } @@ -239,7 +239,7 @@ public class InternalStepRunner implements StepRunner { case ACTIVATION_CONFLICT: case APPLICATION_LOCK_FAILURE: logger.log("Deployment failed with possibly transient error " + e.code() + - ", will retry: " + e.getMessage()); + ", will retry: " + e.getMessage()); return result; case LOAD_BALANCER_NOT_READY: case PARENT_HOST_NOT_READY: @@ -725,7 +725,7 @@ public class InternalStepRunner implements StepRunner { updater.accept("invalid application configuration, or timeout of other deployments of the same application"); return; case installationFailed: - updater.accept("nodes were not able to start the new Java containers"); + updater.accept("nodes were not able to upgrade to the new configuration"); return; case testFailure: updater.accept("one or more verification tests against the deployment failed"); diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/dns/NameServiceForwarder.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/dns/NameServiceForwarder.java index d9928bffba4..f8c390edb3b 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/dns/NameServiceForwarder.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/dns/NameServiceForwarder.java @@ -28,7 +28,7 @@ import java.util.stream.Collectors; */ public class NameServiceForwarder { - private static final int maxQueuedRequests = 200; + private static final int QUEUE_CAPACITY = 300; private static final Logger log = Logger.getLogger(NameServiceForwarder.class.getName()); @@ -73,13 +73,13 @@ public class NameServiceForwarder { try (Lock lock = db.lockNameServiceQueue()) { NameServiceQueue queue = db.readNameServiceQueue(); var queued = queue.requests().size(); - if (queued >= maxQueuedRequests) { + if (queued >= QUEUE_CAPACITY) { log.log(Level.WARNING, "Queue is at capacity (size: " + queued + "), dropping older " + "requests. This likely means that the name service is not successfully " + "executing requests"); } log.log(Level.FINE, () -> "Queueing name service request: " + request); - db.writeNameServiceQueue(queue.with(request, priority).last(maxQueuedRequests)); + db.writeNameServiceQueue(queue.with(request, priority).last(QUEUE_CAPACITY)); } } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ApplicationMetaDataGarbageCollector.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ApplicationMetaDataGarbageCollector.java index 7d94a4c728f..9ec8e4d1a2d 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ApplicationMetaDataGarbageCollector.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ApplicationMetaDataGarbageCollector.java @@ -19,14 +19,14 @@ public class ApplicationMetaDataGarbageCollector extends ControllerMaintainer { } @Override - protected boolean maintain() { + protected double maintain() { try { controller().applications().applicationStore().pruneMeta(controller().clock().instant().minus(Duration.ofDays(365))); - return true; + return 1.0; } catch (Exception e) { log.log(Level.WARNING, "Exception pruning old application meta data", e); - return false; + return 0.0; } } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ApplicationOwnershipConfirmer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ApplicationOwnershipConfirmer.java index 1f20e48edf5..69e0eb26f16 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ApplicationOwnershipConfirmer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ApplicationOwnershipConfirmer.java @@ -18,6 +18,7 @@ import com.yahoo.yolean.Exceptions; import java.time.Duration; import java.util.HashMap; import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicInteger; import java.util.logging.Level; /** @@ -39,15 +40,17 @@ public class ApplicationOwnershipConfirmer extends ControllerMaintainer { } @Override - protected boolean maintain() { - return confirmApplicationOwnerships() & - ensureConfirmationResponses() & - updateConfirmedApplicationOwners(); + protected double maintain() { + return ( confirmApplicationOwnerships() + + ensureConfirmationResponses() + + updateConfirmedApplicationOwners() ) + / 3; } /** File an ownership issue with the owners of all applications we know about. */ - private boolean confirmApplicationOwnerships() { - AtomicBoolean success = new AtomicBoolean(true); + private double confirmApplicationOwnerships() { + AtomicInteger attempts = new AtomicInteger(0); + AtomicInteger failures = new AtomicInteger(0); applications() .withProjectId() .withProductionDeployment() @@ -56,6 +59,7 @@ public class ApplicationOwnershipConfirmer extends ControllerMaintainer { .filter(application -> application.createdAt().isBefore(controller().clock().instant().minus(Duration.ofDays(90)))) .forEach(application -> { try { + attempts.incrementAndGet(); // TODO jvenstad: Makes sense to require, and run this only in main? tenantOf(application.id()).contact().flatMap(contact -> { return ownershipIssues.confirmOwnership(application.ownershipIssueId(), @@ -65,17 +69,17 @@ public class ApplicationOwnershipConfirmer extends ControllerMaintainer { }).ifPresent(newIssueId -> store(newIssueId, application.id())); } catch (RuntimeException e) { // Catch errors due to wrong data in the controller, or issues client timeout. - success.set(false); + failures.incrementAndGet(); log.log(Level.INFO, "Exception caught when attempting to file an issue for '" + application.id() + "': " + Exceptions.toMessageString(e)); } }); - return success.get(); + return asSuccessFactor(attempts.get(), failures.get()); } private ApplicationSummary summaryOf(TenantAndApplicationId application) { var app = applications.requireApplication(application); var metrics = new HashMap<ZoneId, ApplicationSummary.Metric>(); - for (Instance instance : app.instances().values()) + for (Instance instance : app.instances().values()) { for (var kv : instance.deployments().entrySet()) { var zone = kv.getKey(); var deploymentMetrics = kv.getValue().metrics(); @@ -83,28 +87,31 @@ public class ApplicationOwnershipConfirmer extends ControllerMaintainer { deploymentMetrics.queriesPerSecond(), deploymentMetrics.writesPerSecond())); } + } return new ApplicationSummary(app.id().defaultInstance(), app.activity().lastQueried(), app.activity().lastWritten(), app.latestVersion().flatMap(version -> version.buildTime()), metrics); } /** Escalate ownership issues which have not been closed before a defined amount of time has passed. */ - private boolean ensureConfirmationResponses() { - AtomicBoolean success = new AtomicBoolean(true); + private double ensureConfirmationResponses() { + AtomicInteger attempts = new AtomicInteger(0); + AtomicInteger failures = new AtomicInteger(0); for (Application application : applications()) application.ownershipIssueId().ifPresent(issueId -> { try { + attempts.incrementAndGet(); Tenant tenant = tenantOf(application.id()); ownershipIssues.ensureResponse(issueId, tenant.contact()); } catch (RuntimeException e) { - success.set(false); + failures.incrementAndGet(); log.log(Level.INFO, "Exception caught when attempting to escalate issue with id '" + issueId + "': " + Exceptions.toMessageString(e)); } }); - return success.get(); + return asSuccessFactor(attempts.get(), failures.get()); } - private boolean updateConfirmedApplicationOwners() { + private double updateConfirmedApplicationOwners() { applications() .withProjectId() .withProductionDeployment() @@ -118,7 +125,7 @@ public class ApplicationOwnershipConfirmer extends ControllerMaintainer { controller().applications().store(lockedApplication.withOwner(owner))); }); }); - return true; + return 1.0; } private ApplicationList applications() { diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ArchiveAccessMaintainer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ArchiveAccessMaintainer.java index 1a9889284e1..b096a853541 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ArchiveAccessMaintainer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ArchiveAccessMaintainer.java @@ -37,8 +37,7 @@ public class ArchiveAccessMaintainer extends ControllerMaintainer { } @Override - protected boolean maintain() { - + protected double maintain() { // Count buckets - so we can alert if we get close to the account limit of 1000 zoneRegistry.zones().all().ids().forEach(zoneId -> metric.set(bucketCountMetricName, archiveBucketDb.buckets(zoneId).size(), @@ -59,6 +58,7 @@ public class ArchiveAccessMaintainer extends ControllerMaintainer { ) ); - return true; + return 1.0; } + } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ArchiveUriUpdater.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ArchiveUriUpdater.java index d2141b097b3..ab8e5efa0bd 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ArchiveUriUpdater.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ArchiveUriUpdater.java @@ -38,7 +38,7 @@ public class ArchiveUriUpdater extends ControllerMaintainer { } @Override - protected boolean maintain() { + protected double maintain() { Map<ZoneId, Set<TenantName>> tenantsByZone = new HashMap<>(); for (var application : applications.asList()) { for (var instance : application.instances().values()) { @@ -63,7 +63,7 @@ public class ArchiveUriUpdater extends ControllerMaintainer { .forEach(tenant -> nodeRepository.removeArchiveUri(zone, tenant)); }); - return true; + return 1.0; } } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ChangeRequestMaintainer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ChangeRequestMaintainer.java index 1f360c477b9..14e3e685a8a 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ChangeRequestMaintainer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ChangeRequestMaintainer.java @@ -43,14 +43,14 @@ public class ChangeRequestMaintainer extends ControllerMaintainer { @Override - protected boolean maintain() { + protected double maintain() { var currentChangeRequests = pruneOldChangeRequests(); var changeRequests = changeRequestClient.getChangeRequests(currentChangeRequests); logger.fine(() -> "Found requests: " + changeRequests); storeChangeRequests(changeRequests); - return true; + return 1.0; } private void storeChangeRequests(List<ChangeRequest> changeRequests) { diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/CloudEventReporter.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/CloudEventReporter.java index d923db936cb..5acd0c63670 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/CloudEventReporter.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/CloudEventReporter.java @@ -38,7 +38,7 @@ public class CloudEventReporter extends ControllerMaintainer { } @Override - protected boolean maintain() { + protected double maintain() { for (var region : zonesByCloudNativeRegion.keySet()) { List<CloudEvent> events = eventFetcher.getEvents(region); for (var event : events) { @@ -48,7 +48,7 @@ public class CloudEventReporter extends ControllerMaintainer { deprovisionAffectedHosts(region, event); } } - return true; + return 1.0; } /** Deprovision any host affected by given event */ diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ContactInformationMaintainer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ContactInformationMaintainer.java index 7b846fa288c..5ee39f7c8f2 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ContactInformationMaintainer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ContactInformationMaintainer.java @@ -35,12 +35,14 @@ public class ContactInformationMaintainer extends ControllerMaintainer { } @Override - protected boolean maintain() { + protected double maintain() { TenantController tenants = controller().tenants(); - boolean success = true; + int attempts = 0; + int failures = 0; for (Tenant tenant : tenants.asList()) { log.log(FINE, () -> "Updating contact information for " + tenant); try { + attempts++; switch (tenant.type()) { case athenz: tenants.lockIfPresent(tenant.name(), LockedTenant.Athenz.class, lockedTenant -> { @@ -56,13 +58,13 @@ public class ContactInformationMaintainer extends ControllerMaintainer { throw new IllegalArgumentException("Unexpected tenant type '" + tenant.type() + "'."); } } catch (Exception e) { - success = false; + failures++; log.log(Level.WARNING, "Failed to update contact information for " + tenant + ": " + Exceptions.toMessageString(e) + ". Retrying in " + interval()); } } - return success; + return asSuccessFactor(attempts, failures); } } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ContainerImageExpirer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ContainerImageExpirer.java index ff5fc4d2051..f1574381a3d 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ContainerImageExpirer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ContainerImageExpirer.java @@ -34,7 +34,7 @@ public class ContainerImageExpirer extends ControllerMaintainer { } @Override - protected boolean maintain() { + protected double maintain() { Instant now = controller().clock().instant(); VersionStatus versionStatus = controller().readVersionStatus(); List<ContainerImage> imagesToExpire = controller().serviceRegistry().containerRegistry().list().stream() @@ -44,7 +44,7 @@ public class ContainerImageExpirer extends ControllerMaintainer { log.log(Level.INFO, "Expiring " + imagesToExpire.size() + " container images: " + imagesToExpire); controller().serviceRegistry().containerRegistry().deleteAll(imagesToExpire); } - return true; + return 1.0; } /** Returns whether given image is expired */ diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintainer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintainer.java index 03a6268397e..810c412fcc0 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintainer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintainer.java @@ -34,7 +34,7 @@ public abstract class ControllerMaintainer extends Maintainer { public ControllerMaintainer(Controller controller, Duration interval, String name, Set<SystemName> activeSystems) { super(name, interval, controller.clock().instant(), controller.jobControl(), - jobMetrics(controller.metric()), controller.curator().cluster(), true); + new ControllerJobMetrics(controller.metric()), controller.curator().cluster(), true); this.controller = controller; this.activeSystems = Set.copyOf(Objects.requireNonNull(activeSystems)); } @@ -47,10 +47,20 @@ public abstract class ControllerMaintainer extends Maintainer { super.run(); } - private static JobMetrics jobMetrics(Metric metric) { - return new JobMetrics((job, consecutiveFailures) -> { - metric.set("maintenance.consecutiveFailures", consecutiveFailures, metric.createContext(Map.of("job", job))); - }); + private static class ControllerJobMetrics extends JobMetrics { + + private final Metric metric; + + public ControllerJobMetrics(Metric metric) { + this.metric = metric; + } + + @Override + protected void recordCompletion(String job, Long incompleteRuns, double successFactor) { + metric.set("maintenance.consecutiveFailures", incompleteRuns, metric.createContext(Map.of("job", job))); + metric.set("maintenance.successFactor", successFactor, metric.createContext(Map.of("job", job))); + } + } } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/CostReportMaintainer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/CostReportMaintainer.java index 28b64b5bfe0..21cda09d92a 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/CostReportMaintainer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/CostReportMaintainer.java @@ -31,10 +31,10 @@ public class CostReportMaintainer extends ControllerMaintainer { } @Override - protected boolean maintain() { + protected double maintain() { var csv = CostCalculator.resourceShareByPropertyToCsv(nodeRepository, controller(), clock, consumer.fixedAllocations()); consumer.consume(csv); - return true; + return 1.0; } } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentExpirer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentExpirer.java index e5316788802..9e3da506ca8 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentExpirer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentExpirer.java @@ -28,8 +28,9 @@ public class DeploymentExpirer extends ControllerMaintainer { } @Override - protected boolean maintain() { - boolean success = true; + protected double maintain() { + int attempts = 0; + int failures = 0; for (Application application : controller().applications().readable()) { for (Instance instance : application.instances().values()) for (Deployment deployment : instance.deployments().values()) { @@ -37,16 +38,17 @@ public class DeploymentExpirer extends ControllerMaintainer { try { log.log(Level.INFO, "Expiring deployment of " + instance.id() + " in " + deployment.zone()); + attempts++; controller().applications().deactivate(instance.id(), deployment.zone()); } catch (Exception e) { - success = false; + failures++; log.log(Level.WARNING, "Could not expire " + deployment + " of " + instance + ": " + Exceptions.toMessageString(e) + ". Retrying in " + interval()); } } } - return success; + return asSuccessFactor(attempts, failures); } /** Returns whether given deployment has expired according to its TTL */ diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentIssueReporter.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentIssueReporter.java index a3070ef55a0..4e53e07f5af 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentIssueReporter.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentIssueReporter.java @@ -21,6 +21,7 @@ import java.util.Collection; import java.util.List; import java.util.Optional; import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicInteger; import java.util.logging.Level; import static com.yahoo.vespa.hosted.controller.versions.VespaVersion.Confidence.broken; @@ -45,10 +46,11 @@ public class DeploymentIssueReporter extends ControllerMaintainer { } @Override - protected boolean maintain() { - return maintainDeploymentIssues(applications()) & - maintainPlatformIssue(applications()) & - escalateInactiveDeploymentIssues(applications()); + protected double maintain() { + return ( maintainDeploymentIssues(applications()) + + maintainPlatformIssue(applications()) + + escalateInactiveDeploymentIssues(applications())) + / 3; } /** Returns the applications to maintain issue status for. */ @@ -63,7 +65,7 @@ public class DeploymentIssueReporter extends ControllerMaintainer { * and store the issue id for the filed issues. Also, clear the issueIds of applications * where deployment has not failed for this amount of time. */ - private boolean maintainDeploymentIssues(List<Application> applications) { + private double maintainDeploymentIssues(List<Application> applications) { List<TenantAndApplicationId> failingApplications = controller().jobController().deploymentStatuses(ApplicationList.from(applications)) .failingApplicationChangeSince(controller().clock().instant().minus(maxFailureAge)) .mapToList(status -> status.application().id()); @@ -73,7 +75,7 @@ public class DeploymentIssueReporter extends ControllerMaintainer { fileDeploymentIssueFor(application); else store(application.id(), null); - return true; + return 1.0; } /** @@ -81,27 +83,26 @@ public class DeploymentIssueReporter extends ControllerMaintainer { * applications that have been failing the upgrade to the system version for * longer than the set grace period, or update this list if the issue already exists. */ - private boolean maintainPlatformIssue(List<Application> applications) { - boolean success = true; + private double maintainPlatformIssue(List<Application> applications) { if (controller().system() == SystemName.cd) - return success; + return 1.0; VersionStatus versionStatus = controller().readVersionStatus(); Version systemVersion = controller().systemVersion(versionStatus); if (versionStatus.version(systemVersion).confidence() != broken) - return success; + return 1.0; DeploymentStatusList statuses = controller().jobController().deploymentStatuses(ApplicationList.from(applications)); if (statuses.failingUpgradeToVersionSince(systemVersion, controller().clock().instant().minus(upgradeGracePeriod)).isEmpty()) - return success; + return 1.0; List<ApplicationId> failingApplications = statuses.failingUpgradeToVersionSince(systemVersion, controller().clock().instant()) .mapToList(status -> status.application().id().defaultInstance()); // TODO jonmv: Send only tenant and application, here and elsewhere in this. deploymentIssues.fileUnlessOpen(failingApplications, systemVersion); - return success; + return 1.0; } private Tenant ownerOf(TenantAndApplicationId applicationId) { @@ -126,21 +127,23 @@ public class DeploymentIssueReporter extends ControllerMaintainer { } /** Escalate issues for which there has been no activity for a certain amount of time. */ - private boolean escalateInactiveDeploymentIssues(Collection<Application> applications) { - AtomicBoolean success = new AtomicBoolean(true); + private double escalateInactiveDeploymentIssues(Collection<Application> applications) { + AtomicInteger attempts = new AtomicInteger(0); + AtomicInteger failures = new AtomicInteger(0); applications.forEach(application -> application.deploymentIssueId().ifPresent(issueId -> { try { + attempts.incrementAndGet(); Tenant tenant = ownerOf(application.id()); deploymentIssues.escalateIfInactive(issueId, maxInactivity, tenant.type() == Tenant.Type.athenz ? tenant.contact() : Optional.empty()); } catch (RuntimeException e) { - success.set(false); + failures.incrementAndGet(); log.log(Level.INFO, "Exception caught when attempting to escalate issue with id '" + issueId + "': " + Exceptions.toMessageString(e)); } })); - return success.get(); + return asSuccessFactor(attempts.get(), failures.get()); } private void store(TenantAndApplicationId id, IssueId issueId) { diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentMetricsMaintainer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentMetricsMaintainer.java index a8214ac8a09..20154c4f122 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentMetricsMaintainer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentMetricsMaintainer.java @@ -44,7 +44,7 @@ public class DeploymentMetricsMaintainer extends ControllerMaintainer { } @Override - protected boolean maintain() { + protected double maintain() { AtomicInteger failures = new AtomicInteger(0); AtomicInteger attempts = new AtomicInteger(0); AtomicReference<Exception> lastException = new AtomicReference<>(null); @@ -92,7 +92,7 @@ public class DeploymentMetricsMaintainer extends ControllerMaintainer { } catch (InterruptedException e) { throw new RuntimeException(e); } - return lastException.get() == null; + return asSuccessFactor(attempts.get(), failures.get()); } static DeploymentMetrics updateDeploymentMetrics(DeploymentMetrics current, List<ClusterMetrics> metrics) { diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/EndpointCertificateMaintainer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/EndpointCertificateMaintainer.java index 55a957f0247..85a69b0f338 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/EndpointCertificateMaintainer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/EndpointCertificateMaintainer.java @@ -54,7 +54,7 @@ public class EndpointCertificateMaintainer extends ControllerMaintainer { } @Override - protected boolean maintain() { + protected double maintain() { try { // In order of importance deployRefreshedCertificates(); @@ -62,10 +62,10 @@ public class EndpointCertificateMaintainer extends ControllerMaintainer { deleteUnusedCertificates(); } catch (Exception e) { log.log(LogLevel.ERROR, "Exception caught while maintaining endpoint certificates", e); - return false; + return 0.0; } - return true; + return 1.0; } private void updateRefreshedCertificates() { diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/HostInfoUpdater.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/HostInfoUpdater.java index 83ccda422e6..10e6f9eb039 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/HostInfoUpdater.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/HostInfoUpdater.java @@ -38,7 +38,7 @@ public class HostInfoUpdater extends ControllerMaintainer { } @Override - protected boolean maintain() { + protected double maintain() { Map<String, NodeEntity> nodeEntities = controller().serviceRegistry().entityService().listNodes().stream() .collect(Collectors.toMap(NodeEntity::hostname, Function.identity())); @@ -62,7 +62,7 @@ public class HostInfoUpdater extends ControllerMaintainer { LOG.info("Updated information for " + hostsUpdated + " hosts(s)"); } } - return true; + return 1.0; } private static Optional<String> buildModelName(NodeEntity nodeEntity) { diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/InfrastructureUpgrader.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/InfrastructureUpgrader.java index 9859d12510a..5101de73a33 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/InfrastructureUpgrader.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/InfrastructureUpgrader.java @@ -39,23 +39,28 @@ public abstract class InfrastructureUpgrader<VERSION> extends ControllerMaintain } @Override - protected boolean maintain() { - targetVersion().ifPresent(target -> upgradeAll(target, managedApplications)); - return true; + protected double maintain() { + if (targetVersion().isEmpty()) return 1.0; + return upgradeAll(targetVersion().get(), managedApplications); } /** Deploy a list of system applications until they converge on the given version */ - private void upgradeAll(VERSION target, List<SystemApplication> applications) { + private double upgradeAll(VERSION target, List<SystemApplication> applications) { + int attempts = 0; + int failures = 0; for (List<ZoneApi> zones : upgradePolicy.asList()) { boolean converged = true; for (ZoneApi zone : zones) { try { + attempts++; converged &= upgradeAll(target, applications, zone); } catch (UnreachableNodeRepositoryException e) { + failures++; converged = false; log.warning(String.format("%s: Failed to communicate with node repository in %s, continuing with next parallel zone: %s", this, zone, Exceptions.toMessageString(e))); } catch (Exception e) { + failures++; converged = false; log.warning(String.format("%s: Failed to upgrade zone: %s, continuing with next parallel zone: %s", this, zone, Exceptions.toMessageString(e))); @@ -65,6 +70,7 @@ public abstract class InfrastructureUpgrader<VERSION> extends ControllerMaintain break; } } + return asSuccessFactor(attempts, failures); } /** Returns whether all applications have converged to the target version in zone */ diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/JobRunner.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/JobRunner.java index b84cfe5af9b..25207b733f0 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/JobRunner.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/JobRunner.java @@ -49,10 +49,10 @@ public class JobRunner extends ControllerMaintainer { } @Override - protected boolean maintain() { + protected double maintain() { jobs.active().forEach(this::advance); jobs.collectGarbage(); - return true; + return 1.0; } @Override diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/MetricsReporter.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/MetricsReporter.java index b26b94f0b28..3f65c2e49cd 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/MetricsReporter.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/MetricsReporter.java @@ -73,7 +73,7 @@ public class MetricsReporter extends ControllerMaintainer { } @Override - public boolean maintain() { + public double maintain() { reportDeploymentMetrics(); reportRemainingRotations(); reportQueuedNameServiceRequests(); @@ -82,7 +82,7 @@ public class MetricsReporter extends ControllerMaintainer { reportAuditLog(); reportBrokenSystemVersion(versionStatus); reportTenantMetrics(); - return true; + return 1.0; } private void reportBrokenSystemVersion(VersionStatus versionStatus) { diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/NameServiceDispatcher.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/NameServiceDispatcher.java index e57affdc15d..fe20db00e05 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/NameServiceDispatcher.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/NameServiceDispatcher.java @@ -37,13 +37,12 @@ public class NameServiceDispatcher extends ControllerMaintainer { } @Override - protected boolean maintain() { - boolean success = true; + protected double maintain() { try (var lock = db.lockNameServiceQueue()) { var queue = db.readNameServiceQueue(); var instant = clock.instant(); var remaining = queue.dispatchTo(nameService, requestCount); - if (queue == remaining) return success; // Queue unchanged + if (queue == remaining) return 1.0; // Queue unchanged var dispatched = queue.first(requestCount); if (!dispatched.requests().isEmpty()) { @@ -54,7 +53,7 @@ public class NameServiceDispatcher extends ControllerMaintainer { } db.writeNameServiceQueue(remaining); } - return success; + return 1.0; } private static int requestCount(SystemName system) { diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/OsUpgradeScheduler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/OsUpgradeScheduler.java index e1618f05a7d..666d1c3b23a 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/OsUpgradeScheduler.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/OsUpgradeScheduler.java @@ -42,13 +42,13 @@ public class OsUpgradeScheduler extends ControllerMaintainer { } @Override - protected boolean maintain() { + protected double maintain() { for (var cloud : supportedClouds()) { Optional<Version> newTarget = newTargetIn(cloud); if (newTarget.isEmpty()) continue; controller().upgradeOsIn(cloud, newTarget.get(), upgradeBudget(), false); } - return true; + return 1.0; } /** Returns the new target version for given cloud, if any */ diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/OsVersionStatusUpdater.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/OsVersionStatusUpdater.java index cbd9207fda4..271dd277e1c 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/OsVersionStatusUpdater.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/OsVersionStatusUpdater.java @@ -18,16 +18,16 @@ public class OsVersionStatusUpdater extends ControllerMaintainer { } @Override - protected boolean maintain() { + protected double maintain() { try { OsVersionStatus newStatus = OsVersionStatus.compute(controller()); controller().updateOsVersionStatus(newStatus); - return true; + return 1.0; } catch (Exception e) { log.log(Level.WARNING, "Failed to compute OS version status: " + Exceptions.toMessageString(e) + ". Retrying in " + interval()); } - return false; + return 0.0; } } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/OutstandingChangeDeployer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/OutstandingChangeDeployer.java index a032f266de5..9d93ac719b7 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/OutstandingChangeDeployer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/OutstandingChangeDeployer.java @@ -19,13 +19,13 @@ public class OutstandingChangeDeployer extends ControllerMaintainer { } @Override - protected boolean maintain() { + protected double maintain() { for (Application application : ApplicationList.from(controller().applications().readable()) .withProductionDeployment() .withDeploymentSpec() .asList()) controller().applications().deploymentTrigger().triggerNewRevision(application.id()); - return true; + return 1.0; } } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ReadyJobsTrigger.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ReadyJobsTrigger.java index a626f21359a..ffe958cb63a 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ReadyJobsTrigger.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ReadyJobsTrigger.java @@ -17,9 +17,9 @@ public class ReadyJobsTrigger extends ControllerMaintainer { } @Override - public boolean maintain() { + public double maintain() { controller().applications().deploymentTrigger().triggerReadyJobs(); - return true; + return 1.0; } } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ReindexingTriggerer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ReindexingTriggerer.java index 263a33cf266..0bd74c844ae 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ReindexingTriggerer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ReindexingTriggerer.java @@ -40,7 +40,7 @@ public class ReindexingTriggerer extends ControllerMaintainer { } @Override - protected boolean maintain() { + protected double maintain() { try { Instant now = controller().clock().instant(); for (Application application : controller().applications().asList()) @@ -51,11 +51,11 @@ public class ReindexingTriggerer extends ControllerMaintainer { && reindexingIsReady(controller().applications().applicationReindexing(id, deployment.zone()), now)) controller().applications().reindex(id, deployment.zone(), List.of(), List.of(), true); }); - return true; + return 1.0; } catch (RuntimeException e) { log.log(Level.WARNING, "Failed to trigger reindexing: " + Exceptions.toMessageString(e)); - return false; + return 0.0; } } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ResourceMeterMaintainer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ResourceMeterMaintainer.java index b40f2232504..39ad233ce46 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ResourceMeterMaintainer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ResourceMeterMaintainer.java @@ -1,17 +1,23 @@ // Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.maintenance; -import com.yahoo.config.provision.CloudName; +import com.google.common.util.concurrent.UncheckedTimeoutException; +import com.yahoo.config.provision.ClusterResources; +import com.yahoo.config.provision.InstanceName; +import com.yahoo.config.provision.NodeResources; import com.yahoo.config.provision.SystemName; import com.yahoo.config.provision.zone.ZoneApi; import com.yahoo.config.provision.zone.ZoneId; import com.yahoo.jdisc.Metric; +import com.yahoo.vespa.hosted.controller.ApplicationController; import com.yahoo.vespa.hosted.controller.Controller; import com.yahoo.vespa.hosted.controller.api.integration.configserver.Node; import com.yahoo.vespa.hosted.controller.api.integration.configserver.NodeRepository; import com.yahoo.vespa.hosted.controller.api.integration.resource.MeteringClient; +import com.yahoo.vespa.hosted.controller.api.integration.resource.ResourceAllocation; import com.yahoo.vespa.hosted.controller.api.integration.resource.ResourceSnapshot; import com.yahoo.vespa.hosted.controller.application.SystemApplication; +import com.yahoo.vespa.hosted.controller.application.TenantAndApplicationId; import com.yahoo.vespa.hosted.controller.persistence.CuratorDb; import com.yahoo.yolean.Exceptions; @@ -20,7 +26,9 @@ import java.time.Duration; import java.time.Instant; import java.util.Collection; import java.util.Collections; +import java.util.EnumSet; import java.util.List; +import java.util.Map; import java.util.Set; import java.util.concurrent.TimeoutException; import java.util.logging.Level; @@ -33,11 +41,23 @@ import java.util.stream.Collectors; */ public class ResourceMeterMaintainer extends ControllerMaintainer { - private final Clock clock; - private final Metric metric; + /** + * Checks if the node is in some state where it is in active use by the tenant, + * and not transitioning out of use, in a failed state, etc. + */ + private static final Set<Node.State> METERABLE_NODE_STATES = EnumSet.of( + Node.State.reserved, // an application will soon use this node + Node.State.active, // an application is currently using this node + Node.State.inactive // an application is not using it, but it is reserved for being re-introduced or decommissioned + ); + + private final ApplicationController applications; private final NodeRepository nodeRepository; private final MeteringClient meteringClient; private final CuratorDb curator; + private final SystemName systemName; + private final Metric metric; + private final Clock clock; private static final String METERING_LAST_REPORTED = "metering_last_reported"; private static final String METERING_TOTAL_REPORTED = "metering_total_reported"; @@ -48,28 +68,57 @@ public class ResourceMeterMaintainer extends ControllerMaintainer { Duration interval, Metric metric, MeteringClient meteringClient) { - super(controller, interval, null, SystemName.allOf(SystemName::isPublic)); - this.clock = controller.clock(); + super(controller, interval); + this.applications = controller.applications(); this.nodeRepository = controller.serviceRegistry().configServer().nodeRepository(); - this.metric = metric; this.meteringClient = meteringClient; this.curator = controller.curator(); + this.systemName = controller.serviceRegistry().zoneRegistry().system(); + this.metric = metric; + this.clock = controller.clock(); } @Override - protected boolean maintain() { + protected double maintain() { + Collection<ResourceSnapshot> resourceSnapshots; try { - collectResourceSnapshots(); - return true; + resourceSnapshots = getAllResourceSnapshots(); } catch (Exception e) { log.log(Level.WARNING, "Failed to collect resource snapshots. Retrying in " + interval() + ". Error: " + Exceptions.toMessageString(e)); + return 0.0; + } + + if (systemName.isPublic()) reportResourceSnapshots(resourceSnapshots); + updateDeploymentCost(resourceSnapshots); + return 1.0; + } + + void updateDeploymentCost(Collection<ResourceSnapshot> resourceSnapshots) { + resourceSnapshots.stream() + .collect(Collectors.groupingBy(snapshot -> TenantAndApplicationId.from(snapshot.getApplicationId()), + Collectors.groupingBy(snapshot -> snapshot.getApplicationId().instance()))) + .forEach(this::updateDeploymentCost); + } + + private void updateDeploymentCost(TenantAndApplicationId tenantAndApplication, Map<InstanceName, List<ResourceSnapshot>> snapshotsByInstance) { + try { + applications.lockApplicationIfPresent(tenantAndApplication, locked -> { + for (InstanceName instanceName : locked.get().instances().keySet()) { + Map<ZoneId, Double> deploymentCosts = snapshotsByInstance.getOrDefault(instanceName, List.of()).stream() + .collect(Collectors.toUnmodifiableMap( + ResourceSnapshot::getZoneId, + snapshot -> cost(snapshot.allocation(), systemName))); + locked = locked.with(instanceName, i -> i.withDeploymentCosts(deploymentCosts)); + } + applications.store(locked); + }); + } catch (UncheckedTimeoutException ignored) { + // Will be retried on next maintenance, avoid throwing so we can update the other apps instead } - return false; } - private void collectResourceSnapshots() { - Collection<ResourceSnapshot> resourceSnapshots = getAllResourceSnapshots(); + private void reportResourceSnapshots(Collection<ResourceSnapshot> resourceSnapshots) { meteringClient.consume(resourceSnapshots); metric.set(METERING_LAST_REPORTED, clock.millis() / 1000, metric.createContext(Collections.emptyMap())); @@ -89,9 +138,8 @@ public class ResourceMeterMaintainer extends ControllerMaintainer { } } - private Collection<ResourceSnapshot> getAllResourceSnapshots() { + private List<ResourceSnapshot> getAllResourceSnapshots() { return controller().zoneRegistry().zones() - .ofCloud(CloudName.from("aws")) .reachable().zones().stream() .map(ZoneApi::getId) .map(zoneId -> createResourceSnapshotsFromNodes(zoneId, nodeRepository.list(zoneId, false))) @@ -103,7 +151,7 @@ public class ResourceMeterMaintainer extends ControllerMaintainer { return nodes.stream() .filter(this::unlessNodeOwnerIsSystemApplication) .filter(this::isNodeStateMeterable) - .filter(this::isNodeTypeMeterable) + .filter(this::isClusterTypeMeterable) .collect(Collectors.groupingBy(node -> node.owner().get(), Collectors.collectingAndThen(Collectors.toList(), @@ -120,21 +168,11 @@ public class ResourceMeterMaintainer extends ControllerMaintainer { .orElse(false); } - /** - * Checks if the node is in some state where it is in active use by the tenant, - * and not transitioning out of use, in a failed state, etc. - */ - private static final Set<Node.State> METERABLE_NODE_STATES = Set.of( - Node.State.reserved, // an application will soon use this node - Node.State.active, // an application is currently using this node - Node.State.inactive // an application is not using it, but it is reserved for being re-introduced or decommissioned - ); - private boolean isNodeStateMeterable(Node node) { return METERABLE_NODE_STATES.contains(node.state()); } - private boolean isNodeTypeMeterable(Node node) { + private boolean isClusterTypeMeterable(Node node) { return node.clusterType() != Node.ClusterType.admin; // log servers and shared cluster controllers } @@ -144,4 +182,15 @@ public class ResourceMeterMaintainer extends ControllerMaintainer { .isAfter(Instant.ofEpochMilli(lastRefreshTimestamp)); } + public static double cost(ClusterResources clusterResources, SystemName systemName) { + NodeResources nr = clusterResources.nodeResources(); + return cost(new ResourceAllocation(nr.vcpu(), nr.memoryGb(), nr.diskGb()).multiply(clusterResources.nodes()), systemName); + } + + private static double cost(ResourceAllocation allocation, SystemName systemName) { + // Divide cost by 3 in non-public zones to show approx. AWS equivalent cost + double costDivisor = systemName.isPublic() ? 1.0 : 3.0; + double cost = new NodeResources(allocation.getCpuCores(), allocation.getMemoryGb(), allocation.getDiskGb(), 0).cost(); + return Math.round(cost * 100.0 / costDivisor) / 100.0; + } } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ResourceTagMaintainer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ResourceTagMaintainer.java index c7bf7e765ed..ab988bcf0ac 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ResourceTagMaintainer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ResourceTagMaintainer.java @@ -28,7 +28,7 @@ public class ResourceTagMaintainer extends ControllerMaintainer { } @Override - public boolean maintain() { + public double maintain() { controller().zoneRegistry().zones() .ofCloud(CloudName.from("aws")) .reachable() @@ -38,7 +38,7 @@ public class ResourceTagMaintainer extends ControllerMaintainer { if (taggedResources > 0) log.log(Level.INFO, "Tagged " + taggedResources + " resources in " + zone.getId()); }); - return true; + return 1.0; } private Map<HostName, Optional<ApplicationId>> getTenantOfParentHosts(ZoneId zoneId) { diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/SystemRoutingPolicyMaintainer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/SystemRoutingPolicyMaintainer.java index 3b0a1fca4af..e40d772a673 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/SystemRoutingPolicyMaintainer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/SystemRoutingPolicyMaintainer.java @@ -21,14 +21,14 @@ public class SystemRoutingPolicyMaintainer extends ControllerMaintainer { } @Override - protected boolean maintain() { + protected double maintain() { for (var zone : controller().zoneRegistry().zones().all().ids()) { for (var application : SystemApplication.values()) { if (!application.hasEndpoint()) continue; controller().routing().policies().refresh(application.id(), DeploymentSpec.empty, zone); } } - return true; + return 1.0; } } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/TenantRoleMaintainer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/TenantRoleMaintainer.java index 1265d687850..637ae10bcc6 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/TenantRoleMaintainer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/TenantRoleMaintainer.java @@ -23,7 +23,7 @@ public class TenantRoleMaintainer extends ControllerMaintainer { } @Override - protected boolean maintain() { + protected double maintain() { var roleService = controller().serviceRegistry().roleService(); var tenants = controller().tenants().asList(); var tenantsWithRoles = tenants.stream() @@ -31,7 +31,7 @@ public class TenantRoleMaintainer extends ControllerMaintainer { .filter(this::hasProductionDeployment) .collect(Collectors.toList()); roleService.maintainRoles(tenantsWithRoles); - return true; + return 1.0; } private boolean hasProductionDeployment(TenantName tenant) { diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/TrafficShareUpdater.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/TrafficShareUpdater.java index fbe9faa9754..0af0d01478b 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/TrafficShareUpdater.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/TrafficShareUpdater.java @@ -36,30 +36,34 @@ public class TrafficShareUpdater extends ControllerMaintainer { } @Override - protected boolean maintain() { - boolean success = false; + protected double maintain() { Exception lastException = null; + int attempts = 0; + int failures = 0; for (var application : applications.asList()) { for (var instance : application.instances().values()) { for (var deployment : instance.deployments().values()) { if ( ! deployment.zone().environment().isProduction()) continue; try { - success |= updateTrafficFraction(instance, deployment); + attempts++; + updateTrafficFraction(instance, deployment); } catch (Exception e) { // Some failures due to locked applications are expected and benign + failures++; lastException = e; } } } } - if ( ! success && lastException != null) // log on complete failure + double successFactor = asSuccessFactor(attempts, failures); + if ( successFactor == 0 ) log.log(Level.WARNING, "Could not update traffic share on any applications", lastException); - return success; + return successFactor; } - private boolean updateTrafficFraction(Instance instance, Deployment deployment) { + private void updateTrafficFraction(Instance instance, Deployment deployment) { double qpsInZone = deployment.metrics().queriesPerSecond(); double totalQps = instance.deployments().values().stream() .filter(i -> i.zone().environment().isProduction()) @@ -73,7 +77,6 @@ public class TrafficShareUpdater extends ControllerMaintainer { maxReadShare = currentReadShare; // distribution can be incorrect nodeRepository.patchApplication(deployment.zone(), instance.id(), currentReadShare, maxReadShare); - return true; } } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/Upgrader.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/Upgrader.java index 8d5019904fa..2326f7b66ee 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/Upgrader.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/Upgrader.java @@ -51,7 +51,7 @@ public class Upgrader extends ControllerMaintainer { * Schedule application upgrades. Note that this implementation must be idempotent. */ @Override - public boolean maintain() { + public double maintain() { // Determine target versions for each upgrade policy VersionStatus versionStatus = controller().readVersionStatus(); Version canaryTarget = controller().systemVersion(versionStatus); @@ -91,7 +91,7 @@ public class Upgrader extends ControllerMaintainer { upgrade(instances.with(UpgradePolicy.canary), canaryTarget, targetMajorVersion, instances.size()); defaultTargets.forEach(target -> upgrade(instances.with(UpgradePolicy.defaultPolicy), target, targetMajorVersion, numberOfApplicationsToUpgrade())); conservativeTargets.forEach(target -> upgrade(instances.with(UpgradePolicy.conservative), target, targetMajorVersion, numberOfApplicationsToUpgrade())); - return true; + return 1.0; } /** Returns the target versions for given confidence, one per major version in the system */ diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/VCMRMaintainer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/VCMRMaintainer.java index fedf3d90760..4cd24289676 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/VCMRMaintainer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/VCMRMaintainer.java @@ -57,7 +57,7 @@ public class VCMRMaintainer extends ControllerMaintainer { } @Override - protected boolean maintain() { + protected double maintain() { var changeRequests = curator.readChangeRequests() .stream() .filter(shouldUpdate()) @@ -81,7 +81,7 @@ public class VCMRMaintainer extends ControllerMaintainer { }); } }); - return true; + return 1.0; } /** diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/VersionStatusUpdater.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/VersionStatusUpdater.java index a3e9672b715..e4866c43f13 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/VersionStatusUpdater.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/VersionStatusUpdater.java @@ -29,7 +29,7 @@ public class VersionStatusUpdater extends ControllerMaintainer { } @Override - protected boolean maintain() { + protected double maintain() { try { VersionStatus newStatus = VersionStatus.compute(controller()); controller().updateVersionStatus(newStatus); @@ -37,12 +37,12 @@ public class VersionStatusUpdater extends ControllerMaintainer { controller().serviceRegistry().systemMonitor().reportSystemVersion(version.versionNumber(), convert(version.confidence())); }); - return true; + return 1.0; } catch (Exception e) { log.log(Level.WARNING, "Failed to compute version status: " + Exceptions.toMessageString(e) + ". Retrying in " + interval()); } - return false; + return 0.0; } static SystemMonitor.Confidence convert(VespaVersion.Confidence confidence) { diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializer.java index 24b553e5153..06442779b9c 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializer.java @@ -142,6 +142,8 @@ public class ApplicationSerializer { // Quota usage fields private static final String quotaUsageRateField = "quotaUsageRate"; + private static final String deploymentCostField = "cost"; + // ------------------ Serialization public Slime toSlime(Application application) { @@ -196,6 +198,7 @@ public class ApplicationSerializer { deployment.activity().lastQueriesPerSecond().ifPresent(value -> object.setDouble(lastQueriesPerSecondField, value)); deployment.activity().lastWritesPerSecond().ifPresent(value -> object.setDouble(lastWritesPerSecondField, value)); object.setDouble(quotaUsageRateField, deployment.quota().rate()); + deployment.cost().ifPresent(cost -> object.setDouble(deploymentCostField, cost)); } private void deploymentMetricsToSlime(DeploymentMetrics metrics, Cursor object) { @@ -357,7 +360,8 @@ public class ApplicationSerializer { Serializers.optionalInstant(deploymentObject.field(lastWrittenField)), Serializers.optionalDouble(deploymentObject.field(lastQueriesPerSecondField)), Serializers.optionalDouble(deploymentObject.field(lastWritesPerSecondField))), - QuotaUsage.create(Serializers.optionalDouble(deploymentObject.field(quotaUsageRateField)))); + QuotaUsage.create(Serializers.optionalDouble(deploymentObject.field(quotaUsageRateField))), + Serializers.optionalDouble(deploymentObject.field(deploymentCostField))); } private DeploymentMetrics deploymentMetricsFromSlime(Inspector object) { diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/RunSerializer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/RunSerializer.java index 91d85e62fcb..60d8afe0f5e 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/RunSerializer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/RunSerializer.java @@ -95,7 +95,6 @@ class RunSerializer { private static final String lastTestRecordField = "lastTestRecord"; private static final String lastVespaLogTimestampField = "lastVespaLogTimestamp"; private static final String noNodesDownSinceField = "noNodesDownSince"; - private static final String oldConvergenceSummaryField = "convergenceSummary"; // TODO (freva): Remove after 7.410 private static final String convergenceSummaryField = "convergenceSummaryV2"; private static final String testerCertificateField = "testerCertificate"; @@ -137,8 +136,7 @@ class RunSerializer { runObject.field(lastTestRecordField).asLong(), Instant.EPOCH.plus(runObject.field(lastVespaLogTimestampField).asLong(), ChronoUnit.MICROS), Serializers.optionalInstant(runObject.field(noNodesDownSinceField)), - convergenceSummaryFrom(runObject.field(convergenceSummaryField)) - .or(() ->convergenceSummaryFrom(runObject.field(oldConvergenceSummaryField))), + convergenceSummaryFrom(runObject.field(convergenceSummaryField)), Optional.of(runObject.field(testerCertificateField)) .filter(Inspector::valid) .map(certificate -> X509CertificateUtils.fromPem(certificate.asString()))); @@ -223,10 +221,7 @@ class RunSerializer { runObject.setLong(lastTestRecordField, run.lastTestLogEntry()); runObject.setLong(lastVespaLogTimestampField, Instant.EPOCH.until(run.lastVespaLogTimestamp(), ChronoUnit.MICROS)); run.noNodesDownSince().ifPresent(noNodesDownSince -> runObject.setLong(noNodesDownSinceField, noNodesDownSince.toEpochMilli())); - run.convergenceSummary().ifPresent(convergenceSummary -> { - toSlime(convergenceSummary, runObject.setArray(convergenceSummaryField), false); - toSlime(convergenceSummary, runObject.setArray(oldConvergenceSummaryField), true); - }); + run.convergenceSummary().ifPresent(convergenceSummary -> toSlime(convergenceSummary, runObject.setArray(convergenceSummaryField))); run.testerCertificate().ifPresent(certificate -> runObject.setString(testerCertificateField, X509CertificateUtils.toPem(certificate))); Cursor stepsObject = runObject.setObject(stepsField); @@ -263,7 +258,7 @@ class RunSerializer { } // Don't change this - introduce a separate array with new values if needed. - private void toSlime(ConvergenceSummary summary, Cursor summaryArray, boolean oldFormat) { + private void toSlime(ConvergenceSummary summary, Cursor summaryArray) { summaryArray.addLong(summary.nodes()); summaryArray.addLong(summary.down()); summaryArray.addLong(summary.upgradingOs()); @@ -276,8 +271,7 @@ class RunSerializer { summaryArray.addLong(summary.restarting()); summaryArray.addLong(summary.services()); summaryArray.addLong(summary.needNewConfig()); - if (!oldFormat) - summaryArray.addLong(summary.retiring()); + summaryArray.addLong(summary.retiring()); } static String valueOf(Step step) { diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java index 7dff745bda5..d1cbe8e14b4 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java @@ -31,6 +31,7 @@ import com.yahoo.restapi.Path; import com.yahoo.restapi.ResourceResponse; import com.yahoo.restapi.SlimeJsonResponse; import com.yahoo.security.KeyUtils; +import com.yahoo.security.X509CertificateUtils; import com.yahoo.slime.Cursor; import com.yahoo.slime.Inspector; import com.yahoo.slime.JsonParseException; @@ -88,6 +89,7 @@ import com.yahoo.vespa.hosted.controller.deployment.DeploymentTrigger.ChangesToC import com.yahoo.vespa.hosted.controller.deployment.JobStatus; import com.yahoo.vespa.hosted.controller.deployment.Run; import com.yahoo.vespa.hosted.controller.deployment.TestConfigSerializer; +import com.yahoo.vespa.hosted.controller.maintenance.ResourceMeterMaintainer; import com.yahoo.vespa.hosted.controller.notification.Notification; import com.yahoo.vespa.hosted.controller.notification.NotificationSource; import com.yahoo.vespa.hosted.controller.persistence.SupportAccessSerializer; @@ -121,6 +123,7 @@ import java.net.URISyntaxException; import java.security.DigestInputStream; import java.security.Principal; import java.security.PublicKey; +import java.security.cert.X509Certificate; import java.time.DayOfWeek; import java.time.Duration; import java.time.Instant; @@ -898,6 +901,7 @@ public class ApplicationApiHandler extends AuditLoggingRequestHandler { cluster.suggested().ifPresent(suggested -> toSlime(suggested, clusterObject.setObject("suggested"))); utilizationToSlime(cluster.utilization(), clusterObject.setObject("utilization")); scalingEventsToSlime(cluster.scalingEvents(), clusterObject.setArray("scalingEvents")); + clusterObject.setString("autoscalingStatusCode", cluster.autoscalingStatusCode()); clusterObject.setString("autoscalingStatus", cluster.autoscalingStatus()); clusterObject.setLong("scalingDuration", cluster.scalingDuration().toMillis()); clusterObject.setDouble("maxQueryGrowthRate", cluster.maxQueryGrowthRate()); @@ -968,7 +972,9 @@ public class ApplicationApiHandler extends AuditLoggingRequestHandler { return new HttpResponse(200) { @Override public void render(OutputStream outputStream) throws IOException { - logStream.transferTo(outputStream); + try (logStream) { + logStream.transferTo(outputStream); + } } }; } @@ -976,7 +982,7 @@ public class ApplicationApiHandler extends AuditLoggingRequestHandler { private HttpResponse supportAccess(String tenantName, String applicationName, String instanceName, String environment, String region, Map<String, String> queryParameters) { DeploymentId deployment = new DeploymentId(ApplicationId.from(tenantName, applicationName, instanceName), requireZone(environment, region)); SupportAccess supportAccess = controller.supportAccess().forDeployment(deployment); - return new SlimeJsonResponse(SupportAccessSerializer.toSlime(supportAccess, false, Optional.ofNullable(controller.clock().instant()))); + return new SlimeJsonResponse(SupportAccessSerializer.toSlime(supportAccess, false, Optional.of(controller.clock().instant()))); } // TODO support access: only let tenants (not operators!) allow access @@ -984,15 +990,16 @@ public class ApplicationApiHandler extends AuditLoggingRequestHandler { private HttpResponse allowSupportAccess(String tenantName, String applicationName, String instanceName, String environment, String region, HttpRequest request) { DeploymentId deployment = new DeploymentId(ApplicationId.from(tenantName, applicationName, instanceName), requireZone(environment, region)); Principal principal = requireUserPrincipal(request); - SupportAccess allowed = controller.supportAccess().allow(deployment, Instant.now().plus(7, ChronoUnit.DAYS), principal.getName()); - return new SlimeJsonResponse(SupportAccessSerializer.toSlime(allowed, false, Optional.ofNullable(controller.clock().instant()))); + Instant now = controller.clock().instant(); + SupportAccess allowed = controller.supportAccess().allow(deployment, now.plus(7, ChronoUnit.DAYS), principal.getName()); + return new SlimeJsonResponse(SupportAccessSerializer.toSlime(allowed, false, Optional.of(now))); } private HttpResponse disallowSupportAccess(String tenantName, String applicationName, String instanceName, String environment, String region, HttpRequest request) { DeploymentId deployment = new DeploymentId(ApplicationId.from(tenantName, applicationName, instanceName), requireZone(environment, region)); Principal principal = requireUserPrincipal(request); SupportAccess disallowed = controller.supportAccess().disallow(deployment, principal.getName()); - return new SlimeJsonResponse(SupportAccessSerializer.toSlime(disallowed, false, Optional.ofNullable(controller.clock().instant()))); + return new SlimeJsonResponse(SupportAccessSerializer.toSlime(disallowed, false, Optional.of(controller.clock().instant()))); } private HttpResponse metrics(String tenantName, String applicationName, String instanceName, String environment, String region) { @@ -1400,6 +1407,9 @@ public class ApplicationApiHandler extends AuditLoggingRequestHandler { }); } + response.setDouble("quota", deployment.quota().rate()); + deployment.cost().ifPresent(cost -> response.setDouble("cost", cost)); + controller.archiveBucketDb().archiveUriFor(deploymentId.zoneId(), deploymentId.applicationId().tenant()) .ifPresent(archiveUri -> response.setString("archiveUri", archiveUri.toString())); @@ -2112,9 +2122,8 @@ public class ApplicationApiHandler extends AuditLoggingRequestHandler { object.setLong("groups", resources.groups()); toSlime(resources.nodeResources(), object.setObject("nodeResources")); - // Divide cost by 3 in non-public zones to show approx. AWS equivalent cost - double costDivisor = controller.zoneRegistry().system().isPublic() ? 1.0 : 3.0; - object.setDouble("cost", Math.round(resources.nodes() * resources.nodeResources().cost() * 100.0 / costDivisor) / 100.0); + double cost = ResourceMeterMaintainer.cost(resources, controller.serviceRegistry().zoneRegistry().system()); + object.setDouble("cost", cost); } private void utilizationToSlime(Cluster.Utilization utilization, Cursor utilizationObject) { diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/controller/AccessRequestResponse.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/controller/AccessRequestResponse.java new file mode 100644 index 00000000000..e17421764e5 --- /dev/null +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/controller/AccessRequestResponse.java @@ -0,0 +1,28 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +package com.yahoo.vespa.hosted.controller.restapi.controller; + +import com.yahoo.restapi.SlimeJsonResponse; +import com.yahoo.slime.Cursor; +import com.yahoo.slime.Slime; +import com.yahoo.vespa.athenz.api.AthenzIdentity; +import com.yahoo.vespa.athenz.api.AthenzUser; + +import java.util.Collection; + +public class AccessRequestResponse extends SlimeJsonResponse { + + public AccessRequestResponse(Collection<AthenzUser> members) { + super(toSlime(members)); + } + + private static Slime toSlime(Collection<AthenzUser> members) { + Slime slime = new Slime(); + Cursor root = slime.setObject(); + Cursor array = root.setArray("members"); + members.stream() + .map(AthenzIdentity::getFullName) + .forEach(array::addString); + return slime; + } +} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/controller/ControllerApiHandler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/controller/ControllerApiHandler.java index 98a9ade1b16..cba89fe39cf 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/controller/ControllerApiHandler.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/controller/ControllerApiHandler.java @@ -2,27 +2,41 @@ package com.yahoo.vespa.hosted.controller.restapi.controller; import com.yahoo.component.Version; +import com.yahoo.config.provision.ApplicationId; +import com.yahoo.config.provision.Zone; +import com.yahoo.config.provision.zone.ZoneId; import com.yahoo.container.jdisc.HttpRequest; import com.yahoo.container.jdisc.HttpResponse; import com.yahoo.container.jdisc.LoggingRequestHandler; import com.yahoo.io.IOUtils; import com.yahoo.restapi.ErrorResponse; +import com.yahoo.restapi.MessageResponse; import com.yahoo.restapi.Path; import com.yahoo.restapi.ResourceResponse; +import com.yahoo.security.X509CertificateUtils; import com.yahoo.slime.Inspector; import com.yahoo.slime.SlimeUtils; +import com.yahoo.vespa.athenz.api.AthenzUser; import com.yahoo.vespa.hosted.controller.Controller; +import com.yahoo.vespa.hosted.controller.api.identifiers.DeploymentId; +import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType; import com.yahoo.vespa.hosted.controller.auditlog.AuditLoggingRequestHandler; import com.yahoo.vespa.hosted.controller.maintenance.ControllerMaintenance; import com.yahoo.vespa.hosted.controller.maintenance.Upgrader; +import com.yahoo.vespa.hosted.controller.support.access.SupportAccess; import com.yahoo.vespa.hosted.controller.versions.VespaVersion.Confidence; import com.yahoo.yolean.Exceptions; +import javax.ws.rs.InternalServerErrorException; import java.io.IOException; import java.io.InputStream; import java.io.UncheckedIOException; +import java.security.Principal; +import java.security.cert.X509Certificate; +import java.time.Instant; import java.util.Optional; import java.util.Scanner; +import java.util.function.Function; import java.util.logging.Level; /** @@ -77,9 +91,53 @@ public class ControllerApiHandler extends AuditLoggingRequestHandler { private HttpResponse post(HttpRequest request) { Path path = new Path(request.getUri()); if (path.matches("/controller/v1/jobs/upgrader/confidence/{version}")) return overrideConfidence(request, path.get("version")); + if (path.matches("/controller/v1/access/requests/{user}")) return approveMembership(request, path.get("user")); + if (path.matches("/controller/v1/access/grants/{user}")) return grantAccess(request, path.get("user")); return notFound(path); } + private HttpResponse approveMembership(HttpRequest request, String user) { + AthenzUser athenzUser = AthenzUser.fromUserId(user); + byte[] jsonBytes = toJsonBytes(request.getData()); + Inspector inspector = SlimeUtils.jsonToSlime(jsonBytes).get(); + ApplicationId applicationId = requireField(inspector, "applicationId", ApplicationId::fromSerializedForm); + ZoneId zone = requireField(inspector, "zone", ZoneId::from); + if(controller.supportAccess().allowDataplaneMembership(athenzUser, new DeploymentId(applicationId, zone))) { + return new AccessRequestResponse(controller.serviceRegistry().accessControlService().listMembers()); + } else { + return new MessageResponse(400, "Unable to approve membership request"); + } + } + + private HttpResponse grantAccess(HttpRequest request, String user) { + Principal principal = requireUserPrincipal(request); + Instant now = controller.clock().instant(); + + byte[] jsonBytes = toJsonBytes(request.getData()); + Inspector requestObject = SlimeUtils.jsonToSlime(jsonBytes).get(); + X509Certificate certificate = requireField(requestObject, "certificate", X509CertificateUtils::fromPem); + ApplicationId applicationId = requireField(requestObject, "applicationId", ApplicationId::fromSerializedForm); + ZoneId zone = requireField(requestObject, "zone", ZoneId::from); + DeploymentId deployment = new DeploymentId(applicationId, zone); + + // Register grant + SupportAccess supportAccess = controller.supportAccess().registerGrant(deployment, principal.getName(), certificate); + + // Trigger deployment to include operator cert + JobType jobType = JobType.from(controller.system(), deployment.zoneId()) + .orElseThrow(() -> new IllegalStateException("No job found to trigger for " + deployment.toUserFriendlyString())); + + String jobName = controller.applications().deploymentTrigger() + .reTrigger(deployment.applicationId(), jobType).type().jobName(); + return new MessageResponse(String.format("Operator %s granted access and job %s triggered", principal.getName(), jobName)); + } + + private <T> T requireField(Inspector inspector, String field, Function<String, T> mapper) { + return SlimeUtils.optionalString(inspector.field(field)) + .map(mapper::apply) + .orElseThrow(() -> new IllegalArgumentException("Expected field \"" + field + "\" in request")); + } + private HttpResponse delete(HttpRequest request) { Path path = new Path(request.getUri()); if (path.matches("/controller/v1/jobs/upgrader/confidence/{version}")) return removeConfidenceOverride(path.get("version")); @@ -145,4 +203,9 @@ public class ControllerApiHandler extends AuditLoggingRequestHandler { } } + private static Principal requireUserPrincipal(HttpRequest request) { + Principal principal = request.getJDiscRequest().getUserPrincipal(); + if (principal == null) throw new InternalServerErrorException("Expected a user principal"); + return principal; + } } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/support/access/SupportAccessControl.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/support/access/SupportAccessControl.java index 4a550ad3379..6bbec918ba9 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/support/access/SupportAccessControl.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/support/access/SupportAccessControl.java @@ -1,16 +1,20 @@ // Copyright 2021 Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.support.access; +import com.yahoo.vespa.athenz.api.AthenzIdentity; +import com.yahoo.vespa.athenz.api.AthenzUser; import com.yahoo.vespa.curator.Lock; import com.yahoo.vespa.hosted.controller.Controller; import com.yahoo.vespa.hosted.controller.api.identifiers.DeploymentId; +import java.security.Principal; import java.security.cert.X509Certificate; import java.time.Instant; import java.time.Period; import java.util.List; import java.util.stream.Collectors; +import static com.yahoo.vespa.hosted.controller.support.access.SupportAccess.State.ALLOWED; import static com.yahoo.vespa.hosted.controller.support.access.SupportAccess.State.NOT_ALLOWED; /** @@ -83,8 +87,19 @@ public class SupportAccessControl { if (supportAccess.currentStatus(now).state() == NOT_ALLOWED) return List.of(); return supportAccess.grantHistory().stream() - .filter(grant -> !grant.certificate().getNotBefore().toInstant().isBefore(now)) - .filter(grant -> !grant.certificate().getNotAfter().toInstant().isAfter(now)) + .filter(grant -> now.isAfter(grant.certificate().getNotBefore().toInstant())) + .filter(grant -> now.isBefore(grant.certificate().getNotAfter().toInstant())) .collect(Collectors.toUnmodifiableList()); } + + public boolean allowDataplaneMembership(AthenzUser identity, DeploymentId deploymentId) { + Instant instant = controller.clock().instant(); + SupportAccess supportAccess = forDeployment(deploymentId); + SupportAccess.CurrentStatus currentStatus = supportAccess.currentStatus(instant); + if(currentStatus.state() == ALLOWED) { + return controller.serviceRegistry().accessControlService().approveDataPlaneAccess(identity, currentStatus.allowedUntil().orElse(instant.plus(MAX_SUPPORT_ACCESS_TIME))); + } else { + return false; + } + } } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/VersionStatus.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/VersionStatus.java index 625154693da..c45cea29c33 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/VersionStatus.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/VersionStatus.java @@ -229,7 +229,9 @@ public class VersionStatus { // Keep existing confidence if we cannot raise it at this moment in time if (!confidenceIsOverridden && - !existingVespaVersion.confidence().canChangeTo(confidence, controller.clock().instant())) { + !existingVespaVersion.confidence().canChangeTo(confidence, + controller.serviceRegistry().zoneRegistry().system(), + controller.clock().instant())) { confidence = existingVespaVersion.confidence(); } } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/VespaVersion.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/VespaVersion.java index 593b1438856..3510a4f78cd 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/VespaVersion.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/VespaVersion.java @@ -2,10 +2,10 @@ package com.yahoo.vespa.hosted.controller.versions; import com.yahoo.component.Version; +import com.yahoo.config.provision.SystemName; import com.yahoo.vespa.hosted.controller.Controller; import com.yahoo.vespa.hosted.controller.application.ApplicationList; import com.yahoo.vespa.hosted.controller.application.InstanceList; -import com.yahoo.vespa.hosted.controller.deployment.DeploymentStatus; import java.time.Instant; import java.time.ZoneOffset; @@ -147,14 +147,15 @@ public class VespaVersion implements Comparable<VespaVersion> { } /** Returns true if this can be changed to target at given instant */ - public boolean canChangeTo(Confidence target, Instant instant) { + public boolean canChangeTo(Confidence target, SystemName system, Instant instant) { if (this.equalOrHigherThan(normal)) return true; // Confidence can always change from >= normal if (!target.equalOrHigherThan(normal)) return true; // Confidence can always change to < normal var hourOfDay = instant.atZone(ZoneOffset.UTC).getHour(); var dayOfWeek = instant.atZone(ZoneOffset.UTC).getDayOfWeek(); - // Confidence can only be raised between 05:00:00 and 11:59:59 UTC, and not during weekends or Friday. - return hourOfDay >= 5 && hourOfDay <= 11 + var hourEnd = system == SystemName.Public ? 13 : 11; + // Confidence can only be raised between 05:00:00 and 11:59:59Z (13:59:59Z for public), and not during weekends or Friday. + return hourOfDay >= 5 && hourOfDay <= hourEnd && dayOfWeek.getValue() < 5; } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/application/EndpointTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/application/EndpointTest.java index 468c92d3539..da641d17a8a 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/application/EndpointTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/application/EndpointTest.java @@ -64,24 +64,28 @@ public class EndpointTest { "https://r2.i2.a2.t2.global.vespa.oath.cloud/", Endpoint.of(app2).target(EndpointId.of("r2")).on(Port.tls()).routingMethod(RoutingMethod.exclusive).in(SystemName.main), - // Main endpoint in public system + // Main endpoint in public system (legacy) "https://a1.t1.global.public.vespa.oath.cloud/", - Endpoint.of(app1).target(endpointId).on(Port.tls()).routingMethod(RoutingMethod.exclusive).in(SystemName.Public) + Endpoint.of(app1).target(endpointId).on(Port.tls()).routingMethod(RoutingMethod.exclusive).legacy().in(SystemName.Public) ); tests.forEach((expected, endpoint) -> assertEquals(expected, endpoint.url().toString())); Map<String, Endpoint> tests2 = Map.of( - // Default endpoint in public system using new domain + // Main endpoint in public CD system (legacy) + "https://publiccd.a1.t1.global.public-cd.vespa.oath.cloud/", + Endpoint.of(app1).target(endpointId).on(Port.tls()).routingMethod(RoutingMethod.exclusive).legacy().in(SystemName.PublicCd), + + // Default endpoint in public system "https://a1.t1.g.vespa-app.cloud/", - Endpoint.of(app1).target(endpointId).on(Port.tls()).routingMethod(RoutingMethod.exclusive).legacy().in(SystemName.Public), + Endpoint.of(app1).target(endpointId).on(Port.tls()).routingMethod(RoutingMethod.exclusive).in(SystemName.Public), - // Default endpoint in public CD system using new domain + // Default endpoint in public CD system "https://a1.t1.g.cd.vespa-app.cloud/", - Endpoint.of(app1).target(endpointId).on(Port.tls()).routingMethod(RoutingMethod.exclusive).legacy().in(SystemName.PublicCd), + Endpoint.of(app1).target(endpointId).on(Port.tls()).routingMethod(RoutingMethod.exclusive).in(SystemName.PublicCd), - // Custom instance in public system, using new domain + // Custom instance in public system "https://i2.a2.t2.g.vespa-app.cloud/", - Endpoint.of(app2).target(endpointId).on(Port.tls()).routingMethod(RoutingMethod.exclusive).legacy().in(SystemName.Public) + Endpoint.of(app2).target(endpointId).on(Port.tls()).routingMethod(RoutingMethod.exclusive).in(SystemName.Public) ); tests2.forEach((expected, endpoint) -> assertEquals(expected, endpoint.url().toString())); } @@ -127,16 +131,20 @@ public class EndpointTest { "https://r2.i2.a2.t2.global.vespa.oath.cloud/", Endpoint.of(app2).target(EndpointId.of("r2")).on(Port.tls()).routingMethod(RoutingMethod.exclusive).in(SystemName.main), - // Main endpoint in public system + // Main endpoint in public system (legacy) "https://a1.t1.global.public.vespa.oath.cloud/", - Endpoint.of(app1).target(endpointId).on(Port.tls()).routingMethod(RoutingMethod.exclusive).in(SystemName.Public) + Endpoint.of(app1).target(endpointId).on(Port.tls()).routingMethod(RoutingMethod.exclusive).legacy().in(SystemName.Public) ); tests.forEach((expected, endpoint) -> assertEquals(expected, endpoint.url().toString())); Map<String, Endpoint> tests2 = Map.of( - // Custom endpoint and instance in public system, using new domain + // Custom endpoint and instance in public CD system (legacy) + "https://foo.publiccd.i2.a2.t2.global.public-cd.vespa.oath.cloud/", + Endpoint.of(app2).target(EndpointId.of("foo")).on(Port.tls()).routingMethod(RoutingMethod.exclusive).legacy().in(SystemName.PublicCd), + + // Custom endpoint and instance in public system "https://foo.i2.a2.t2.g.vespa-app.cloud/", - Endpoint.of(app2).target(EndpointId.of("foo")).on(Port.tls()).routingMethod(RoutingMethod.exclusive).legacy().in(SystemName.Public) + Endpoint.of(app2).target(EndpointId.of("foo")).on(Port.tls()).routingMethod(RoutingMethod.exclusive).in(SystemName.Public) ); tests2.forEach((expected, endpoint) -> assertEquals(expected, endpoint.url().toString())); } @@ -176,13 +184,13 @@ public class EndpointTest { "https://i2--a2--t2.us-north-1.vespa.oath.cloud:4443/", Endpoint.of(app2).target(cluster, prodZone).on(Port.tls(4443)).in(SystemName.main), - // Non-default cluster in public + // Non-default cluster in public (legacy) "https://c1.a1.t1.us-north-1.public.vespa.oath.cloud/", - Endpoint.of(app1).target(ClusterSpec.Id.from("c1"), prodZone).on(Port.tls()).routingMethod(RoutingMethod.exclusive).in(SystemName.Public), + Endpoint.of(app1).target(ClusterSpec.Id.from("c1"), prodZone).on(Port.tls()).routingMethod(RoutingMethod.exclusive).legacy().in(SystemName.Public), - // Non-default cluster and instance in public + // Non-default cluster and instance in public (legacy) "https://c2.i2.a2.t2.us-north-1.public.vespa.oath.cloud/", - Endpoint.of(app2).target(ClusterSpec.Id.from("c2"), prodZone).on(Port.tls()).routingMethod(RoutingMethod.exclusive).in(SystemName.Public), + Endpoint.of(app2).target(ClusterSpec.Id.from("c2"), prodZone).on(Port.tls()).routingMethod(RoutingMethod.exclusive).legacy().in(SystemName.Public), // Endpoint in main using shared layer 4 "https://a1.t1.us-north-1.vespa.oath.cloud/", @@ -191,17 +199,21 @@ public class EndpointTest { tests.forEach((expected, endpoint) -> assertEquals(expected, endpoint.url().toString())); Map<String, Endpoint> tests2 = Map.of( - // Custom cluster name in public, using new domain + // Non-default cluster and instance in public CD (legacy) + "https://c2.publiccd.i2.a2.t2.us-north-1.public-cd.vespa.oath.cloud/", + Endpoint.of(app2).target(ClusterSpec.Id.from("c2"), prodZone).on(Port.tls()).routingMethod(RoutingMethod.exclusive).legacy().in(SystemName.PublicCd), + + // Custom cluster name in public "https://c1.a1.t1.us-north-1.z.vespa-app.cloud/", - Endpoint.of(app1).target(ClusterSpec.Id.from("c1"), prodZone).on(Port.tls()).routingMethod(RoutingMethod.exclusive).legacy().in(SystemName.Public), + Endpoint.of(app1).target(ClusterSpec.Id.from("c1"), prodZone).on(Port.tls()).routingMethod(RoutingMethod.exclusive).in(SystemName.Public), - // Default cluster name in non-production zone in public, using new domain + // Default cluster name in non-production zone in public "https://a1.t1.us-north-2.test.z.vespa-app.cloud/", - Endpoint.of(app1).target(ClusterSpec.Id.from("default"), testZone).on(Port.tls()).routingMethod(RoutingMethod.exclusive).legacy().in(SystemName.Public), + Endpoint.of(app1).target(ClusterSpec.Id.from("default"), testZone).on(Port.tls()).routingMethod(RoutingMethod.exclusive).in(SystemName.Public), - // Default cluster name in public CD, using new domain + // Default cluster name in public CD "https://a1.t1.us-north-1.z.cd.vespa-app.cloud/", - Endpoint.of(app1).target(ClusterSpec.Id.from("default"), prodZone).on(Port.tls()).routingMethod(RoutingMethod.exclusive).legacy().in(SystemName.PublicCd) + Endpoint.of(app1).target(ClusterSpec.Id.from("default"), prodZone).on(Port.tls()).routingMethod(RoutingMethod.exclusive).in(SystemName.PublicCd) ); tests2.forEach((expected, endpoint) -> assertEquals(expected, endpoint.url().toString())); } @@ -219,6 +231,7 @@ public class EndpointTest { .target(EndpointId.defaultId()) .routingMethod(RoutingMethod.exclusive) .on(Port.tls()) + .legacy() .in(SystemName.Public), // Wildcard to match other rotations @@ -227,6 +240,7 @@ public class EndpointTest { .wildcard() .routingMethod(RoutingMethod.exclusive) .on(Port.tls()) + .legacy() .in(SystemName.Public), // Default cluster in zone @@ -235,6 +249,7 @@ public class EndpointTest { .target(defaultCluster, prodZone) .routingMethod(RoutingMethod.exclusive) .on(Port.tls()) + .legacy() .in(SystemName.Public), // Wildcard to match other clusters in zone @@ -243,6 +258,7 @@ public class EndpointTest { .wildcard(prodZone) .routingMethod(RoutingMethod.exclusive) .on(Port.tls()) + .legacy() .in(SystemName.Public), // Default cluster in test zone @@ -251,6 +267,7 @@ public class EndpointTest { .target(defaultCluster, testZone) .routingMethod(RoutingMethod.exclusive) .on(Port.tls()) + .legacy() .in(SystemName.Public), // Wildcard to match other clusters in test zone @@ -259,6 +276,15 @@ public class EndpointTest { .wildcard(testZone) .routingMethod(RoutingMethod.exclusive) .on(Port.tls()) + .legacy() + .in(SystemName.Public), + + // Wildcard to match other clusters in zone + "https://*.a1.t1.us-north-1.z.vespa-app.cloud/", + Endpoint.of(app1) + .wildcard(prodZone) + .routingMethod(RoutingMethod.exclusive) + .on(Port.tls()) .in(SystemName.Public) ); @@ -275,25 +301,27 @@ public class EndpointTest { .targetRegion(cluster, ZoneId.from("prod", "us-north-1a")) .routingMethod(RoutingMethod.exclusive) .on(Port.tls()) + .legacy() .in(SystemName.Public), "https://a1.t1.us-north-2-w.public.vespa.oath.cloud/", Endpoint.of(app1) .targetRegion(cluster, prodZone) .routingMethod(RoutingMethod.exclusive) .on(Port.tls()) + .legacy() .in(SystemName.Public), "https://a1.t1.us-north-2-w.test.public.vespa.oath.cloud/", Endpoint.of(app1) .targetRegion(cluster, ZoneId.from("test", "us-north-2")) .routingMethod(RoutingMethod.exclusive) .on(Port.tls()) + .legacy() .in(SystemName.Public), "https://c1.a1.t1.us-north-2.r.vespa-app.cloud/", Endpoint.of(app1) .targetRegion(ClusterSpec.Id.from("c1"), prodZone) .routingMethod(RoutingMethod.exclusive) .on(Port.tls()) - .legacy() .in(SystemName.Public) ); tests.forEach((expected, endpoint) -> assertEquals(expected, endpoint.url().toString())); diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ConfigServerMock.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ConfigServerMock.java index c3a527a0bd9..4203051965b 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ConfigServerMock.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ConfigServerMock.java @@ -122,7 +122,8 @@ public class ConfigServerMock extends AbstractComponent implements ConfigServer current, Instant.ofEpochMilli(1234), Optional.of(Instant.ofEpochMilli(2234)))), - "the autoscaling status", + "ideal", + "Cluster is ideally scaled", Duration.ofMinutes(6), 0.7, 0.3); diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ServiceRegistryMock.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ServiceRegistryMock.java index 702ce83d116..4a068681a50 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ServiceRegistryMock.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ServiceRegistryMock.java @@ -9,11 +9,12 @@ import com.yahoo.test.ManualClock; import com.yahoo.vespa.hosted.controller.api.integration.ServiceRegistry; import com.yahoo.vespa.hosted.controller.api.integration.archive.ArchiveService; import com.yahoo.vespa.hosted.controller.api.integration.archive.MockArchiveService; +import com.yahoo.vespa.hosted.controller.api.integration.athenz.AccessControlService; +import com.yahoo.vespa.hosted.controller.api.integration.athenz.MockAccessControlService; import com.yahoo.vespa.hosted.controller.api.integration.aws.MockRoleService; import com.yahoo.vespa.hosted.controller.api.integration.aws.RoleService; import com.yahoo.vespa.hosted.controller.api.integration.aws.MockAwsEventFetcher; import com.yahoo.vespa.hosted.controller.api.integration.aws.MockResourceTagger; -import com.yahoo.vespa.hosted.controller.api.integration.aws.NoopRoleService; import com.yahoo.vespa.hosted.controller.api.integration.aws.ResourceTagger; import com.yahoo.vespa.hosted.controller.api.integration.billing.BillingController; import com.yahoo.vespa.hosted.controller.api.integration.billing.MockBillingController; @@ -35,7 +36,6 @@ import com.yahoo.vespa.hosted.controller.api.integration.stubs.MockMailer; import com.yahoo.vespa.hosted.controller.api.integration.stubs.MockMeteringClient; import com.yahoo.vespa.hosted.controller.api.integration.stubs.MockRunDataStore; import com.yahoo.vespa.hosted.controller.api.integration.stubs.MockTesterCloud; -import com.yahoo.vespa.hosted.controller.api.integration.vcmr.ChangeRequestClient; import com.yahoo.vespa.hosted.controller.api.integration.vcmr.MockChangeRequestClient; /** @@ -73,6 +73,7 @@ public class ServiceRegistryMock extends AbstractComponent implements ServiceReg private final NoopTenantSecretService tenantSecretService = new NoopTenantSecretService(); private final ArchiveService archiveService = new MockArchiveService(); private final MockChangeRequestClient changeRequestClient = new MockChangeRequestClient(); + private final AccessControlService accessControlService = new MockAccessControlService(); public ServiceRegistryMock(SystemName system) { this.zoneRegistryMock = new ZoneRegistryMock(system); @@ -229,6 +230,11 @@ public class ServiceRegistryMock extends AbstractComponent implements ServiceReg return changeRequestClient; } + @Override + public AccessControlService accessControlService() { + return accessControlService; + } + public ConfigServerMock configServerMock() { return configServerMock; } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ZoneRegistryMock.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ZoneRegistryMock.java index bc81924225c..ca3909af0ba 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ZoneRegistryMock.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ZoneRegistryMock.java @@ -53,7 +53,8 @@ public class ZoneRegistryMock extends AbstractComponent implements ZoneRegistry this.zones = system.isPublic() ? List.of(ZoneApiMock.fromId("test.aws-us-east-1c"), ZoneApiMock.fromId("staging.aws-us-east-1c"), - ZoneApiMock.fromId("prod.aws-us-east-1c")) : + ZoneApiMock.fromId("prod.aws-us-east-1c"), + ZoneApiMock.fromId("prod.aws-eu-west-1a")) : List.of(ZoneApiMock.fromId("test.us-east-1"), ZoneApiMock.fromId("staging.us-east-3"), ZoneApiMock.fromId("dev.us-east-1"), diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintainerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintainerTest.java index 27b4f3744e7..7dc5cb34818 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintainerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintainerTest.java @@ -39,13 +39,16 @@ public class ControllerMaintainerTest { TestControllerMaintainer maintainer = new TestControllerMaintainer(tester.controller(), SystemName.main, new AtomicInteger()); maintainer.run(); assertEquals(0L, consecutiveFailuresMetric()); + assertEquals(1.0, successFactorMetric(), 0.0000001); maintainer.success = false; maintainer.run(); maintainer.run(); assertEquals(2L, consecutiveFailuresMetric()); + assertEquals(0.0, successFactorMetric(), 0.0000001); maintainer.success = true; maintainer.run(); assertEquals(0, consecutiveFailuresMetric()); + assertEquals(1.0, successFactorMetric(), 0.0000001); } private long consecutiveFailuresMetric() { @@ -54,6 +57,12 @@ public class ControllerMaintainerTest { "maintenance.consecutiveFailures").get().longValue(); } + private long successFactorMetric() { + MetricsMock metrics = (MetricsMock) tester.controller().metric(); + return metrics.getMetric((context) -> "TestControllerMaintainer".equals(context.get("job")), + "maintenance.successFactor").get().longValue(); + } + private static class TestControllerMaintainer extends ControllerMaintainer { private final AtomicInteger executions; @@ -65,9 +74,9 @@ public class ControllerMaintainerTest { } @Override - protected boolean maintain() { + protected double maintain() { executions.incrementAndGet(); - return success; + return success ? 1.0 : 0.0; } } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/EndpointCertificateMaintainerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/EndpointCertificateMaintainerTest.java index 66bda66bbf9..ce219b8beed 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/EndpointCertificateMaintainerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/EndpointCertificateMaintainerTest.java @@ -33,7 +33,7 @@ public class EndpointCertificateMaintainerTest { @Test public void old_and_unused_cert_is_deleted() { tester.curator().writeEndpointCertificateMetadata(ApplicationId.defaultId(), exampleMetadata); - assertTrue(maintainer.maintain()); + assertEquals(1.0, maintainer.maintain(), 0.0000001); assertTrue(tester.curator().readEndpointCertificateMetadata(ApplicationId.defaultId()).isEmpty()); } @@ -41,7 +41,7 @@ public class EndpointCertificateMaintainerTest { public void unused_but_recently_used_cert_is_not_deleted() { EndpointCertificateMetadata recentlyRequestedCert = exampleMetadata.withLastRequested(tester.clock().instant().minusSeconds(3600).getEpochSecond()); tester.curator().writeEndpointCertificateMetadata(ApplicationId.defaultId(), recentlyRequestedCert); - assertTrue(maintainer.maintain()); + assertEquals(1.0, maintainer.maintain(), 0.0000001); assertEquals(Optional.of(recentlyRequestedCert), tester.curator().readEndpointCertificateMetadata(ApplicationId.defaultId())); } @@ -53,7 +53,7 @@ public class EndpointCertificateMaintainerTest { secretStore.setSecret(exampleMetadata.keyName(), "foo", 1); secretStore.setSecret(exampleMetadata.certName(), "bar", 1); - assertTrue(maintainer.maintain()); + assertEquals(1.0, maintainer.maintain(), 0.0000001); var updatedCert = Optional.of(recentlyRequestedCert.withLastRefreshed(tester.clock().instant().getEpochSecond()).withVersion(1)); @@ -77,7 +77,7 @@ public class EndpointCertificateMaintainerTest { tester.curator().writeEndpointCertificateMetadata(appId, exampleMetadata); - assertTrue(maintainer.maintain()); + assertEquals(1.0, maintainer.maintain(), 0.0000001); assertTrue(tester.curator().readEndpointCertificateMetadata(appId).isPresent()); // cert should not be deleted, the app is deployed! } @@ -97,7 +97,7 @@ public class EndpointCertificateMaintainerTest { tester.curator().writeEndpointCertificateMetadata(appId, exampleMetadata); - assertTrue(maintainer.maintain()); + assertEquals(1.0, maintainer.maintain(), 0.0000001); assertTrue(tester.curator().readEndpointCertificateMetadata(appId).isPresent()); // cert should not be deleted, the app is deployed! tester.clock().advance(Duration.ofDays(3)); diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/ResourceMeterMaintainerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/ResourceMeterMaintainerTest.java index 0deaa21d13b..e61516cbb1a 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/ResourceMeterMaintainerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/ResourceMeterMaintainerTest.java @@ -6,18 +6,25 @@ import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.HostName; import com.yahoo.config.provision.NodeResources; import com.yahoo.config.provision.NodeType; +import com.yahoo.config.provision.SystemName; +import com.yahoo.config.provision.zone.ZoneId; import com.yahoo.vespa.hosted.controller.ControllerTester; import com.yahoo.vespa.hosted.controller.api.integration.configserver.Node; import com.yahoo.vespa.hosted.controller.api.integration.resource.ResourceSnapshot; import com.yahoo.vespa.hosted.controller.api.integration.stubs.MockMeteringClient; +import com.yahoo.vespa.hosted.controller.application.ApplicationPackage; +import com.yahoo.vespa.hosted.controller.deployment.ApplicationPackageBuilder; +import com.yahoo.vespa.hosted.controller.deployment.DeploymentTester; import com.yahoo.vespa.hosted.controller.integration.MetricsMock; import com.yahoo.vespa.hosted.controller.integration.ZoneApiMock; import org.junit.Test; import java.time.Duration; -import java.util.Arrays; +import java.time.Instant; import java.util.Collection; import java.util.List; +import java.util.Map; +import java.util.function.BiConsumer; import java.util.stream.Collectors; import java.util.stream.Stream; @@ -28,17 +35,52 @@ import static org.junit.Assert.*; */ public class ResourceMeterMaintainerTest { - private final ControllerTester tester = new ControllerTester(); + private final ControllerTester tester = new ControllerTester(SystemName.Public); private final MockMeteringClient snapshotConsumer = new MockMeteringClient(); private final MetricsMock metrics = new MetricsMock(); + private final ResourceMeterMaintainer maintainer = + new ResourceMeterMaintainer(tester.controller(), Duration.ofMinutes(5), metrics, snapshotConsumer); + + @Test + public void updates_deployment_costs() { + ApplicationId app1 = ApplicationId.from("t1", "a1", "default"); + ApplicationId app2 = ApplicationId.from("t2", "a1", "default"); + ZoneId z1 = ZoneId.from("prod.aws-us-east-1c"); + ZoneId z2 = ZoneId.from("prod.aws-eu-west-1a"); + + DeploymentTester deploymentTester = new DeploymentTester(tester); + ApplicationPackage applicationPackage = new ApplicationPackageBuilder().region(z1.region()).region(z2.region()).trustDefaultCertificate().build(); + List.of(app1, app2).forEach(app -> deploymentTester.newDeploymentContext(app).submit(applicationPackage).deploy()); + + BiConsumer<ApplicationId, Map<ZoneId, Double>> assertCost = (appId, costs) -> + assertEquals(costs, tester.controller().applications().getInstance(appId).get().deployments().entrySet().stream() + .filter(entry -> entry.getValue().cost().isPresent()) + .collect(Collectors.toMap(Map.Entry::getKey, entry -> entry.getValue().cost().getAsDouble()))); + + List<ResourceSnapshot> resourceSnapshots = List.of( + new ResourceSnapshot(app1, 12, 34, 56, Instant.EPOCH, z1), + new ResourceSnapshot(app1, 23, 45, 67, Instant.EPOCH, z2), + new ResourceSnapshot(app2, 34, 56, 78, Instant.EPOCH, z1)); + maintainer.updateDeploymentCost(resourceSnapshots); + assertCost.accept(app1, Map.of(z1, 1.40, z2, 2.50)); + assertCost.accept(app2, Map.of(z1, 3.59)); + + // Remove a region from app1 and add region to app2 + resourceSnapshots = List.of( + new ResourceSnapshot(app1, 23, 45, 67, Instant.EPOCH, z2), + new ResourceSnapshot(app2, 34, 56, 78, Instant.EPOCH, z1), + new ResourceSnapshot(app2, 45, 67, 89, Instant.EPOCH, z2)); + maintainer.updateDeploymentCost(resourceSnapshots); + assertCost.accept(app1, Map.of(z2, 2.50)); + assertCost.accept(app2, Map.of(z1, 3.59, z2, 4.68)); + } @Test public void testMaintainer() { setUpZones(); - ResourceMeterMaintainer resourceMeterMaintainer = new ResourceMeterMaintainer(tester.controller(), Duration.ofMinutes(5), metrics, snapshotConsumer); long lastRefreshTime = tester.clock().millis(); tester.curator().writeMeteringRefreshTime(lastRefreshTime); - resourceMeterMaintainer.maintain(); + maintainer.maintain(); Collection<ResourceSnapshot> consumedResources = snapshotConsumer.consumedResources(); // The mocked repository contains two applications, so we should also consume two ResourceSnapshots @@ -63,26 +105,18 @@ public class ResourceMeterMaintainerTest { var millisAdvanced = 3600 * 1000; tester.clock().advance(Duration.ofMillis(millisAdvanced)); - resourceMeterMaintainer.maintain(); + maintainer.maintain(); assertTrue(snapshotConsumer.isRefreshed()); assertEquals(lastRefreshTime + millisAdvanced, tester.curator().readMeteringRefreshTime()); } private void setUpZones() { - ZoneApiMock nonAwsZone = ZoneApiMock.newBuilder().withId("test.region-1").build(); - ZoneApiMock awsZone1 = ZoneApiMock.newBuilder().withId("prod.region-2").withCloud("aws").build(); - ZoneApiMock awsZone2 = ZoneApiMock.newBuilder().withId("test.region-3").withCloud("aws").build(); - tester.zoneRegistry().setZones( - nonAwsZone, - awsZone1, - awsZone2); - tester.configServer().nodeRepository().setFixedNodes(nonAwsZone.getId()); - tester.configServer().nodeRepository().setFixedNodes(awsZone1.getId()); - tester.configServer().nodeRepository().setFixedNodes(awsZone2.getId()); - tester.configServer().nodeRepository().putNodes( - awsZone1.getId(), - createNodes() - ); + ZoneApiMock zone1 = ZoneApiMock.newBuilder().withId("prod.region-2").build(); + ZoneApiMock zone2 = ZoneApiMock.newBuilder().withId("test.region-3").build(); + tester.zoneRegistry().setZones(zone1, zone2); + tester.configServer().nodeRepository().setFixedNodes(zone1.getId()); + tester.configServer().nodeRepository().setFixedNodes(zone2.getId()); + tester.configServer().nodeRepository().putNodes(zone1.getId(), createNodes()); } private List<Node> createNodes() { @@ -92,23 +126,21 @@ public class ResourceMeterMaintainerTest { Node.State.failed, Node.State.parked, Node.State.active) - .map(state -> { - return new Node.Builder() - .hostname(HostName.from("host" + state)) - .parentHostname(HostName.from("parenthost" + state)) - .state(state) - .type(NodeType.tenant) - .owner(ApplicationId.from("tenant1", "app1", "default")) - .currentVersion(Version.fromString("7.42")) - .wantedVersion(Version.fromString("7.42")) - .currentOsVersion(Version.fromString("7.6")) - .wantedOsVersion(Version.fromString("7.6")) - .serviceState(Node.ServiceState.expectedUp) - .resources(new NodeResources(24, 24, 500, 1)) - .clusterId("clusterA") - .clusterType(state == Node.State.active ? Node.ClusterType.admin : Node.ClusterType.container) - .build(); - }) + .map(state -> new Node.Builder() + .hostname(HostName.from("host" + state)) + .parentHostname(HostName.from("parenthost" + state)) + .state(state) + .type(NodeType.tenant) + .owner(ApplicationId.from("tenant1", "app1", "default")) + .currentVersion(Version.fromString("7.42")) + .wantedVersion(Version.fromString("7.42")) + .currentOsVersion(Version.fromString("7.6")) + .wantedOsVersion(Version.fromString("7.6")) + .serviceState(Node.ServiceState.expectedUp) + .resources(new NodeResources(24, 24, 500, 1)) + .clusterId("clusterA") + .clusterType(state == Node.State.active ? Node.ClusterType.admin : Node.ClusterType.container) + .build()) .collect(Collectors.toUnmodifiableList()); } } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/TrafficShareUpdaterTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/TrafficShareUpdaterTest.java index 2afa3a0faea..7b4882de3ff 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/TrafficShareUpdaterTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/TrafficShareUpdaterTest.java @@ -39,7 +39,7 @@ public class TrafficShareUpdaterTest { // Single zone setQpsMetric(50.0, application.application().id().defaultInstance(), prod1, tester); deploymentMetricsMaintainer.maintain(); - assertTrue(updater.maintain()); + assertEquals(1.0, updater.maintain(), 0.0000001); assertTrafficFraction(1.0, 1.0, application.instanceId(), prod1, tester); // Two zones @@ -48,14 +48,14 @@ public class TrafficShareUpdaterTest { setQpsMetric(50.0, application.application().id().defaultInstance(), prod1, tester); setQpsMetric(0.0, application.application().id().defaultInstance(), prod2, tester); deploymentMetricsMaintainer.maintain(); - assertTrue(updater.maintain()); + assertEquals(1.0, updater.maintain(), 0.0000001); assertTrafficFraction(1.0, 1.0, application.instanceId(), prod1, tester); assertTrafficFraction(0.0, 1.0, application.instanceId(), prod2, tester); // - both hot setQpsMetric(53.0, application.application().id().defaultInstance(), prod1, tester); setQpsMetric(47.0, application.application().id().defaultInstance(), prod2, tester); deploymentMetricsMaintainer.maintain(); - assertTrue(updater.maintain()); + assertEquals(1.0, updater.maintain(), 0.0000001); assertTrafficFraction(0.53, 1.0, application.instanceId(), prod1, tester); assertTrafficFraction(0.47, 1.0, application.instanceId(), prod2, tester); @@ -66,7 +66,7 @@ public class TrafficShareUpdaterTest { setQpsMetric(47.0, application.application().id().defaultInstance(), prod2, tester); setQpsMetric(0.0, application.application().id().defaultInstance(), prod3, tester); deploymentMetricsMaintainer.maintain(); - assertTrue(updater.maintain()); + assertEquals(1.0, updater.maintain(), 0.0000001); assertTrafficFraction(0.53, 0.53, application.instanceId(), prod1, tester); assertTrafficFraction(0.47, 0.50, application.instanceId(), prod2, tester); assertTrafficFraction(0.00, 0.50, application.instanceId(), prod3, tester); @@ -75,7 +75,7 @@ public class TrafficShareUpdaterTest { setQpsMetric(25.0, application.application().id().defaultInstance(), prod2, tester); setQpsMetric(25.0, application.application().id().defaultInstance(), prod3, tester); deploymentMetricsMaintainer.maintain(); - assertTrue(updater.maintain()); + assertEquals(1.0, updater.maintain(), 0.0000001); assertTrafficFraction(0.50, 0.5, application.instanceId(), prod1, tester); assertTrafficFraction(0.25, 0.5, application.instanceId(), prod2, tester); assertTrafficFraction(0.25, 0.5, application.instanceId(), prod3, tester); diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializerTest.java index 2dcf012ac6d..37a173ffc37 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializerTest.java @@ -96,14 +96,15 @@ public class ApplicationSerializerTest { Version.fromString("6.3.1"), Instant.ofEpochMilli(496)); Instant activityAt = Instant.parse("2018-06-01T10:15:30.00Z"); deployments.add(new Deployment(zone1, applicationVersion1, Version.fromString("1.2.3"), Instant.ofEpochMilli(3), - DeploymentMetrics.none, DeploymentActivity.none, QuotaUsage.none)); + DeploymentMetrics.none, DeploymentActivity.none, QuotaUsage.none, OptionalDouble.empty())); deployments.add(new Deployment(zone2, applicationVersion2, Version.fromString("1.2.3"), Instant.ofEpochMilli(5), new DeploymentMetrics(2, 3, 4, 5, 6, Optional.of(Instant.now().truncatedTo(ChronoUnit.MILLIS)), Map.of(DeploymentMetrics.Warning.all, 3)), DeploymentActivity.create(Optional.of(activityAt), Optional.of(activityAt), OptionalDouble.of(200), OptionalDouble.of(10)), - QuotaUsage.create(OptionalDouble.of(23.5)))); + QuotaUsage.create(OptionalDouble.of(23.5)), + OptionalDouble.of(12.3))); var rotationStatus = RotationStatus.from(Map.of(new RotationId("my-rotation"), new RotationStatus.Targets( diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/RunSerializerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/RunSerializerTest.java index 8df2fd87398..6e12373640d 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/RunSerializerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/RunSerializerTest.java @@ -19,14 +19,12 @@ import com.yahoo.vespa.hosted.controller.deployment.StepInfo; import org.junit.Test; import java.io.IOException; -import java.nio.charset.StandardCharsets; import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; import java.time.Instant; import java.util.Collections; import java.util.Optional; -import java.util.function.BiConsumer; import static com.yahoo.vespa.hosted.controller.deployment.RunStatus.aborted; import static com.yahoo.vespa.hosted.controller.deployment.RunStatus.running; @@ -155,18 +153,4 @@ public class RunSerializerTest { assertEquals(initial, serializer.runFromSlime(serializer.toSlime(initial))); } - @Test - public void convergenceSummaryMigrationTest() throws IOException { - String data = Files.readString(runFile); - BiConsumer<String, ConvergenceSummary> replaceAndAssert = (replace, convergenceSummaryOrNull) -> { - byte[] newData = data.replace("\"convergenceSummaryV2\": [1, 1, 2, 3, 5, 8, 13, 21, 34, 55, 89, 144, 233],", replace).getBytes(StandardCharsets.UTF_8); - assertEquals(convergenceSummaryOrNull, serializer.runsFromSlime(SlimeUtils.jsonToSlime(newData)).get(id).convergenceSummary().orElse(null)); - }; - - replaceAndAssert.accept("", null); - replaceAndAssert.accept("\"convergenceSummary\": [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12],", new ConvergenceSummary(1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 0)); - replaceAndAssert.accept("\"convergenceSummaryV2\": [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13],\n" + - "\"convergenceSummary\": [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12],", new ConvergenceSummary(1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13)); - } - } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java index 47aa3e6b9d4..f4b8a643e28 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java @@ -18,6 +18,7 @@ import com.yahoo.security.KeyAlgorithm; import com.yahoo.security.KeyUtils; import com.yahoo.security.SignatureAlgorithm; import com.yahoo.security.X509CertificateBuilder; +import com.yahoo.security.X509CertificateUtils; import com.yahoo.vespa.athenz.api.AthenzDomain; import com.yahoo.vespa.athenz.api.AthenzIdentity; import com.yahoo.vespa.athenz.api.AthenzPrincipal; @@ -70,6 +71,7 @@ import com.yahoo.vespa.hosted.controller.restapi.ControllerContainerTest; import com.yahoo.vespa.hosted.controller.routing.GlobalRouting; import com.yahoo.vespa.hosted.controller.security.AthenzCredentials; import com.yahoo.vespa.hosted.controller.security.AthenzTenantSpec; +import com.yahoo.vespa.hosted.controller.support.access.SupportAccessGrant; import com.yahoo.vespa.hosted.controller.tenant.AthenzTenant; import com.yahoo.vespa.hosted.controller.tenant.LastLoginInfo; import com.yahoo.vespa.hosted.controller.versions.VespaVersion; @@ -82,6 +84,7 @@ import java.io.File; import java.math.BigInteger; import java.net.URI; import java.security.cert.X509Certificate; +import java.time.Duration; import java.time.Instant; import java.time.format.DateTimeFormatter; import java.time.temporal.ChronoUnit; @@ -1501,6 +1504,7 @@ public class ApplicationApiTest extends ControllerContainerTest { var zone = ZoneId.from(Environment.prod, RegionName.from("us-west-1")); deploymentTester.controllerTester().zoneRegistry().setRoutingMethod(ZoneApiMock.from(zone), List.of(RoutingMethod.exclusive, RoutingMethod.shared)); + addUserToHostedOperatorRole(HostedAthenzIdentities.from(HOSTED_VESPA_OPERATOR)); ApplicationPackage applicationPackage = new ApplicationPackageBuilder() .athenzIdentity(com.yahoo.config.provision.AthenzDomain.from("domain"), AthenzService.from("service")) .compileVersion(RoutingController.DIRECT_ROUTING_MIN_VERSION) @@ -1528,26 +1532,40 @@ public class ApplicationApiTest extends ControllerContainerTest { // Grant access to support user X509Certificate support_cert = grantCertificate(now, now.plusSeconds(3600)); - tester.controller().supportAccess().registerGrant(app.deploymentIdIn(zone), "user.andreer", support_cert); + String grantPayload= "{\n" + + " \"applicationId\": \"tenant1:application1:instance1\",\n" + + " \"zone\": \"prod.us-west-1\",\n" + + " \"certificate\":\""+X509CertificateUtils.toPem(support_cert)+ "\"\n" + + "}"; + tester.assertResponse(request("/controller/v1/access/grants/"+HOSTED_VESPA_OPERATOR.id(), POST) + .data(grantPayload) + .userIdentity(HOSTED_VESPA_OPERATOR), + "{\"message\":\"Operator user.johnoperator granted access and job production-us-west-1 triggered\"}"); // GET shows grant String grantResponse = allowedResponse.replaceAll("\"grants\":\\[]", - "\"grants\":[{\"requestor\":\"user.andreer\",\"notBefore\":\"" + serializeInstant(now) + "\",\"notAfter\":\"" + serializeInstant(now.plusSeconds(3600)) + "\"}]"); + "\"grants\":[{\"requestor\":\"user.johnoperator\",\"notBefore\":\"" + serializeInstant(now) + "\",\"notAfter\":\"" + serializeInstant(now.plusSeconds(3600)) + "\"}]"); tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/instance/instance1/environment/prod/region/us-west-1/access/support", GET) .userIdentity(USER_ID), grantResponse, 200 ); + // Should be 1 available grant + List<SupportAccessGrant> activeGrants = tester.controller().supportAccess().activeGrantsFor(new DeploymentId(ApplicationId.fromSerializedForm("tenant1:application1:instance1"), zone)); + assertEquals(1, activeGrants.size()); + // DELETE removes access - System.out.println("grantresponse:\n"+grantResponse+"\n"); String disallowedResponse = grantResponse .replaceAll("ALLOWED\".*?}", "NOT_ALLOWED\"}") .replace("history\":[", "history\":[{\"state\":\"disallowed\",\"at\":\""+ serializeInstant(now) +"\",\"by\":\"user.myuser\"},"); - System.out.println("disallowedResponse:\n"+disallowedResponse+"\n"); tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/instance/instance1/environment/prod/region/us-west-1/access/support", DELETE) .userIdentity(USER_ID), disallowedResponse, 200 ); + + // Should be no available grant + activeGrants = tester.controller().supportAccess().activeGrantsFor(new DeploymentId(ApplicationId.fromSerializedForm("tenant1:application1:instance1"), zone)); + assertEquals(0, activeGrants.size()); } private static String serializeInstant(Instant i) { diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/application-clusters.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/application-clusters.json index cd1fe5acf6a..fc40a9ce692 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/application-clusters.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/application-clusters.json @@ -97,7 +97,8 @@ "completion": 2234 } ], - "autoscalingStatus": "the autoscaling status", + "autoscalingStatusCode": "ideal", + "autoscalingStatus": "Cluster is ideally scaled", "scalingDuration": 360000, "maxQueryGrowthRate": 0.7, "currentQueryFractionOfMax":0.3 diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/deployment-cloud.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/deployment-cloud.json index 3353d80204e..a214969485d 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/deployment-cloud.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/deployment-cloud.json @@ -8,7 +8,7 @@ { "cluster": "default", "tls": true, - "url": "https://albums.scoober.aws-us-east-1c.public.vespa.oath.cloud/", + "url": "https://albums.scoober.aws-us-east-1c.z.vespa-app.cloud/", "scope": "zone", "routingMethod": "exclusive", "legacy": false @@ -36,6 +36,7 @@ "commit": "commit1" }, "status": "complete", + "quota": "(ignore)", "archiveUri": "s3://bucketName/scoober/", "activity": {}, "metrics": { diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/deployment-with-routing-policy-legacy.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/deployment-with-routing-policy-legacy.json index 6efcc822264..eb508b2459e 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/deployment-with-routing-policy-legacy.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/deployment-with-routing-policy-legacy.json @@ -60,6 +60,7 @@ "commit": "commit1" }, "status": "complete", + "quota": "(ignore)", "activity": {}, "metrics": { "queriesPerSecond": 0.0, diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/deployment-with-routing-policy.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/deployment-with-routing-policy.json index 8767c369bc3..97ac87fb5a0 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/deployment-with-routing-policy.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/deployment-with-routing-policy.json @@ -44,6 +44,7 @@ "commit": "commit1" }, "status": "complete", + "quota": "(ignore)", "activity": {}, "metrics": { "queriesPerSecond": 0.0, diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/deployment-without-shared-endpoints.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/deployment-without-shared-endpoints.json index b59c1d6cf73..39b8c779184 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/deployment-without-shared-endpoints.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/deployment-without-shared-endpoints.json @@ -36,6 +36,7 @@ "commit": "commit1" }, "status": "complete", + "quota": "(ignore)", "activity": {}, "metrics": { "queriesPerSecond": 0.0, diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/deployment.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/deployment.json index 6c00d654008..3ce83528b2c 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/deployment.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/deployment.json @@ -53,6 +53,7 @@ "commit": "commit1" }, "status": "complete", + "quota": "(ignore)", "activity": { "lastQueried": 1527848130000, "lastWritten": 1527848130000, diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/dev-us-east-1.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/dev-us-east-1.json index 1084afc9388..d61bebc81d1 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/dev-us-east-1.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/dev-us-east-1.json @@ -21,6 +21,7 @@ "revision": "(ignore)", "deployTimeEpochMs": "(ignore)", "screwdriverId": "123", + "quota": "(ignore)", "activity": { "lastQueried": 1527848130000, "lastWritten": 1527848130000, diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/prod-us-central-1.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/prod-us-central-1.json index 9059ea338b1..4edbc58121b 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/prod-us-central-1.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/prod-us-central-1.json @@ -56,6 +56,7 @@ "commit": "commit1" }, "status": "complete", + "quota": "(ignore)", "activity": { "lastQueried": 1527848130000, "lastWritten": 1527848130000, diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/controller/ControllerApiTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/controller/ControllerApiTest.java index 8f6988dbc27..fc83c58cc67 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/controller/ControllerApiTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/controller/ControllerApiTest.java @@ -6,8 +6,11 @@ import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.zone.ZoneId; import com.yahoo.container.jdisc.HttpRequest; import com.yahoo.test.ManualClock; +import com.yahoo.vespa.athenz.api.AthenzUser; import com.yahoo.vespa.flags.InMemoryFlagSource; import com.yahoo.vespa.flags.PermanentFlags; +import com.yahoo.vespa.hosted.controller.api.identifiers.DeploymentId; +import com.yahoo.vespa.hosted.controller.api.integration.athenz.MockAccessControlService; import com.yahoo.vespa.hosted.controller.api.integration.configserver.Application; import com.yahoo.vespa.hosted.controller.api.integration.resource.ResourceSnapshot; import com.yahoo.vespa.hosted.controller.auditlog.AuditLogger; @@ -165,4 +168,25 @@ public class ControllerApiTest extends ControllerContainerTest { ); } + @Test + public void testApproveMembership() { + ApplicationId applicationId = ApplicationId.from("tenant", "app", "instance"); + DeploymentId deployment = new DeploymentId(applicationId, ZoneId.defaultId()); + String requestBody = "{\n" + + " \"applicationId\": \"" + deployment.applicationId().serializedForm() + "\",\n" + + " \"zone\": \"" + deployment.zoneId().value() + "\"\n" + + "}"; + + MockAccessControlService accessControlService = (MockAccessControlService) tester.serviceRegistry().accessControlService(); + tester.assertResponse(operatorRequest("http://localhost:8080/controller/v1/access/requests/"+hostedOperator.getName(), requestBody, Request.Method.POST), + "{\"message\":\"Unable to approve membership request\"}", 400); + + accessControlService.addPendingMember(hostedOperator); + tester.assertResponse(operatorRequest("http://localhost:8080/controller/v1/access/requests/"+hostedOperator.getName(), requestBody, Request.Method.POST), + "{\"message\":\"Unable to approve membership request\"}", 400); + + tester.controller().supportAccess().allow(deployment, Instant.now().plus(Duration.ofHours(1)), "tenantx"); + tester.assertResponse(operatorRequest("http://localhost:8080/controller/v1/access/requests/"+hostedOperator.getName(), requestBody, Request.Method.POST), + "{\"members\":[\"user.alice\"]}"); + } } diff --git a/default_build_settings.cmake b/default_build_settings.cmake index 326481e25c7..3fd93bd0c25 100644 --- a/default_build_settings.cmake +++ b/default_build_settings.cmake @@ -302,8 +302,8 @@ function(vespa_use_default_cxx_compiler) unset(DEFAULT_CMAKE_CXX_COMPILER) if(NOT DEFINED VESPA_COMPILER_VARIANT OR VESPA_COMPILER_VARIANT STREQUAL "gcc") if(APPLE) - set(DEFAULT_CMAKE_C_COMPILER "/usr/local/bin/gcc-10") - set(DEFAULT_CMAKE_CXX_COMPILER "/usr/local/bin/g++-10") + set(DEFAULT_CMAKE_C_COMPILER "/usr/local/bin/gcc-11") + set(DEFAULT_CMAKE_CXX_COMPILER "/usr/local/bin/g++-11") elseif(VESPA_OS_DISTRO_COMBINED STREQUAL "amzn 2") set(DEFAULT_CMAKE_C_COMPILER "/usr/bin/gcc10-gcc") set(DEFAULT_CMAKE_CXX_COMPILER "/usr/bin/gcc10-g++") diff --git a/dist/vespa.spec b/dist/vespa.spec index de92077fdbd..34afc46c80c 100644 --- a/dist/vespa.spec +++ b/dist/vespa.spec @@ -724,9 +724,12 @@ fi %defattr(-,%{_vespa_user},%{_vespa_group},-) %endif %dir %{_prefix} +%dir %{_prefix}/conf +%dir %{_prefix}/conf/vespa-feed-client %dir %{_prefix}/lib %dir %{_prefix}/lib/jars %{_prefix}/bin/vespa-feed-client +%{_prefix}/conf/vespa-feed-client/logging.properties %{_prefix}/lib/jars/vespa-http-client-jar-with-dependencies.jar %{_prefix}/lib/jars/vespa-feed-client-cli.jar diff --git a/flags/src/main/java/com/yahoo/vespa/flags/Flags.java b/flags/src/main/java/com/yahoo/vespa/flags/Flags.java index aeec579183d..cfa14979d3f 100644 --- a/flags/src/main/java/com/yahoo/vespa/flags/Flags.java +++ b/flags/src/main/java/com/yahoo/vespa/flags/Flags.java @@ -41,15 +41,8 @@ import static com.yahoo.vespa.flags.FetchVector.Dimension.ZONE_ID; * @author hakonhall */ public class Flags { - private static volatile TreeMap<FlagId, FlagDefinition> flags = new TreeMap<>(); - public static final UnboundStringFlag ALLOCATE_OS_REQUIREMENT = defineStringFlag( - "allocate-os-requirement", "any", - List.of("hakonhall"), "2021-01-26", "2021-07-26", - "Allocations of new nodes are limited to the given host OS. Must be one of 'rhel7', " + - "'rhel8', or 'any'", - "Takes effect on next (re)deployment.", - APPLICATION_ID); + private static volatile TreeMap<FlagId, FlagDefinition> flags = new TreeMap<>(); public static final UnboundDoubleFlag DEFAULT_TERM_WISE_LIMIT = defineDoubleFlag( "default-term-wise-limit", 1.0, @@ -102,28 +95,22 @@ public class Flags { public static final UnboundBooleanFlag USE_THREE_PHASE_UPDATES = defineFeatureFlag( "use-three-phase-updates", false, - List.of("vekterli"), "2020-12-02", "2021-06-01", + List.of("vekterli"), "2020-12-02", "2021-08-01", "Whether to enable the use of three-phase updates when bucket replicas are out of sync.", "Takes effect at redeployment", ZONE_ID, APPLICATION_ID); - public static final UnboundBooleanFlag PROVISION_TENANT_ROLES = defineFeatureFlag( - "provision-tenant-roles", false, - List.of("tokle"), "2020-12-02", "2021-06-01", - "Whether tenant roles should be provisioned", - "Takes effect on next deployment (controller)", - TENANT_ID); - + // TODO: Remove when models referring to this are gone in all systems public static final UnboundBooleanFlag TENANT_IAM_ROLE = defineFeatureFlag( "application-iam-roles", false, - List.of("tokle"), "2020-12-02", "2021-06-01", + List.of("tokle"), "2020-12-02", "2021-08-01", "Allow separate iam roles when provisioning/assigning hosts", "Takes effect immediately on new hosts, on next redeploy for applications", TENANT_ID); public static final UnboundBooleanFlag HIDE_SHARED_ROUTING_ENDPOINT = defineFeatureFlag( "hide-shared-routing-endpoint", false, - List.of("tokle"), "2020-12-02", "2021-06-01", + List.of("tokle", "bjormel"), "2020-12-02", "2021-09-01", "Whether the controller should hide shared routing layer endpoint", "Takes effect immediately", APPLICATION_ID @@ -143,13 +130,6 @@ public class Flags { "Takes effect at redeployment", ZONE_ID, APPLICATION_ID); - public static final UnboundBooleanFlag USE_BUCKET_EXECUTOR_FOR_PRUNE_REMOVED = defineFeatureFlag( - "use-bucket-executor-for-prune-removed", true, - List.of("baldersheim"), "2021-05-04", "2021-06-01", - "Wheter to use content-level bucket executor or legacy frozen buckets for prune removed", - "Takes effect on next internal redeployment", - APPLICATION_ID); - public static final UnboundBooleanFlag GROUP_SUSPENSION = defineFeatureFlag( "group-suspension", true, List.of("hakon"), "2021-01-22", "2021-06-22", @@ -159,14 +139,14 @@ public class Flags { public static final UnboundBooleanFlag ENCRYPT_DISK = defineFeatureFlag( "encrypt-disk", false, - List.of("hakonhall"), "2021-05-05", "2021-06-05", + List.of("hakonhall"), "2021-05-05", "2021-08-05", "Allow migrating an unencrypted data partition to being encrypted.", "Takes effect on next host-admin tick."); public static final UnboundBooleanFlag ENCRYPT_DIRTY_DISK = defineFeatureFlag( "encrypt-dirty-disk", false, - List.of("hakonhall"), "2021-05-14", "2021-06-05", - "Allow migrating an unencrypted data partition to being encrypted when provisioned or dirty.", + List.of("hakonhall"), "2021-05-14", "2021-08-05", + "Allow migrating an unencrypted data partition to being encrypted when (de)provisioned.", "Takes effect on next host-admin tick."); public static final UnboundBooleanFlag ENABLE_FEED_BLOCK_IN_DISTRIBUTOR = defineFeatureFlag( @@ -224,6 +204,20 @@ public class Flags { "Takes effect after distributor restart", ZONE_ID, APPLICATION_ID); + public static final UnboundIntFlag MAX_CONCURRENT_MERGES_PER_NODE = defineIntFlag( + "max-concurrent-merges-per-node", 16, + List.of("balder", "vekterli"), "2021-06-06", "2021-08-01", + "Specifies max concurrent merges per content node.", + "Takes effect at redeploy", + ZONE_ID, APPLICATION_ID); + + public static final UnboundIntFlag MAX_MERGE_QUEUE_SIZE = defineIntFlag( + "max-merge-queue-size", 1024, + List.of("balder", "vekterli"), "2021-06-06", "2021-08-01", + "Specifies max size of merge queue.", + "Takes effect at redeploy", + ZONE_ID, APPLICATION_ID); + public static final UnboundBooleanFlag USE_EXTERNAL_RANK_EXPRESSION = defineFeatureFlag( "use-external-rank-expression", false, List.of("baldersheim"), "2021-05-24", "2021-07-01", @@ -265,6 +259,13 @@ public class Flags { "The maximum number of hosts allowed to encrypt their disk concurrently", "Takes effect on next run of HostEncrypter, but any currently encrypting hosts will not be cancelled when reducing the limit"); + public static final UnboundBooleanFlag REQUIRE_CONNECTIVITY_CHECK = defineFeatureFlag( + "require-connectivity-check", false, + List.of("arnej"), "2021-06-03", "2021-09-01", + "Require that config-sentinel connectivity check passes with good quality before starting services", + "Takes effect on next restart", + ZONE_ID, APPLICATION_ID); + /** WARNING: public for testing: All flags should be defined in {@link Flags}. */ public static UnboundBooleanFlag defineFeatureFlag(String flagId, boolean defaultValue, List<String> owners, String createdAt, String expiresAt, String description, diff --git a/fnet/src/tests/connect/connect_test.cpp b/fnet/src/tests/connect/connect_test.cpp index 377035726dd..bb2acdb141e 100644 --- a/fnet/src/tests/connect/connect_test.cpp +++ b/fnet/src/tests/connect/connect_test.cpp @@ -65,6 +65,7 @@ struct BlockingCryptoSocket : public CryptoSocket { ssize_t write(const char *buf, size_t len) override { return socket.write(buf, len); } ssize_t flush() override { return 0; } ssize_t half_close() override { return socket.half_close(); } + void drop_empty_buffers() override {} }; struct BlockingCryptoEngine : public CryptoEngine { diff --git a/fnet/src/tests/frt/parallel_rpc/parallel_rpc_test.cpp b/fnet/src/tests/frt/parallel_rpc/parallel_rpc_test.cpp index 58acb928540..b03df359715 100644 --- a/fnet/src/tests/frt/parallel_rpc/parallel_rpc_test.cpp +++ b/fnet/src/tests/frt/parallel_rpc/parallel_rpc_test.cpp @@ -18,8 +18,8 @@ struct Rpc : FRT_Invokable { FastOS_ThreadPool thread_pool; FNET_Transport transport; FRT_Supervisor orb; - Rpc(CryptoEngine::SP crypto, size_t num_threads) - : thread_pool(128_Ki), transport(TransportConfig(num_threads).crypto(std::move(crypto))), orb(&transport) {} + Rpc(CryptoEngine::SP crypto, size_t num_threads, bool drop_empty) + : thread_pool(128_Ki), transport(TransportConfig(num_threads).crypto(std::move(crypto)).drop_empty_buffers(drop_empty)), orb(&transport) {} void start() { ASSERT_TRUE(transport.Start(&thread_pool)); } @@ -38,7 +38,7 @@ struct Rpc : FRT_Invokable { struct Server : Rpc { uint32_t port; - Server(CryptoEngine::SP crypto, size_t num_threads) : Rpc(std::move(crypto), num_threads), port(listen()) { + Server(CryptoEngine::SP crypto, size_t num_threads, bool drop_empty = false) : Rpc(std::move(crypto), num_threads, drop_empty), port(listen()) { init_rpc(); start(); } @@ -58,7 +58,7 @@ struct Server : Rpc { struct Client : Rpc { uint32_t port; - Client(CryptoEngine::SP crypto, size_t num_threads, const Server &server) : Rpc(std::move(crypto), num_threads), port(server.port) { + Client(CryptoEngine::SP crypto, size_t num_threads, const Server &server, bool drop_empty = false) : Rpc(std::move(crypto), num_threads, drop_empty), port(server.port) { start(); } FRT_Target *connect() { return Rpc::connect(port); } @@ -85,7 +85,16 @@ struct Result { } }; -void perform_test(size_t thread_id, Client &client, Result &result) { +bool verbose = false; +double budget = 1.5; + +void perform_test(size_t thread_id, Client &client, Result &result, bool vital = false) { + if (!vital && !verbose) { + if (thread_id == 0) { + fprintf(stderr, "... skipping non-vital test; run with 'verbose' to enable\n"); + } + return; + } uint64_t seq = 0; FRT_Target *target = client.connect(); FRT_RPCRequest *req = client.orb.AllocRPCRequest(); @@ -101,7 +110,7 @@ void perform_test(size_t thread_id, Client &client, Result &result) { }; size_t loop_cnt = 8; BenchmarkTimer::benchmark(invoke, invoke, 0.5); - BenchmarkTimer timer(1.5); + BenchmarkTimer timer(budget); while (timer.has_budget()) { timer.before(); for (size_t i = 0; i < loop_cnt; ++i) { @@ -139,13 +148,29 @@ TEST_MT_FFF("parallel rpc with 1/1 transport threads and num_cores user threads TEST_MT_FFF("parallel rpc with 1/1 transport threads and num_cores user threads (tls encryption)", getNumThreads(), Server(tls_crypto, 1), Client(tls_crypto, 1, f1), Result(num_threads)) { perform_test(thread_id, f2, f3); } +TEST_MT_FFF("parallel rpc with 1/1 transport threads and num_cores user threads (tls encryption + drop empty buffers)", + getNumThreads(), Server(tls_crypto, 1, true), Client(tls_crypto, 1, f1, true), Result(num_threads)) { perform_test(thread_id, f2, f3); } + TEST_MT_FFF("parallel rpc with 8/8 transport threads and num_cores user threads (no encryption)", - getNumThreads(), Server(null_crypto, 8), Client(null_crypto, 8, f1), Result(num_threads)) { perform_test(thread_id, f2, f3); } + getNumThreads(), Server(null_crypto, 8), Client(null_crypto, 8, f1), Result(num_threads)) { perform_test(thread_id, f2, f3, true); } TEST_MT_FFF("parallel rpc with 8/8 transport threads and num_cores user threads (xor encryption)", getNumThreads(), Server(xor_crypto, 8), Client(xor_crypto, 8, f1), Result(num_threads)) { perform_test(thread_id, f2, f3); } TEST_MT_FFF("parallel rpc with 8/8 transport threads and num_cores user threads (tls encryption)", - getNumThreads(), Server(tls_crypto, 8), Client(tls_crypto, 8, f1), Result(num_threads)) { perform_test(thread_id, f2, f3); } + getNumThreads(), Server(tls_crypto, 8), Client(tls_crypto, 8, f1), Result(num_threads)) { perform_test(thread_id, f2, f3, true); } -TEST_MAIN() { TEST_RUN_ALL(); } +TEST_MT_FFF("parallel rpc with 8/8 transport threads and num_cores user threads (tls encryption + drop empty buffers)", + getNumThreads(), Server(tls_crypto, 8, true), Client(tls_crypto, 8, f1, true), Result(num_threads)) { perform_test(thread_id, f2, f3); } + +//----------------------------------------------------------------------------- + +int main(int argc, char **argv) { + TEST_MASTER.init(__FILE__); + if ((argc == 2) && (argv[1] == std::string("verbose"))) { + verbose = true; + budget = 10.0; + } + TEST_RUN_ALL(); + return (TEST_MASTER.fini() ? 0 : 1); +} diff --git a/fnet/src/vespa/fnet/config.cpp b/fnet/src/vespa/fnet/config.cpp index 856d0ab4c1c..daf2b57429f 100644 --- a/fnet/src/vespa/fnet/config.cpp +++ b/fnet/src/vespa/fnet/config.cpp @@ -7,6 +7,7 @@ FNET_Config::FNET_Config() _events_before_wakeup(1), _maxInputBufferSize(0x10000), _maxOutputBufferSize(0x10000), - _tcpNoDelay(true) + _tcpNoDelay(true), + _drop_empty_buffers(false) { } diff --git a/fnet/src/vespa/fnet/config.h b/fnet/src/vespa/fnet/config.h index fd6f9f8557d..6cb1306a114 100644 --- a/fnet/src/vespa/fnet/config.h +++ b/fnet/src/vespa/fnet/config.h @@ -16,6 +16,7 @@ public: uint32_t _maxInputBufferSize; uint32_t _maxOutputBufferSize; bool _tcpNoDelay; + bool _drop_empty_buffers; FNET_Config(); }; diff --git a/fnet/src/vespa/fnet/connection.cpp b/fnet/src/vespa/fnet/connection.cpp index a2e6fe25edc..47d6a1e429a 100644 --- a/fnet/src/vespa/fnet/connection.cpp +++ b/fnet/src/vespa/fnet/connection.cpp @@ -347,6 +347,10 @@ done_read: } UpdateTimeOut(); + if (_flags._drop_empty_buffers) { + _socket->drop_empty_buffers(); + _input.Shrink(0); + } uint32_t maxSize = getConfig()._maxInputBufferSize; if (maxSize > 0 && _input.GetBufSize() > maxSize) { @@ -430,6 +434,10 @@ FNET_Connection::Write() } } + if (_flags._drop_empty_buffers) { + _socket->drop_empty_buffers(); + _output.Shrink(0); + } uint32_t maxSize = getConfig()._maxOutputBufferSize; if (maxSize > 0 && _output.GetBufSize() > maxSize) { _output.Shrink(maxSize); @@ -477,7 +485,7 @@ FNET_Connection::FNET_Connection(FNET_TransportThread *owner, _resolve_handler(nullptr), _context(), _state(FNET_CONNECTING), - _flags(), + _flags(owner->owner().getConfig()), _packetLength(0), _packetCode(0), _packetCHID(0), @@ -511,7 +519,7 @@ FNET_Connection::FNET_Connection(FNET_TransportThread *owner, _resolve_handler(nullptr), _context(context), _state(FNET_CONNECTING), - _flags(), + _flags(owner->owner().getConfig()), _packetLength(0), _packetCode(0), _packetCHID(0), diff --git a/fnet/src/vespa/fnet/connection.h b/fnet/src/vespa/fnet/connection.h index c9c49c5151a..3da9b58f928 100644 --- a/fnet/src/vespa/fnet/connection.h +++ b/fnet/src/vespa/fnet/connection.h @@ -2,6 +2,7 @@ #pragma once +#include "config.h" #include "iocomponent.h" #include "databuffer.h" #include "context.h" @@ -67,13 +68,14 @@ public: private: struct Flags { - Flags() : + Flags(const FNET_Config &cfg) : _gotheader(false), _inCallback(false), _callbackWait(false), _discarding(false), _framed(false), - _handshake_work_pending(false) + _handshake_work_pending(false), + _drop_empty_buffers(cfg._drop_empty_buffers) { } bool _gotheader; bool _inCallback; @@ -81,6 +83,7 @@ private: bool _discarding; bool _framed; bool _handshake_work_pending; + bool _drop_empty_buffers; }; struct ResolveHandler : public vespalib::AsyncResolver::ResultHandler { FNET_Connection *connection; diff --git a/fnet/src/vespa/fnet/transport.h b/fnet/src/vespa/fnet/transport.h index 766eaa3ccaa..6a59f9da66c 100644 --- a/fnet/src/vespa/fnet/transport.h +++ b/fnet/src/vespa/fnet/transport.h @@ -53,6 +53,11 @@ public: _config._tcpNoDelay = v; return *this; } + TransportConfig &drop_empty_buffers(bool v) { + _config._drop_empty_buffers = v; + return *this; + } + private: FNET_Config _config; vespalib::AsyncResolver::SP _resolver; diff --git a/jdisc_core/src/main/java/com/yahoo/jdisc/handler/ContentChannel.java b/jdisc_core/src/main/java/com/yahoo/jdisc/handler/ContentChannel.java index a1a4503eff2..b8b2d123a13 100644 --- a/jdisc_core/src/main/java/com/yahoo/jdisc/handler/ContentChannel.java +++ b/jdisc_core/src/main/java/com/yahoo/jdisc/handler/ContentChannel.java @@ -47,4 +47,15 @@ public interface ContentChannel { */ void close(CompletionHandler handler); + + /** + * Invoked when an error occurs during processing of request content. Signals that the caller was + * unable to write all data to this ContentChannel. + * + * This method can be invoked at any time after the content channel is created, but it's never invoked after {@link #close(CompletionHandler)}. + * {@link #close(CompletionHandler)} will be invoked immediately after this method returning + * (no intermediate calls to #{@link #write(ByteBuffer, CompletionHandler)}). + */ + default void onError(Throwable error) {} + } diff --git a/jrt/src/com/yahoo/jrt/Connection.java b/jrt/src/com/yahoo/jrt/Connection.java index 7393d30fc81..891558684ed 100644 --- a/jrt/src/com/yahoo/jrt/Connection.java +++ b/jrt/src/com/yahoo/jrt/Connection.java @@ -36,6 +36,7 @@ class Connection extends Target { private final Buffer output = new Buffer(0x1000); // Start off with small buffer. private int maxInputSize = 64*1024; private int maxOutputSize = 64*1024; + private boolean dropEmptyBuffers = false; private final boolean tcpNoDelay; private final Map<Integer, ReplyHandler> replyMap = new HashMap<>(); private final Map<TargetWatcher, TargetWatcher> watchers = new IdentityHashMap<>(); @@ -119,6 +120,10 @@ class Connection extends Target { maxOutputSize = bytes; } + public void setDropEmptyBuffers(boolean value) { + dropEmptyBuffers = value; + } + public TransportThread transportThread() { return parent; } @@ -307,6 +312,10 @@ class Connection extends Target { while (socket.drain(input.getChannelWritable(readSize)) > 0) { handlePackets(); } + if (dropEmptyBuffers) { + socket.dropEmptyBuffers(); + input.shrink(0); + } if (maxInputSize > 0) { input.shrink(maxInputSize); } @@ -363,6 +372,10 @@ class Connection extends Target { if (disableWrite) { disableWrite(); } + if (dropEmptyBuffers) { + socket.dropEmptyBuffers(); + output.shrink(0); + } if (maxOutputSize > 0) { output.shrink(maxOutputSize); } diff --git a/jrt/src/com/yahoo/jrt/CryptoSocket.java b/jrt/src/com/yahoo/jrt/CryptoSocket.java index e0489dec7a3..b7de30f9236 100644 --- a/jrt/src/com/yahoo/jrt/CryptoSocket.java +++ b/jrt/src/com/yahoo/jrt/CryptoSocket.java @@ -96,6 +96,13 @@ public interface CryptoSocket { public FlushResult flush() throws IOException; /** + * This function can be called at any time to drop any currently + * empty internal buffers. Typically called after drain or flush + * indicates that no further progress can be made. + **/ + public void dropEmptyBuffers(); + + /** * Returns the security context for the current connection (given handshake completed), * or empty if the current connection is not secure. */ diff --git a/jrt/src/com/yahoo/jrt/MaybeTlsCryptoSocket.java b/jrt/src/com/yahoo/jrt/MaybeTlsCryptoSocket.java index 60b7f342c9c..d84b33143f3 100644 --- a/jrt/src/com/yahoo/jrt/MaybeTlsCryptoSocket.java +++ b/jrt/src/com/yahoo/jrt/MaybeTlsCryptoSocket.java @@ -129,5 +129,6 @@ public class MaybeTlsCryptoSocket implements CryptoSocket { @Override public int drain(ByteBuffer dst) throws IOException { return socket.drain(dst); } @Override public int write(ByteBuffer src) throws IOException { return socket.write(src); } @Override public FlushResult flush() throws IOException { return socket.flush(); } + @Override public void dropEmptyBuffers() { socket.dropEmptyBuffers(); } @Override public Optional<SecurityContext> getSecurityContext() { return Optional.ofNullable(socket).flatMap(CryptoSocket::getSecurityContext); } } diff --git a/jrt/src/com/yahoo/jrt/NullCryptoSocket.java b/jrt/src/com/yahoo/jrt/NullCryptoSocket.java index 83359bb65a5..81995d69a40 100644 --- a/jrt/src/com/yahoo/jrt/NullCryptoSocket.java +++ b/jrt/src/com/yahoo/jrt/NullCryptoSocket.java @@ -30,4 +30,5 @@ public class NullCryptoSocket implements CryptoSocket { @Override public int drain(ByteBuffer dst) throws IOException { return 0; } @Override public int write(ByteBuffer src) throws IOException { return channel.write(src); } @Override public FlushResult flush() throws IOException { return FlushResult.DONE; } + @Override public void dropEmptyBuffers() {} } diff --git a/jrt/src/com/yahoo/jrt/Supervisor.java b/jrt/src/com/yahoo/jrt/Supervisor.java index bcd525c9596..d7c2c83ea69 100644 --- a/jrt/src/com/yahoo/jrt/Supervisor.java +++ b/jrt/src/com/yahoo/jrt/Supervisor.java @@ -23,6 +23,7 @@ public class Supervisor { private final AtomicReference<HashMap<String, Method>> methodMap = new AtomicReference<>(new HashMap<>()); private int maxInputBufferSize = 0; private int maxOutputBufferSize = 0; + private boolean dropEmptyBuffers = false; /** * Create a new Supervisor based on the given {@link Transport} @@ -46,6 +47,18 @@ public class Supervisor { } /** + * Drop empty buffers. This will reduce memory footprint for idle + * connections at the cost of extra allocations when buffer space + * is needed again. + * + * @param value true means drop empty buffers + **/ + public Supervisor setDropEmptyBuffers(boolean value) { + dropEmptyBuffers = value; + return this; + } + + /** * Set maximum input buffer size. This value will only affect * connections that use a common input buffer when decoding * incoming packets. Note that this value is not an absolute @@ -193,6 +206,7 @@ public class Supervisor { Connection conn = (Connection) target; conn.setMaxInputSize(maxInputBufferSize); conn.setMaxOutputSize(maxOutputBufferSize); + conn.setDropEmptyBuffers(dropEmptyBuffers); } SessionHandler handler = sessionHandler; if (handler != null) { diff --git a/jrt/src/com/yahoo/jrt/TlsCryptoSocket.java b/jrt/src/com/yahoo/jrt/TlsCryptoSocket.java index f140bb38c66..7ba83d6718e 100644 --- a/jrt/src/com/yahoo/jrt/TlsCryptoSocket.java +++ b/jrt/src/com/yahoo/jrt/TlsCryptoSocket.java @@ -216,6 +216,11 @@ public class TlsCryptoSocket implements CryptoSocket { return wrapBuffer.bytes() > 0 ? FlushResult.NEED_WRITE : FlushResult.DONE; } + @Override public void dropEmptyBuffers() { + wrapBuffer.shrink(0); + unwrapBuffer.shrink(0); + } + @Override public Optional<SecurityContext> getSecurityContext() { try { diff --git a/jrt/src/com/yahoo/jrt/XorCryptoSocket.java b/jrt/src/com/yahoo/jrt/XorCryptoSocket.java index 55f7b18c661..7477cd5816d 100644 --- a/jrt/src/com/yahoo/jrt/XorCryptoSocket.java +++ b/jrt/src/com/yahoo/jrt/XorCryptoSocket.java @@ -119,5 +119,8 @@ public class XorCryptoSocket implements CryptoSocket { return FlushResult.DONE; } } - + @Override public void dropEmptyBuffers() { + input.shrink(0); + output.shrink(0); + } } diff --git a/jrt/src/com/yahoo/jrt/slobrok/api/Mirror.java b/jrt/src/com/yahoo/jrt/slobrok/api/Mirror.java index 996459dc5db..058a1380480 100644 --- a/jrt/src/com/yahoo/jrt/slobrok/api/Mirror.java +++ b/jrt/src/com/yahoo/jrt/slobrok/api/Mirror.java @@ -31,16 +31,16 @@ import java.util.logging.Level; */ public class Mirror implements IMirror { - private static Logger log = Logger.getLogger(Mirror.class.getName()); + private static final Logger log = Logger.getLogger(Mirror.class.getName()); private final Supervisor orb; private final SlobrokList slobroks; - private String currSlobrok; - private final BackOffPolicy backOff; - private volatile int updates = 0; + private String currSlobrok; + private final BackOffPolicy backOff; + private volatile int updates = 0; private boolean requestDone = false; private boolean logOnSuccess = true; - private AtomicReference<Entry[]> specs = new AtomicReference<>(new Entry[0]); + private final AtomicReference<Entry[]> specs = new AtomicReference<>(new Entry[0]); private int specsGeneration = 0; private final TransportThread transportThread; private final Task updateTask; @@ -55,7 +55,7 @@ public class Mirror implements IMirror { * @param orb the Supervisor to use * @param slobroks slobrok connect spec list * @param bop custom backoff policy, mostly useful for testing - **/ + */ public Mirror(Supervisor orb, SlobrokList slobroks, BackOffPolicy bop) { this.orb = orb; this.slobroks = slobroks; diff --git a/jrt/src/com/yahoo/jrt/slobrok/api/SlobrokList.java b/jrt/src/com/yahoo/jrt/slobrok/api/SlobrokList.java index 10d8923d9f5..654ccd7e350 100644 --- a/jrt/src/com/yahoo/jrt/slobrok/api/SlobrokList.java +++ b/jrt/src/com/yahoo/jrt/slobrok/api/SlobrokList.java @@ -27,7 +27,6 @@ public class SlobrokList { } } - public String nextSlobrokSpec() { checkUpdate(); if (idx < slobroks.length) { @@ -90,4 +89,5 @@ public class SlobrokList { return Arrays.toString(slobroks); } } + } diff --git a/jrt/tests/com/yahoo/jrt/LatencyTest.java b/jrt/tests/com/yahoo/jrt/LatencyTest.java index c8ead8ebf77..1c736fb28ea 100644 --- a/jrt/tests/com/yahoo/jrt/LatencyTest.java +++ b/jrt/tests/com/yahoo/jrt/LatencyTest.java @@ -17,12 +17,15 @@ public class LatencyTest { private final Supervisor server; private final Supervisor client; private final Acceptor acceptor; - public Network(CryptoEngine crypto, int threads) throws ListenFailedException { + public Network(CryptoEngine crypto, int threads, boolean dropEmpty) throws ListenFailedException { server = new Supervisor(new Transport("server", crypto, threads)); client = new Supervisor(new Transport("client", crypto, threads)); + server.setDropEmptyBuffers(dropEmpty); + client.setDropEmptyBuffers(dropEmpty); server.addMethod(new Method("inc", "i", "i", this::rpc_inc)); acceptor = server.listen(new Spec(0)); } + public Network(CryptoEngine crypto, int threads) throws ListenFailedException { this(crypto, threads, false); } public Target connect() { return client.connect(new Spec("localhost", acceptor.port())); } @@ -188,6 +191,13 @@ public class LatencyTest { } @org.junit.Test + public void testTlsCryptoWithDropEmptyBuffersLatency() throws Throwable { + try (Network network = new Network(new TlsCryptoEngine(createTestTlsContext()), 1, true)) { + new Client(false, network, 1).measureLatency("[tls crypto, drop empty, no reconnect] "); + } + } + + @org.junit.Test public void testTransportThreadScaling() throws Throwable { try (Network network = new Network(new NullCryptoEngine(), 1)) { new Client(false, network, 64).measureLatency("[64 clients, 1/1 transport] "); diff --git a/messagebus/src/main/java/com/yahoo/messagebus/network/rpc/SlobrokConfigSubscriber.java b/messagebus/src/main/java/com/yahoo/messagebus/network/rpc/SlobrokConfigSubscriber.java index 7f4c27a45f9..d576ec50af7 100755 --- a/messagebus/src/main/java/com/yahoo/messagebus/network/rpc/SlobrokConfigSubscriber.java +++ b/messagebus/src/main/java/com/yahoo/messagebus/network/rpc/SlobrokConfigSubscriber.java @@ -12,13 +12,13 @@ import com.yahoo.cloud.config.SlobroksConfig; */ public class SlobrokConfigSubscriber implements ConfigSubscriber.SingleSubscriber<SlobroksConfig>{ - private SlobrokList slobroks = new SlobrokList(); + private final SlobrokList slobroks = new SlobrokList(); private ConfigSubscriber subscriber; /** * Constructs a new config subscriber for a given config id. * - * @param configId The id of the config to subscribe to. + * @param configId the id of the config to subscribe to */ public SlobrokConfigSubscriber(String configId) { subscriber = new ConfigSubscriber(); @@ -55,4 +55,5 @@ public class SlobrokConfigSubscriber implements ConfigSubscriber.SingleSubscribe subscriber.close(); } } + } diff --git a/metrics/src/vespa/metrics/countmetric.h b/metrics/src/vespa/metrics/countmetric.h index 02a6827d1ce..1701071104e 100644 --- a/metrics/src/vespa/metrics/countmetric.h +++ b/metrics/src/vespa/metrics/countmetric.h @@ -105,7 +105,7 @@ public: void addToSnapshot(Metric&, std::vector<Metric::UP> &) const override; }; -typedef CountMetric<uint64_t, true> LongCountMetric; +using LongCountMetric = CountMetric<uint64_t, true>; } // metrics diff --git a/metrics/src/vespa/metrics/metricvalueset.h b/metrics/src/vespa/metrics/metricvalueset.h index 2463990378e..c522876f5b1 100644 --- a/metrics/src/vespa/metrics/metricvalueset.h +++ b/metrics/src/vespa/metrics/metricvalueset.h @@ -76,12 +76,6 @@ public: */ bool setValues(const ValueClass& values); - /** - * Retrieve and reset in a single operation, to minimize chance of - * alteration in the process. - */ - ValueClass getValuesAndReset(); - void reset() { setFlag(RESET); } @@ -105,9 +99,6 @@ public: _flags.store(_flags.load(std::memory_order_relaxed) & ~flags, std::memory_order_relaxed); } - uint32_t getFlags() const { - return _flags.load(std::memory_order_relaxed); - } }; } // metrics diff --git a/metrics/src/vespa/metrics/metricvalueset.hpp b/metrics/src/vespa/metrics/metricvalueset.hpp index 8c5b32afcf8..57b3e7f9901 100644 --- a/metrics/src/vespa/metrics/metricvalueset.hpp +++ b/metrics/src/vespa/metrics/metricvalueset.hpp @@ -70,14 +70,6 @@ MetricValueSet<ValueClass>::setValues(const ValueClass& values) { } template<typename ValueClass> -ValueClass -MetricValueSet<ValueClass>::getValuesAndReset() { - ValueClass result(getValues()); - setFlag(RESET); - return result; -} - -template<typename ValueClass> std::string MetricValueSet<ValueClass>::toString() { std::ostringstream ost; diff --git a/node-admin/src/main/application/services.xml b/node-admin/src/main/application/services.xml index 7e1d45c3ff2..d978b358032 100644 --- a/node-admin/src/main/application/services.xml +++ b/node-admin/src/main/application/services.xml @@ -3,7 +3,7 @@ <services version="1.0" xmlns:preprocess="properties"> <container id="node-admin" version="1.0"> <!-- Please update container test when changing this file --> - <accesslog type="json" fileNamePattern="logs/vespa/node-admin/access-json.log.%Y%m%d%H%M%S" symlinkName="access-json.log" compressOnRotation="true" compressionType="zstd"/> + <accesslog type="json" fileNamePattern="logs/vespa/node-admin/access-json.log.%Y%m%d%H%M%S" symlinkName="access-json.log" compressOnRotation="true" compressionType="zstd" bufferSize='262144' queueSize='1024'/> <component id="docker-api" class="com.yahoo.vespa.hosted.dockerapi.DockerEngine" bundle="docker-api"/> <component id="metrics" class="com.yahoo.vespa.hosted.dockerapi.metrics.Metrics" bundle="docker-api"/> diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/StorageMaintainer.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/StorageMaintainer.java index 10e0dd50761..9ced178cff9 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/StorageMaintainer.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/StorageMaintainer.java @@ -188,6 +188,10 @@ public class StorageMaintainer { attributes.put("application", owner.application().value()); attributes.put("instance", owner.instance().value()); }); + context.node().membership().ifPresent(membership -> { + attributes.put("cluster_id", membership.clusterId()); + attributes.put("cluster_type", membership.type().value()); + }); return Collections.unmodifiableMap(attributes); } diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/coredump/CoredumpHandler.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/coredump/CoredumpHandler.java index 09c0a4ae491..7f79efdeab9 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/coredump/CoredumpHandler.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/coredump/CoredumpHandler.java @@ -174,7 +174,10 @@ public class CoredumpHandler { Path coredumpFilePathInContainer = context.pathInNodeFromPathOnHost(coredumpFilePathOnHost); Map<String, Object> metadata = new HashMap<>(coreCollector.collect(context, coredumpFilePathInContainer)); metadata.putAll(nodeAttributesSupplier.get()); - metadata.put("coredump_path", doneCoredumpsPath.resolve(context.containerName().asString()).resolve(coredumpDirectory.getFileName()).toString()); + metadata.put("coredump_path", doneCoredumpsPath + .resolve(context.containerName().asString()) + .resolve(coredumpDirectory.getFileName()) + .resolve(coredumpFilePathOnHost.getFileName()).toString()); String metadataFields = objectMapper.writeValueAsString(Map.of("fields", metadata)); metadataPath.writeUtf8File(metadataFields); diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/NodeAdmin.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/NodeAdmin.java index 6d9eae5c4dc..e510618c5a4 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/NodeAdmin.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/NodeAdmin.java @@ -4,7 +4,6 @@ package com.yahoo.vespa.hosted.node.admin.nodeadmin; import com.yahoo.vespa.hosted.node.admin.nodeagent.NodeAgentContext; import java.time.Duration; -import java.util.List; import java.util.Set; /** @@ -41,10 +40,9 @@ public interface NodeAdmin { Duration subsystemFreezeDuration(); /** - * Stop services on these nodes - * @param nodes List of hostnames to suspend + * Stop all services on these nodes */ - void stopNodeAgentServices(List<String> nodes); + void stopNodeAgentServices(); /** * Start node-admin schedulers. diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/NodeAdminImpl.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/NodeAdminImpl.java index caffe5ef2f1..ef9520969af 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/NodeAdminImpl.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/NodeAdminImpl.java @@ -15,7 +15,6 @@ import java.time.Clock; import java.time.Duration; import java.time.Instant; import java.util.HashSet; -import java.util.List; import java.util.Map; import java.util.Set; import java.util.concurrent.ConcurrentHashMap; @@ -157,12 +156,11 @@ public class NodeAdminImpl implements NodeAdmin { } @Override - public void stopNodeAgentServices(List<String> hostnames) { + public void stopNodeAgentServices() { // Each container may spend 1-1:30 minutes stopping - hostnames.parallelStream() - .filter(nodeAgentWithSchedulerByHostname::containsKey) - .map(nodeAgentWithSchedulerByHostname::get) - .forEach(NodeAgentWithScheduler::stopForHostSuspension); + nodeAgentWithSchedulerByHostname.values() + .parallelStream() + .forEach(NodeAgentWithScheduler::stopForHostSuspension); } @Override diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/NodeAdminStateUpdater.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/NodeAdminStateUpdater.java index 58ca4ae3f41..c24b2261f42 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/NodeAdminStateUpdater.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/NodeAdminStateUpdater.java @@ -150,7 +150,7 @@ public class NodeAdminStateUpdater { // The node agent services are stopped by this thread, which is OK only // because the node agents are frozen (see above). - nodeAdmin.stopNodeAgentServices(nodesInActiveState); + nodeAdmin.stopNodeAgentServices(); break; default: throw new IllegalStateException("Unknown wanted state " + wantedState); diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/coredump/CoredumpHandlerTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/coredump/CoredumpHandlerTest.java index 4f2f2f985b6..6dd7241fd63 100644 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/coredump/CoredumpHandlerTest.java +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/coredump/CoredumpHandlerTest.java @@ -159,7 +159,7 @@ public class CoredumpHandlerTest { "\"backtrace\":[\"call 1\",\"function 2\",\"something something\"]," + "\"vespa_version\":\"6.48.4\"," + "\"bin_path\":\"/bin/bash\"," + - "\"coredump_path\":\"/home/docker/dumps/container-123/id-123\"," + + "\"coredump_path\":\"/home/docker/dumps/container-123/id-123/dump_core.456\"," + "\"docker_image\":\"vespa/ci:6.48.4\"" + "}}"; diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/NodeAdminStateUpdaterTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/NodeAdminStateUpdaterTest.java index 4a678597e41..8ee3a95744b 100644 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/NodeAdminStateUpdaterTest.java +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/NodeAdminStateUpdaterTest.java @@ -102,7 +102,7 @@ public class NodeAdminStateUpdaterTest { // At this point orchestrator will say its OK to suspend, but something goes wrong when we try to stop services final String exceptionMessage = "Failed to stop services"; verify(orchestrator, times(0)).suspend(eq(hostHostname.value()), eq(suspendHostnames)); - doThrow(new RuntimeException(exceptionMessage)).doNothing().when(nodeAdmin).stopNodeAgentServices(eq(activeHostnames)); + doThrow(new RuntimeException(exceptionMessage)).doNothing().when(nodeAdmin).stopNodeAgentServices(); assertConvergeError(SUSPENDED, exceptionMessage); verify(orchestrator, times(1)).suspend(eq(hostHostname.value()), eq(suspendHostnames)); // Make sure we dont roll back if we fail to stop services - we will try to stop again next tick diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/NodeRepository.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/NodeRepository.java index d113ca68d01..f084b83bf97 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/NodeRepository.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/NodeRepository.java @@ -109,7 +109,7 @@ public class NodeRepository extends AbstractComponent { "dynamicProvisioning property must be 1-to-1 with availability of HostProvisioner, was: dynamicProvisioning=%s, hostProvisioner=%s", zone.getCloud().dynamicProvisioning(), provisionServiceProvider.getHostProvisioner().map(__ -> "present").orElse("empty"))); - this.db = new CuratorDatabaseClient(flavors, curator, clock, zone, useCuratorClientCache, nodeCacheSize); + this.db = new CuratorDatabaseClient(flavors, curator, clock, useCuratorClientCache, nodeCacheSize); this.zone = zone; this.clock = clock; this.nodes = new Nodes(db, zone, clock); diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/applications/Application.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/applications/Application.java index 5eb01b4fe72..c8d5e4361a5 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/applications/Application.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/applications/Application.java @@ -62,7 +62,7 @@ public class Application { public Application withCluster(ClusterSpec.Id id, boolean exclusive, ClusterResources min, ClusterResources max) { Cluster cluster = clusters.get(id); if (cluster == null) - cluster = new Cluster(id, exclusive, min, max, Optional.empty(), Optional.empty(), List.of(), ""); + cluster = new Cluster(id, exclusive, min, max, Optional.empty(), Optional.empty(), List.of(), AutoscalingStatus.empty()); else cluster = cluster.withConfiguration(exclusive, min, max); return with(cluster); diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/applications/AutoscalingStatus.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/applications/AutoscalingStatus.java new file mode 100644 index 00000000000..c40408c9109 --- /dev/null +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/applications/AutoscalingStatus.java @@ -0,0 +1,69 @@ +package com.yahoo.vespa.hosted.provision.applications; + +import com.yahoo.vespa.hosted.provision.autoscale.Autoscaler; + +import java.util.Objects; + +/** + * The current autoscaling status of a cluster. + * A value object. + * + * @author bratseth + */ +public class AutoscalingStatus { + + public enum Status { + + /** No status is available: Aautoscaling is disabled, or a brand new application. */ + unavailable, + + /** Autoscaling is not taking any action at the moment due to recent changes or a lack of data */ + waiting, + + /** The cluster is ideally scaled to the current load */ + ideal, + + /** The cluster should be rescaled further, but no better configuration is allowed by the current limits */ + insufficient, + + /** Rescaling of this cluster has been scheduled */ + rescaling + + }; + + private final Status status; + private final String description; + + public AutoscalingStatus(Status status, String description) { + this.status = status; + this.description = description; + } + + public Status status() { return status; } + public String description() { return description; } + + public static AutoscalingStatus empty() { return new AutoscalingStatus(Status.unavailable, ""); } + + @Override + public boolean equals(Object o) { + if (o == this) return true; + if ( ! ( o instanceof AutoscalingStatus)) return false; + + AutoscalingStatus other = (AutoscalingStatus)o; + if ( other.status != this.status ) return false; + if ( ! other.description.equals(this.description) ) return false; + return true; + } + + @Override + public int hashCode() { + return Objects.hash(status, description); + } + + @Override + public String toString() { + return "autoscaling status: " + status + + ( description.isEmpty() ? "" : " (" + description + ")"); + } + +} diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/applications/Cluster.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/applications/Cluster.java index 59b70ff1ef0..d4bbe6adc1b 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/applications/Cluster.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/applications/Cluster.java @@ -5,7 +5,6 @@ import com.yahoo.config.provision.ClusterResources; import com.yahoo.config.provision.ClusterSpec; import com.yahoo.vespa.hosted.provision.autoscale.Autoscaler; -import java.time.Duration; import java.time.Instant; import java.util.ArrayList; import java.util.List; @@ -31,7 +30,7 @@ public class Cluster { /** The maxScalingEvents last scaling events of this, sorted by increasing time (newest last) */ private final List<ScalingEvent> scalingEvents; - private final String autoscalingStatus; + private final AutoscalingStatus autoscalingStatus; public Cluster(ClusterSpec.Id id, boolean exclusive, @@ -40,7 +39,7 @@ public class Cluster { Optional<Suggestion> suggestedResources, Optional<ClusterResources> targetResources, List<ScalingEvent> scalingEvents, - String autoscalingStatus) { + AutoscalingStatus autoscalingStatus) { this.id = Objects.requireNonNull(id); this.exclusive = exclusive; this.min = Objects.requireNonNull(minResources); @@ -95,8 +94,8 @@ public class Cluster { return Optional.of(scalingEvents.get(scalingEvents.size() - 1)); } - /** The latest autoscaling status of this cluster, or empty (never null) if none */ - public String autoscalingStatus() { return autoscalingStatus; } + /** The latest autoscaling status of this cluster, or unknown (never null) if none */ + public AutoscalingStatus autoscalingStatus() { return autoscalingStatus; } public Cluster withConfiguration(boolean exclusive, ClusterResources min, ClusterResources max) { return new Cluster(id, exclusive, min, max, suggested, target, scalingEvents, autoscalingStatus); @@ -124,7 +123,7 @@ public class Cluster { return new Cluster(id, exclusive, min, max, suggested, target, scalingEvents, autoscalingStatus); } - public Cluster withAutoscalingStatus(String autoscalingStatus) { + public Cluster with(AutoscalingStatus autoscalingStatus) { if (autoscalingStatus.equals(this.autoscalingStatus)) return this; return new Cluster(id, exclusive, min, max, suggested, target, scalingEvents, autoscalingStatus); } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/AllocationOptimizer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/AllocationOptimizer.java index 0a1c6c5df6b..6eaae755708 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/AllocationOptimizer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/AllocationOptimizer.java @@ -6,7 +6,6 @@ import com.yahoo.config.provision.ClusterSpec; import com.yahoo.config.provision.NodeResources; import com.yahoo.vespa.hosted.provision.NodeList; import com.yahoo.vespa.hosted.provision.NodeRepository; -import com.yahoo.vespa.hosted.provision.applications.Cluster; import java.util.Optional; @@ -48,7 +47,7 @@ public class AllocationOptimizer { limits = Limits.of(new ClusterResources(minimumNodes, 1, NodeResources.unspecified()), new ClusterResources(maximumNodes, maximumNodes, NodeResources.unspecified())); else - limits = atLeast(minimumNodes, limits); + limits = atLeast(minimumNodes, limits).fullySpecified(current.clusterSpec().type(), nodeRepository); Optional<AllocatableClusterResources> bestAllocation = Optional.empty(); NodeList hosts = nodeRepository.nodes().list().hosts(); for (int groups = limits.min().groups(); groups <= limits.max().groups(); groups++) { diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/Autoscaler.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/Autoscaler.java index 1d0ba3da6c5..ccaf23c49ed 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/Autoscaler.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/Autoscaler.java @@ -7,6 +7,8 @@ import com.yahoo.vespa.hosted.provision.Node; import com.yahoo.vespa.hosted.provision.NodeList; import com.yahoo.vespa.hosted.provision.NodeRepository; import com.yahoo.vespa.hosted.provision.applications.Application; +import com.yahoo.vespa.hosted.provision.applications.AutoscalingStatus; +import com.yahoo.vespa.hosted.provision.applications.AutoscalingStatus.Status; import com.yahoo.vespa.hosted.provision.applications.Cluster; import java.time.Duration; @@ -52,7 +54,8 @@ public class Autoscaler { * @return scaling advice for this cluster */ public Advice autoscale(Application application, Cluster cluster, NodeList clusterNodes) { - if (cluster.minResources().equals(cluster.maxResources())) return Advice.none("Autoscaling is not enabled"); + if (cluster.minResources().equals(cluster.maxResources())) + return Advice.none(Status.unavailable, "Autoscaling is not enabled"); return autoscale(application, cluster, clusterNodes, Limits.of(cluster)); } @@ -65,17 +68,20 @@ public class Autoscaler { nodeRepository.clock()); if ( ! clusterIsStable(clusterNodes, nodeRepository)) - return Advice.none("Cluster change in progress"); + return Advice.none(Status.waiting, "Cluster change in progress"); if (scaledIn(clusterModel.scalingDuration(), cluster)) - return Advice.dontScale("Won't autoscale now: Less than " + clusterModel.scalingDuration() + " since last resource change"); + return Advice.dontScale(Status.waiting, + "Won't autoscale now: Less than " + clusterModel.scalingDuration() + " since last resource change"); if (clusterModel.nodeTimeseries().measurementsPerNode() < minimumMeasurementsPerNode(clusterModel.scalingDuration())) - return Advice.none("Collecting more data before making new scaling decisions: Need to measure for " + + return Advice.none(Status.waiting, + "Collecting more data before making new scaling decisions: Need to measure for " + clusterModel.scalingDuration() + " since the last resource change completed"); if (clusterModel.nodeTimeseries().nodesMeasured() != clusterNodes.size()) - return Advice.none("Collecting more data before making new scaling decisions: " + + return Advice.none(Status.waiting, + "Collecting more data before making new scaling decisions: " + "Have measurements from " + clusterModel.nodeTimeseries().nodesMeasured() + " nodes, but require from " + clusterNodes.size()); @@ -85,13 +91,18 @@ public class Autoscaler { Optional<AllocatableClusterResources> bestAllocation = allocationOptimizer.findBestAllocation(target, currentAllocation, clusterModel, limits); if (bestAllocation.isEmpty()) - return Advice.dontScale("No allocation improvements are possible within configured limits"); + return Advice.dontScale(Status.insufficient, "No allocations are possible within configured limits"); - if (similar(bestAllocation.get().realResources(), currentAllocation.realResources())) - return Advice.dontScale("Cluster is ideally scaled within configured limits"); + if (similar(bestAllocation.get().realResources(), currentAllocation.realResources())) { + if (bestAllocation.get().fulfilment() < 1) + return Advice.dontScale(Status.insufficient, "Configured limits prevents better scaling of this cluster"); + else + return Advice.dontScale(Status.ideal, "Cluster is ideally scaled"); + } if (isDownscaling(bestAllocation.get(), currentAllocation) && scaledIn(clusterModel.scalingDuration().multipliedBy(3), cluster)) - return Advice.dontScale("Waiting " + clusterModel.scalingDuration().multipliedBy(3) + + return Advice.dontScale(Status.waiting, + "Waiting " + clusterModel.scalingDuration().multipliedBy(3) + " since the last change before reducing resources"); return Advice.scaleTo(bestAllocation.get().advertisedResources()); @@ -154,9 +165,9 @@ public class Autoscaler { private final boolean present; private final Optional<ClusterResources> target; - private final String reason; + private final AutoscalingStatus reason; - private Advice(Optional<ClusterResources> target, boolean present, String reason) { + private Advice(Optional<ClusterResources> target, boolean present, AutoscalingStatus reason) { this.target = target; this.present = present; this.reason = Objects.requireNonNull(reason); @@ -175,12 +186,20 @@ public class Autoscaler { public boolean isPresent() { return present; } /** The reason for this advice */ - public String reason() { return reason; } + public AutoscalingStatus reason() { return reason; } + + private static Advice none(Status status, String description) { + return new Advice(Optional.empty(), false, new AutoscalingStatus(status, description)); + } + + private static Advice dontScale(Status status, String description) { + return new Advice(Optional.empty(), true, new AutoscalingStatus(status, description)); + } - private static Advice none(String reason) { return new Advice(Optional.empty(), false, reason); } - private static Advice dontScale(String reason) { return new Advice(Optional.empty(), true, reason); } private static Advice scaleTo(ClusterResources target) { - return new Advice(Optional.of(target), true, "Scheduled scaling to " + target + " due to load changes"); + return new Advice(Optional.of(target), true, + new AutoscalingStatus(AutoscalingStatus.Status.rescaling, + "Scheduled scaling to " + target + " due to load changes")); } @Override diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/ClusterModel.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/ClusterModel.java index e4b85b5317e..bbac3bf93da 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/ClusterModel.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/ClusterModel.java @@ -9,7 +9,10 @@ import com.yahoo.vespa.hosted.provision.applications.ScalingEvent; import java.time.Clock; import java.time.Duration; +import java.util.Optional; import java.util.OptionalDouble; +import java.util.logging.Level; +import java.util.logging.Logger; /** * A cluster with its associated metrics which allows prediction about its future behavior. @@ -19,6 +22,8 @@ import java.util.OptionalDouble; */ public class ClusterModel { + private static final Logger log = Logger.getLogger(ClusterModel.class.getName()); + private static final Duration CURRENT_LOAD_DURATION = Duration.ofMinutes(5); static final double idealQueryCpuLoad = 0.8; @@ -27,18 +32,16 @@ public class ClusterModel { static final double idealDiskLoad = 0.6; private final Application application; - private final Cluster cluster; /** The current nodes of this cluster, or empty if this models a new cluster not yet deployed */ private final NodeList nodes; - private final MetricsDb metricsDb; private final Clock clock; private final Duration scalingDuration; + private final ClusterTimeseries clusterTimeseries; + private final ClusterNodesTimeseries nodeTimeseries; // Lazily initialized members private Double queryFractionOfMax = null; private Double maxQueryGrowthRate = null; - private ClusterNodesTimeseries nodeTimeseries = null; - private ClusterTimeseries clusterTimeseries = null; public ClusterModel(Application application, Cluster cluster, @@ -47,11 +50,11 @@ public class ClusterModel { MetricsDb metricsDb, Clock clock) { this.application = application; - this.cluster = cluster; this.nodes = clusterNodes; - this.metricsDb = metricsDb; this.clock = clock; this.scalingDuration = computeScalingDuration(cluster, clusterSpec); + this.clusterTimeseries = metricsDb.getClusterTimeseries(application.id(), cluster.id()); + this.nodeTimeseries = new ClusterNodesTimeseries(scalingDuration(), cluster, nodes, metricsDb); } /** For testing */ @@ -59,29 +62,23 @@ public class ClusterModel { Cluster cluster, Clock clock, Duration scalingDuration, - ClusterTimeseries clusterTimeseries) { + ClusterTimeseries clusterTimeseries, + ClusterNodesTimeseries nodeTimeseries) { this.application = application; - this.cluster = cluster; this.nodes = null; - this.metricsDb = null; this.clock = clock; this.scalingDuration = scalingDuration; this.clusterTimeseries = clusterTimeseries; + this.nodeTimeseries = nodeTimeseries; } /** Returns the predicted duration of a rescaling of this cluster */ public Duration scalingDuration() { return scalingDuration; } - public ClusterNodesTimeseries nodeTimeseries() { - if (nodeTimeseries != null) return nodeTimeseries; - return nodeTimeseries = new ClusterNodesTimeseries(scalingDuration(), cluster, nodes, metricsDb); - } + public ClusterNodesTimeseries nodeTimeseries() { return nodeTimeseries; } - public ClusterTimeseries clusterTimeseries() { - if (clusterTimeseries != null) return clusterTimeseries; - return clusterTimeseries = metricsDb.getClusterTimeseries(application.id(), cluster.id()); - } + public ClusterTimeseries clusterTimeseries() { return clusterTimeseries; } /** * Returns the predicted max query growth rate per minute as a fraction of the average traffic @@ -188,4 +185,24 @@ public class ClusterModel { return duration; } + /** + * Create a cluster model if possible and logs a warning and returns empty otherwise. + * This is useful in cases where it's possible to continue without the cluser model, + * as QuestDb is known to temporarily fail during reading of data. + */ + public static Optional<ClusterModel> create(Application application, + Cluster cluster, + ClusterSpec clusterSpec, + NodeList clusterNodes, + MetricsDb metricsDb, + Clock clock) { + try { + return Optional.of(new ClusterModel(application, cluster, clusterSpec, clusterNodes, metricsDb, clock)); + } + catch (Exception e) { + log.log(Level.WARNING, "Failed creating a cluster model for " + application + " " + cluster, e); + return Optional.empty(); + } + } + } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/ClusterNodesTimeseries.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/ClusterNodesTimeseries.java index 5f974abd84f..9b77d8b48c8 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/ClusterNodesTimeseries.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/ClusterNodesTimeseries.java @@ -34,6 +34,11 @@ public class ClusterNodesTimeseries { this.timeseries = timeseries; } + private ClusterNodesTimeseries(NodeList clusterNodes, List<NodeTimeseries> timeseries) { + this.clusterNodes = clusterNodes; + this.timeseries = timeseries; + } + /** Returns the average number of measurements per node */ public int measurementsPerNode() { int measurementCount = timeseries.stream().mapToInt(m -> m.size()).sum(); @@ -61,4 +66,8 @@ public class ClusterNodesTimeseries { return timeseries.stream().map(nodeTimeseries -> nodeTimeseries.filter(filter)).collect(Collectors.toList()); } + public static ClusterNodesTimeseries empty() { + return new ClusterNodesTimeseries(NodeList.of(), List.of()); + } + } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/Limits.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/Limits.java index 80ad81f6cdf..cafea4b0eaf 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/Limits.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/Limits.java @@ -3,8 +3,11 @@ package com.yahoo.vespa.hosted.provision.autoscale; import com.yahoo.config.provision.Capacity; import com.yahoo.config.provision.ClusterResources; +import com.yahoo.config.provision.ClusterSpec; import com.yahoo.config.provision.NodeResources; +import com.yahoo.vespa.hosted.provision.NodeRepository; import com.yahoo.vespa.hosted.provision.applications.Cluster; +import com.yahoo.vespa.hosted.provision.provisioning.CapacityPolicies; import java.util.Objects; @@ -56,6 +59,15 @@ public class Limits { return resources; } + public Limits fullySpecified(ClusterSpec.Type type, NodeRepository nodeRepository) { + if (this.isEmpty()) throw new IllegalStateException("Unspecified limits can not be made fully specified"); + + var defaultResources = new CapacityPolicies(nodeRepository).defaultNodeResources(type); + var specifiedMin = min.nodeResources().isUnspecified() ? min.with(defaultResources) : min; + var specifiedMax = max.nodeResources().isUnspecified() ? max.with(defaultResources) : max; + return new Limits(specifiedMin, specifiedMax); + } + private double between(double min, double max, double value) { value = Math.max(min, value); value = Math.min(max, value); diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/MetricsResponse.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/MetricsResponse.java index 665155ecf68..3a037efed98 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/MetricsResponse.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/MetricsResponse.java @@ -112,64 +112,112 @@ public class MetricsResponse { private enum Metric { cpu { // a node resource + + @Override public List<String> metricResponseNames() { return List.of("cpu.util"); } - double computeFinal(List<Double> values) { - return values.stream().mapToDouble(v -> v).average().orElse(0) / 100; // % to ratio + + @Override + double computeFinal(ListMap<String, Double> values) { + return values.values().stream().flatMap(List::stream).mapToDouble(v -> v).average().orElse(0) / 100; // % to ratio } + }, memory { // a node resource - public List<String> metricResponseNames() { return List.of("mem.util"); } - double computeFinal(List<Double> values) { - return values.stream().mapToDouble(v -> v).average().orElse(0) / 100; // % to ratio + + @Override + public List<String> metricResponseNames() { + return List.of("content.proton.resource_usage.memory.average", "mem.util"); } + + @Override + double computeFinal(ListMap<String, Double> values) { + var valueList = values.get("content.proton.resource_usage.memory.average"); // prefer over mem.util + if ( ! valueList.isEmpty()) return valueList.get(0); + + valueList = values.get("mem.util"); + if ( ! valueList.isEmpty()) return valueList.get(0) / 100; // % to ratio + + return 0; + } + }, disk { // a node resource - public List<String> metricResponseNames() { return List.of("disk.util"); } - double computeFinal(List<Double> values) { - return values.stream().mapToDouble(v -> v).average().orElse(0) / 100; // % to ratio + + @Override + public List<String> metricResponseNames() { + return List.of("content.proton.resource_usage.disk.average", "disk.util"); } + + @Override + double computeFinal(ListMap<String, Double> values) { + var valueList = values.get("content.proton.resource_usage.disk.average"); // prefer over mem.util + if ( ! valueList.isEmpty()) return valueList.get(0); + + valueList = values.get("disk.util"); + if ( ! valueList.isEmpty()) return valueList.get(0) / 100; // % to ratio + + return 0; + } + }, generation { // application config generation active on the node + + @Override public List<String> metricResponseNames() { return List.of("application_generation"); } - double computeFinal(List<Double> values) { - return values.stream().mapToDouble(v -> v).min().orElse(-1); + + @Override + double computeFinal(ListMap<String, Double> values) { + return values.values().stream().flatMap(List::stream).mapToDouble(v -> v).min().orElse(-1); } + }, inService { + + @Override public List<String> metricResponseNames() { return List.of("in_service"); } - double computeFinal(List<Double> values) { + + @Override + double computeFinal(ListMap<String, Double> values) { // Really a boolean. Default true. If any is oos -> oos. - return values.stream().anyMatch(v -> v == 0) ? 0 : 1; + return values.values().stream().flatMap(List::stream).anyMatch(v -> v == 0) ? 0 : 1; } + }, queryRate { // queries per second + + @Override public List<String> metricResponseNames() { return List.of("queries.rate", "content.proton.documentdb.matching.queries.rate"); } + }, writeRate { // writes per second + + @Override public List<String> metricResponseNames() { return List.of("feed.http-requests.rate", "vds.filestor.alldisks.allthreads.put.sum.count.rate", "vds.filestor.alldisks.allthreads.remove.sum.count.rate", "vds.filestor.alldisks.allthreads.update.sum.count.rate"); } + }; - /** The name of this metric as emitted from its source */ + /** + * The names of this metric as emitted from its source. + * A map of the values of these names which were present in the response will + * be provided to computeFinal to decide on a single value. + */ public abstract List<String> metricResponseNames(); - double computeFinal(List<Double> values) { return values.stream().mapToDouble(v -> v).sum(); } + /** Computes the final metric value */ + double computeFinal(ListMap<String, Double> values) { + return values.values().stream().flatMap(List::stream).mapToDouble(v -> v).sum(); + } public double from(ListMap<String, Double> metricValues) { - // Multiple metric names may contribute to the same logical metric. - // Usually one per service, but we aggregate here to not require that. - List<Double> values = new ArrayList<>(1); - for (String metricName : metricResponseNames()) { - List<Double> valuesForName = metricValues.get(metricName); - if (valuesForName == null) continue; - values.addAll(valuesForName); - } + ListMap<String, Double> values = new ListMap<>(metricValues); + values.keySet().retainAll(metricResponseNames()); return computeFinal(values); } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/QuestMetricsDb.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/QuestMetricsDb.java index 2c29b9cc560..dc9d5b22f5a 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/QuestMetricsDb.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/QuestMetricsDb.java @@ -80,6 +80,10 @@ public class QuestMetricsDb extends AbstractComponent implements MetricsDb { IOUtils.createDirectory(dataDir + "/" + nodeTable); IOUtils.createDirectory(dataDir + "/" + clusterTable); + // https://stackoverflow.com/questions/67785629/what-does-max-txn-txn-inflight-limit-reached-in-questdb-and-how-to-i-avoid-it + new File(dataDir + "/" + nodeTable + "/_txn_scoreboard").delete(); + new File(dataDir + "/" + clusterTable + "/_txn_scoreboard").delete(); + // silence Questdb's custom logging system IOUtils.writeFile(new File(dataDir, "quest-log.conf"), new byte[0]); System.setProperty("out", dataDir + "/quest-log.conf"); diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/ApplicationMaintainer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/ApplicationMaintainer.java index 9ac1ca2b4c1..24160c19dfa 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/ApplicationMaintainer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/ApplicationMaintainer.java @@ -41,9 +41,9 @@ public abstract class ApplicationMaintainer extends NodeRepositoryMaintainer { } @Override - protected final boolean maintain() { + protected final double maintain() { applicationsNeedingMaintenance().forEach(this::deploy); - return true; + return 1.0; } /** Returns the number of deployments that are pending execution */ diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/AutoscalingMaintainer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/AutoscalingMaintainer.java index 7da6e0d3ebe..3914a0c9f07 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/AutoscalingMaintainer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/AutoscalingMaintainer.java @@ -14,16 +14,13 @@ import com.yahoo.vespa.hosted.provision.applications.Applications; import com.yahoo.vespa.hosted.provision.applications.Cluster; import com.yahoo.vespa.hosted.provision.autoscale.AllocatableClusterResources; import com.yahoo.vespa.hosted.provision.autoscale.Autoscaler; -import com.yahoo.vespa.hosted.provision.autoscale.MetricsDb; import com.yahoo.vespa.hosted.provision.autoscale.NodeMetricSnapshot; -import com.yahoo.vespa.hosted.provision.autoscale.NodeTimeseries; import com.yahoo.vespa.hosted.provision.node.History; import java.time.Duration; import java.time.Instant; import java.util.Map; import java.util.Optional; -import java.util.Set; /** * Maintainer making automatic scaling decisions @@ -47,14 +44,13 @@ public class AutoscalingMaintainer extends NodeRepositoryMaintainer { } @Override - protected boolean maintain() { - if ( ! nodeRepository().nodes().isWorking()) return false; + protected double maintain() { + if ( ! nodeRepository().nodes().isWorking()) return 0.0; - boolean success = true; - if ( ! nodeRepository().zone().environment().isAnyOf(Environment.dev, Environment.prod)) return success; + if ( ! nodeRepository().zone().environment().isAnyOf(Environment.dev, Environment.prod)) return 1.0; activeNodesByApplication().forEach(this::autoscale); - return success; + return 1.0; } private void autoscale(ApplicationId application, NodeList applicationNodes) { @@ -81,7 +77,7 @@ public class AutoscalingMaintainer extends NodeRepositoryMaintainer { // 1. Update cluster info updatedCluster = updateCompletion(cluster.get(), clusterNodes) - .withAutoscalingStatus(advice.reason()) + .with(advice.reason()) .withTarget(advice.target()); applications().put(application.get().with(updatedCluster), lock); if (advice.isPresent() && advice.target().isPresent() && !cluster.get().targetResources().equals(advice.target())) { diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/DynamicProvisioningMaintainer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/DynamicProvisioningMaintainer.java index 4224667a726..0eb2038e233 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/DynamicProvisioningMaintainer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/DynamicProvisioningMaintainer.java @@ -12,13 +12,10 @@ import com.yahoo.config.provision.OutOfCapacityException; import com.yahoo.jdisc.Metric; import com.yahoo.lang.MutableInteger; import com.yahoo.transaction.Mutex; -import com.yahoo.vespa.flags.FetchVector; import com.yahoo.vespa.flags.FlagSource; -import com.yahoo.vespa.flags.Flags; import com.yahoo.vespa.flags.JacksonFlag; import com.yahoo.vespa.flags.ListFlag; import com.yahoo.vespa.flags.PermanentFlags; -import com.yahoo.vespa.flags.StringFlag; import com.yahoo.vespa.flags.custom.ClusterCapacity; import com.yahoo.vespa.flags.custom.SharedHost; import com.yahoo.vespa.hosted.provision.LockedNodeList; @@ -64,7 +61,6 @@ public class DynamicProvisioningMaintainer extends NodeRepositoryMaintainer { private final HostProvisioner hostProvisioner; private final ListFlag<ClusterCapacity> preprovisionCapacityFlag; private final JacksonFlag<SharedHost> sharedHostFlag; - private final StringFlag allocateOsRequirement; DynamicProvisioningMaintainer(NodeRepository nodeRepository, Duration interval, @@ -75,17 +71,16 @@ public class DynamicProvisioningMaintainer extends NodeRepositoryMaintainer { this.hostProvisioner = hostProvisioner; this.preprovisionCapacityFlag = PermanentFlags.PREPROVISION_CAPACITY.bindTo(flagSource); this.sharedHostFlag = PermanentFlags.SHARED_HOST.bindTo(flagSource); - this.allocateOsRequirement = Flags.ALLOCATE_OS_REQUIREMENT.bindTo(flagSource); } @Override - protected boolean maintain() { + protected double maintain() { try (Mutex lock = nodeRepository().nodes().lockUnallocated()) { NodeList nodes = nodeRepository().nodes().list(); resumeProvisioning(nodes, lock); convergeToCapacity(nodes); } - return true; + return 1.0; } /** Resume provisioning of already provisioned hosts and their children */ @@ -301,12 +296,9 @@ public class DynamicProvisioningMaintainer extends NodeRepositoryMaintainer { NodeSpec nodeSpec = NodeSpec.from(clusterCapacity.count(), nodeResources, false, true); int wantedGroups = 1; - String allocateOsRequirement = this.allocateOsRequirement - .with(FetchVector.Dimension.APPLICATION_ID, ApplicationId.defaultId().serializedForm()) - .value(); NodePrioritizer prioritizer = new NodePrioritizer(nodeList, applicationId, clusterSpec, nodeSpec, wantedGroups, true, nodeRepository().nameResolver(), nodeRepository().resourcesCalculator(), - nodeRepository().spareCount(), allocateOsRequirement); + nodeRepository().spareCount()); List<NodeCandidate> nodeCandidates = prioritizer.collect(List.of()); MutableInteger index = new MutableInteger(0); return nodeCandidates diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/Expirer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/Expirer.java index 2443a12d198..25108425e6e 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/Expirer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/Expirer.java @@ -40,7 +40,7 @@ public abstract class Expirer extends NodeRepositoryMaintainer { } @Override - protected boolean maintain() { + protected double maintain() { NodeList expired = nodeRepository().nodes().list(fromState).matching(this::isExpired); if ( ! expired.isEmpty()) { @@ -49,7 +49,7 @@ public abstract class Expirer extends NodeRepositoryMaintainer { } metric.add("expired." + fromState, expired.size(), null); - return true; + return 1.0; } protected boolean isExpired(Node node) { diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/FailedExpirer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/FailedExpirer.java index e98da35aa6a..7505ce42668 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/FailedExpirer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/FailedExpirer.java @@ -66,7 +66,7 @@ public class FailedExpirer extends NodeRepositoryMaintainer { } @Override - protected boolean maintain() { + protected double maintain() { List<Node> remainingNodes = new ArrayList<>(nodeRepository.nodes().list(Node.State.failed) .nodeType(NodeType.tenant, NodeType.host) .asList()); @@ -78,7 +78,7 @@ public class FailedExpirer extends NodeRepositoryMaintainer { recycleIf(remainingNodes, node -> node.allocation().get().membership().cluster().isStateful() && node.history().hasEventBefore(History.Event.Type.failed, clock().instant().minus(statefulExpiry))); - return true; + return 1.0; } /** Recycle the nodes matching condition, and remove those nodes from the nodes list. */ diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/HostEncrypter.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/HostEncrypter.java index 50f77a7df7b..80f74a011c0 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/HostEncrypter.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/HostEncrypter.java @@ -43,14 +43,23 @@ public class HostEncrypter extends NodeRepositoryMaintainer { } @Override - protected boolean maintain() { + protected double maintain() { Instant now = nodeRepository().clock().instant(); NodeList allNodes = nodeRepository().nodes().list(); for (var nodeType : NodeType.values()) { if (!nodeType.isHost()) continue; + if (upgradingVespa(allNodes, nodeType)) continue; unencryptedHosts(allNodes, nodeType).forEach(host -> encrypt(host, now)); } - return true; + return 1.0; + } + + /** Returns whether any node of given type is currently upgrading its Vespa version */ + private boolean upgradingVespa(NodeList allNodes, NodeType hostType) { + return allNodes.state(Node.State.ready, Node.State.active) + .nodeType(hostType) + .changingVersion() + .size() > 0; } /** Returns unencrypted hosts of given type that can be encrypted */ diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/InfrastructureProvisioner.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/InfrastructureProvisioner.java index e317333135c..d9f5ea6a7a9 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/InfrastructureProvisioner.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/InfrastructureProvisioner.java @@ -39,9 +39,9 @@ public class InfrastructureProvisioner extends NodeRepositoryMaintainer { } @Override - protected boolean maintain() { + protected double maintain() { infraDeployer.activateAllSupportedInfraApplications(false); - return true; + return 1.0; } } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/LoadBalancerExpirer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/LoadBalancerExpirer.java index 9665d8872de..ac9d8d6671a 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/LoadBalancerExpirer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/LoadBalancerExpirer.java @@ -1,7 +1,9 @@ // Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.provision.maintenance; +import com.google.common.collect.Sets; import com.yahoo.jdisc.Metric; +import com.yahoo.lang.MutableInteger; import com.yahoo.vespa.hosted.provision.Node; import com.yahoo.vespa.hosted.provision.NodeRepository; import com.yahoo.vespa.hosted.provision.lb.LoadBalancer; @@ -22,6 +24,7 @@ import java.util.concurrent.atomic.AtomicReference; import java.util.function.Consumer; import java.util.function.Predicate; import java.util.logging.Level; +import java.util.logging.Logger; import java.util.stream.Collectors; /** @@ -37,6 +40,8 @@ import java.util.stream.Collectors; */ public class LoadBalancerExpirer extends NodeRepositoryMaintainer { + private static final Logger LOG = Logger.getLogger(LoadBalancerExpirer.class.getName()); + private static final Duration reservedExpiry = Duration.ofHours(1); private static final Duration inactiveExpiry = Duration.ofHours(1); @@ -50,9 +55,9 @@ public class LoadBalancerExpirer extends NodeRepositoryMaintainer { } @Override - protected boolean maintain() { + protected double maintain() { expireReserved(); - return removeInactive() & pruneReals(); + return ( removeInactive() + pruneReals() ) / 2; } /** Move reserved load balancer that have expired to inactive */ @@ -64,7 +69,8 @@ public class LoadBalancerExpirer extends NodeRepositoryMaintainer { } /** Deprovision inactive load balancers that have expired */ - private boolean removeInactive() { + private double removeInactive() { + MutableInteger attempts = new MutableInteger(0); var failed = new ArrayList<LoadBalancerId>(); var lastException = new AtomicReference<Exception>(); var expiry = nodeRepository().clock().instant().minus(inactiveExpiry); @@ -72,6 +78,8 @@ public class LoadBalancerExpirer extends NodeRepositoryMaintainer { lb.changedAt().isBefore(expiry) && allocatedNodes(lb.id()).isEmpty(), lb -> { try { + attempts.add(1); + log.log(Level.INFO, () -> "Removing expired inactive load balancer " + lb.id()); service.remove(lb.id().application(), lb.id().cluster()); db.removeLoadBalancer(lb.id()); } catch (Exception e){ @@ -80,19 +88,19 @@ public class LoadBalancerExpirer extends NodeRepositoryMaintainer { } }); if (!failed.isEmpty()) { - log.log(Level.WARNING, String.format("Failed to remove %d load balancers: %s, retrying in %s", - failed.size(), - failed.stream() - .map(LoadBalancerId::serializedForm) - .collect(Collectors.joining(", ")), - interval()), - lastException.get()); + log.log(Level.WARNING, lastException.get(), () -> String.format("Failed to remove %d load balancers: %s, retrying in %s", + failed.size(), + failed.stream() + .map(LoadBalancerId::serializedForm) + .collect(Collectors.joining(", ")), + interval())); } - return lastException.get() == null; + return asSuccessFactor(attempts.get(), failed.size()); } /** Remove reals from inactive load balancers */ - private boolean pruneReals() { + private double pruneReals() { + var attempts = new MutableInteger(0); var failed = new ArrayList<LoadBalancerId>(); var lastException = new AtomicReference<Exception>(); patchLoadBalancers(lb -> lb.state() == State.inactive, lb -> { @@ -101,7 +109,10 @@ public class LoadBalancerExpirer extends NodeRepositoryMaintainer { var reals = new LinkedHashSet<>(lb.instance().get().reals()); // Remove any real no longer allocated to this application reals.removeIf(real -> !allocatedNodes.contains(real.hostname().value())); + if (reals.equals(lb.instance().get().reals())) return; // Nothing to remove try { + attempts.add(1); + LOG.log(Level.INFO, () -> "Removing reals from inactive load balancer " + lb.id() + ": " + Sets.difference(lb.instance().get().reals(), reals)); service.create(new LoadBalancerSpec(lb.id().application(), lb.id().cluster(), reals), true); db.writeLoadBalancer(lb.with(lb.instance().map(instance -> instance.withReals(reals)))); } catch (Exception e) { @@ -118,7 +129,7 @@ public class LoadBalancerExpirer extends NodeRepositoryMaintainer { interval()), lastException.get()); } - return lastException.get() == null; + return asSuccessFactor(attempts.get(), failed.size()); } /** Patch load balancers matching given filter, while holding lock */ diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/MetricsReporter.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/MetricsReporter.java index 85437b3e78a..3990c5099eb 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/MetricsReporter.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/MetricsReporter.java @@ -65,7 +65,7 @@ public class MetricsReporter extends NodeRepositoryMaintainer { } @Override - public boolean maintain() { + public double maintain() { NodeList nodes = nodeRepository().nodes().list(); ServiceModel serviceModel = serviceMonitor.getServiceModelSnapshot(); @@ -80,7 +80,7 @@ public class MetricsReporter extends NodeRepositoryMaintainer { updateRepairTicketMetrics(nodes); updateAllocationMetrics(nodes); updateExclusiveSwitchMetrics(nodes); - return true; + return 1.0; } private void updateAllocationMetrics(NodeList nodes) { diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeFailer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeFailer.java index effa41dc69f..f16459ee8b9 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeFailer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeFailer.java @@ -72,17 +72,21 @@ public class NodeFailer extends NodeRepositoryMaintainer { } @Override - protected boolean maintain() { - if ( ! nodeRepository().nodes().isWorking()) return false; + protected double maintain() { + if ( ! nodeRepository().nodes().isWorking()) return 0.0; + int attempts = 0; + int failures = 0; int throttledHostFailures = 0; int throttledNodeFailures = 0; // Ready nodes try (Mutex lock = nodeRepository().nodes().lockUnallocated()) { for (Map.Entry<Node, String> entry : getReadyNodesByFailureReason().entrySet()) { + attempts++; Node node = entry.getKey(); if (throttle(node)) { + failures++; if (node.type().isHost()) throttledHostFailures++; else @@ -96,10 +100,12 @@ public class NodeFailer extends NodeRepositoryMaintainer { // Active nodes for (Map.Entry<Node, String> entry : getActiveNodesByFailureReason().entrySet()) { + attempts++; Node node = entry.getKey(); if (!failAllowedFor(node.type())) continue; if (throttle(node)) { + failures++; if (node.type().isHost()) throttledHostFailures++; else @@ -116,11 +122,15 @@ public class NodeFailer extends NodeRepositoryMaintainer { if ( ! activeNodes.childrenOf(host).isEmpty()) continue; Optional<NodeMutex> locked = Optional.empty(); try { + attempts++; locked = nodeRepository().nodes().lockAndGet(host); if (locked.isEmpty()) continue; nodeRepository().nodes().fail(List.of(locked.get().node()), Agent.NodeFailer, "Host should be failed and have no tenant nodes"); } + catch (Exception e) { + failures++; + } finally { locked.ifPresent(NodeMutex::close); } @@ -130,7 +140,7 @@ public class NodeFailer extends NodeRepositoryMaintainer { metric.set(throttlingActiveMetric, throttlingActive, null); metric.set(throttledHostFailuresMetric, throttledHostFailures, null); metric.set(throttledNodeFailuresMetric, throttledNodeFailures, null); - return throttlingActive == 0; + return asSuccessFactor(attempts, failures); } private Map<Node, String> getReadyNodesByFailureReason() { diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeHealthTracker.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeHealthTracker.java index fe2fb5229f9..37969a30b81 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeHealthTracker.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeHealthTracker.java @@ -5,6 +5,7 @@ import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.ApplicationLockException; import com.yahoo.config.provision.HostLivenessTracker; import com.yahoo.jdisc.Metric; +import com.yahoo.lang.MutableInteger; import com.yahoo.transaction.Mutex; import com.yahoo.vespa.applicationmodel.ServiceInstance; import com.yahoo.vespa.applicationmodel.ServiceStatus; @@ -48,13 +49,11 @@ public class NodeHealthTracker extends NodeRepositoryMaintainer { } @Override - protected boolean maintain() { - updateReadyNodeLivenessEvents(); - updateActiveNodeDownState(); - return true; + protected double maintain() { + return ( updateReadyNodeLivenessEvents() + updateActiveNodeDownState() ) / 2; } - private void updateReadyNodeLivenessEvents() { + private double updateReadyNodeLivenessEvents() { // Update node last request events through ZooKeeper to collect request to all config servers. // We do this here ("lazily") to avoid writing to zk for each config request. try (Mutex lock = nodeRepository().nodes().lockUnallocated()) { @@ -69,13 +68,16 @@ public class NodeHealthTracker extends NodeRepositoryMaintainer { } } } + return 1.0; } /** * If the node is down (see {@link #allDown}), and there is no "down" history record, we add it. * Otherwise we remove any "down" history record. */ - private void updateActiveNodeDownState() { + private double updateActiveNodeDownState() { + var attempts = new MutableInteger(0); + var failures = new MutableInteger(0); NodeList activeNodes = nodeRepository().nodes().list(Node.State.active); serviceMonitor.getServiceModelSnapshot().getServiceInstancesByHostName().forEach((hostname, serviceInstances) -> { Optional<Node> node = activeNodes.node(hostname.toString()); @@ -90,6 +92,7 @@ public class NodeHealthTracker extends NodeRepositoryMaintainer { try (var lock = nodeRepository().nodes().lock(owner)) { node = getNode(hostname.toString(), owner, lock); // Re-get inside lock if (node.isEmpty()) return; // Node disappeared or changed allocation + attempts.add(1); if (isDown) { recordAsDown(node.get(), lock); } else { @@ -98,8 +101,10 @@ public class NodeHealthTracker extends NodeRepositoryMaintainer { } catch (ApplicationLockException e) { // Fine, carry on with other nodes. We'll try updating this one in the next run log.log(Level.WARNING, "Could not lock " + owner + ": " + Exceptions.toMessageString(e)); + failures.add(1); } }); + return asSuccessFactor(attempts.get(), failures.get()); } /** diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeMetricsDbMaintainer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeMetricsDbMaintainer.java index 1ea4577f7fe..d671900d08c 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeMetricsDbMaintainer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeMetricsDbMaintainer.java @@ -33,19 +33,21 @@ public class NodeMetricsDbMaintainer extends NodeRepositoryMaintainer { } @Override - protected boolean maintain() { + protected double maintain() { + int attempts = 0; + var failures = new MutableInteger(0); try { - var warnings = new MutableInteger(0); Set<ApplicationId> applications = activeNodesByApplication().keySet(); - if (applications.isEmpty()) return true; + if (applications.isEmpty()) return 1.0; long pauseMs = interval().toMillis() / applications.size() - 1; // spread requests over interval int done = 0; for (ApplicationId application : applications) { + attempts++; metricsFetcher.fetchMetrics(application) .whenComplete((metricsResponse, exception) -> handleResponse(metricsResponse, exception, - warnings, + failures, application)); if (++done < applications.size()) Thread.sleep(pauseMs); @@ -56,23 +58,22 @@ public class NodeMetricsDbMaintainer extends NodeRepositoryMaintainer { nodeRepository().metricsDb().gc(); - // Suppress failures for manual zones for now to avoid noise - return nodeRepository().zone().environment().isManuallyDeployed() || warnings.get() == 0; + return asSuccessFactor(attempts, failures.get()); } catch (InterruptedException e) { - return false; + return asSuccessFactor(attempts, failures.get()); } } private void handleResponse(MetricsResponse response, Throwable exception, - MutableInteger warnings, + MutableInteger failures, ApplicationId application) { if (exception != null) { - if (warnings.get() < maxWarningsPerInvocation) + if (failures.get() < maxWarningsPerInvocation) log.log(Level.WARNING, "Could not update metrics for " + application + ": " + Exceptions.toMessageString(exception)); - warnings.add(1); + failures.add(1); } else if (response != null) { nodeRepository().metricsDb().addNodeMetrics(response.nodeMetrics()); diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeRebooter.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeRebooter.java index 6ee657beadd..c282fcdb7fc 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeRebooter.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeRebooter.java @@ -37,7 +37,7 @@ public class NodeRebooter extends NodeRepositoryMaintainer { } @Override - protected boolean maintain() { + protected double maintain() { // Reboot candidates: Nodes in long-term states, where we know we can safely orchestrate a reboot List<Node> nodesToReboot = nodeRepository().nodes().list(Node.State.active, Node.State.ready).stream() .filter(node -> node.type().isHost()) @@ -46,7 +46,7 @@ public class NodeRebooter extends NodeRepositoryMaintainer { if (!nodesToReboot.isEmpty()) nodeRepository().nodes().reboot(NodeListFilter.from(nodesToReboot)); - return true; + return 1.0; } private boolean shouldReboot(Node node) { diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeRepositoryMaintainer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeRepositoryMaintainer.java index 0a1f6961f9f..fe5cd419b31 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeRepositoryMaintainer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeRepositoryMaintainer.java @@ -25,7 +25,7 @@ public abstract class NodeRepositoryMaintainer extends Maintainer { public NodeRepositoryMaintainer(NodeRepository nodeRepository, Duration interval, Metric metric) { super(null, interval, nodeRepository.clock().instant(), nodeRepository.jobControl(), - jobMetrics(metric), nodeRepository.database().cluster(), true); + new NodeRepositoryJobMetrics(metric), nodeRepository.database().cluster(), true); this.nodeRepository = nodeRepository; } @@ -48,10 +48,20 @@ public abstract class NodeRepositoryMaintainer extends Maintainer { .groupingBy(node -> node.allocation().get().owner()); } - private static JobMetrics jobMetrics(Metric metric) { - return new JobMetrics((job, consecutiveFailures) -> { + private static class NodeRepositoryJobMetrics extends JobMetrics { + + private final Metric metric; + + public NodeRepositoryJobMetrics(Metric metric) { + this.metric = metric; + } + + @Override + protected void recordCompletion(String job, Long consecutiveFailures, double successFactor) { metric.set("maintenance.consecutiveFailures", consecutiveFailures, metric.createContext(Map.of("job", job))); - }); + metric.set("maintenance.successFactor", successFactor, metric.createContext(Map.of("job", job))); + } + } } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/OperatorChangeApplicationMaintainer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/OperatorChangeApplicationMaintainer.java index f620a6d113d..47337518a65 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/OperatorChangeApplicationMaintainer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/OperatorChangeApplicationMaintainer.java @@ -5,6 +5,7 @@ import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.Deployer; import com.yahoo.config.provision.NodeType; import com.yahoo.jdisc.Metric; +import com.yahoo.vespa.hosted.provision.Node; import com.yahoo.vespa.hosted.provision.NodeList; import com.yahoo.vespa.hosted.provision.NodeRepository; import com.yahoo.vespa.hosted.provision.node.Agent; @@ -70,4 +71,18 @@ public class OperatorChangeApplicationMaintainer extends ApplicationMaintainer { .anyMatch(e -> lastDeployTime.get().isBefore(e)); } + @Override + protected boolean canDeployNow(ApplicationId application) { + return activeNodesByApplication().get(application) != null; + } + + @Override + protected Map<ApplicationId, NodeList> activeNodesByApplication() { + return nodeRepository().nodes() + .list(Node.State.active) + .not().tester() + .groupingBy(node -> node.allocation().get().owner()); + } + + } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/OsUpgradeActivator.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/OsUpgradeActivator.java index 3ff4ca89ad4..749603a373d 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/OsUpgradeActivator.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/OsUpgradeActivator.java @@ -23,13 +23,13 @@ public class OsUpgradeActivator extends NodeRepositoryMaintainer { } @Override - protected boolean maintain() { + protected double maintain() { for (var nodeType : NodeType.values()) { if (!nodeType.isHost()) continue; boolean resume = canUpgradeOsOf(nodeType); nodeRepository().osVersions().resumeUpgradeOf(nodeType, resume); } - return true; + return 1.0; } /** Returns whether to allow OS upgrade of nodes of given type */ @@ -38,7 +38,6 @@ public class OsUpgradeActivator extends NodeRepositoryMaintainer { .list(Node.State.ready, Node.State.active) .nodeType(type) .changingVersion() - .asList() .isEmpty(); } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/Rebalancer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/Rebalancer.java index 1543506a78e..7bb748c92c9 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/Rebalancer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/Rebalancer.java @@ -33,19 +33,18 @@ public class Rebalancer extends NodeMover<Rebalancer.Move> { } @Override - protected boolean maintain() { - if ( ! nodeRepository().nodes().isWorking()) return false; + protected double maintain() { + if ( ! nodeRepository().nodes().isWorking()) return 0.0; - boolean success = true; - if (nodeRepository().zone().getCloud().dynamicProvisioning()) return success; // Rebalancing not necessary - if (nodeRepository().zone().environment().isTest()) return success; // Short lived deployments; no need to rebalance + if (nodeRepository().zone().getCloud().dynamicProvisioning()) return 1.0; // Rebalancing not necessary + if (nodeRepository().zone().environment().isTest()) return 1.0; // Short lived deployments; no need to rebalance // Work with an unlocked snapshot as this can take a long time and full consistency is not needed NodeList allNodes = nodeRepository().nodes().list(); updateSkewMetric(allNodes); - if ( ! zoneIsStable(allNodes)) return success; + if ( ! zoneIsStable(allNodes)) return 1.0; findBestMove(allNodes).execute(true, Agent.Rebalancer, deployer, metric, nodeRepository()); - return success; + return 1.0; } @Override diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/RetiredExpirer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/RetiredExpirer.java index f72daf1bc2b..3f5893b368a 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/RetiredExpirer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/RetiredExpirer.java @@ -48,7 +48,10 @@ public class RetiredExpirer extends NodeRepositoryMaintainer { } @Override - protected boolean maintain() { + protected double maintain() { + int attempts = 0; + int successes = 0; + NodeList activeNodes = nodeRepository().nodes().list(Node.State.active); Map<ApplicationId, NodeList> retiredNodesByApplication = activeNodes.retired().groupingBy(node -> node.allocation().get().owner()); for (Map.Entry<ApplicationId, NodeList> entry : retiredNodesByApplication.entrySet()) { @@ -57,17 +60,19 @@ public class RetiredExpirer extends NodeRepositoryMaintainer { List<Node> nodesToRemove = retiredNodes.stream().filter(n -> canRemove(n, activeNodes)).collect(Collectors.toList()); if (nodesToRemove.isEmpty()) continue; + attempts++; try (MaintenanceDeployment deployment = new MaintenanceDeployment(application, deployer, metric, nodeRepository())) { if ( ! deployment.isValid()) continue; nodeRepository().nodes().setRemovable(application, nodesToRemove); boolean success = deployment.activate().isPresent(); - if ( ! success) return success; + if ( ! success) continue; String nodeList = nodesToRemove.stream().map(Node::hostname).collect(Collectors.joining(", ")); log.info("Redeployed " + application + " to deactivate retired nodes: " + nodeList); + successes++; } } - return true; + return attempts == 0 ? 1.0 : ((double)successes / attempts); } /** diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/ScalingSuggestionsMaintainer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/ScalingSuggestionsMaintainer.java index c217580872b..888f06a5004 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/ScalingSuggestionsMaintainer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/ScalingSuggestionsMaintainer.java @@ -36,13 +36,16 @@ public class ScalingSuggestionsMaintainer extends NodeRepositoryMaintainer { } @Override - protected boolean maintain() { - if ( ! nodeRepository().zone().environment().isProduction()) return true; + protected double maintain() { + if ( ! nodeRepository().zone().environment().isProduction()) return 1.0; + int attempts = 0; int successes = 0; - for (var application : activeNodesByApplication().entrySet()) + for (var application : activeNodesByApplication().entrySet()) { + attempts++; successes += suggest(application.getKey(), application.getValue()); - return successes > 0; + } + return attempts == 0 ? 1.0 : ((double)successes / attempts); } private int suggest(ApplicationId application, NodeList applicationNodes) { diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/SpareCapacityMaintainer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/SpareCapacityMaintainer.java index 0307ae13b24..0589571e9d8 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/SpareCapacityMaintainer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/SpareCapacityMaintainer.java @@ -66,12 +66,11 @@ public class SpareCapacityMaintainer extends NodeRepositoryMaintainer { } @Override - protected boolean maintain() { - if ( ! nodeRepository().nodes().isWorking()) return false; + protected double maintain() { + if ( ! nodeRepository().nodes().isWorking()) return 0.0; - boolean success = true; // Don't need to maintain spare capacity in dynamically provisioned zones; can provision more on demand. - if (nodeRepository().zone().getCloud().dynamicProvisioning()) return success; + if (nodeRepository().zone().getCloud().dynamicProvisioning()) return 1.0; NodeList allNodes = nodeRepository().nodes().list(); CapacityChecker capacityChecker = new CapacityChecker(allNodes); @@ -80,6 +79,7 @@ public class SpareCapacityMaintainer extends NodeRepositoryMaintainer { metric.set("overcommittedHosts", overcommittedHosts.size(), null); retireOvercommitedHosts(allNodes, overcommittedHosts); + boolean success = true; Optional<CapacityChecker.HostFailurePath> failurePath = capacityChecker.worstCaseHostLossLeadingToFailure(); if (failurePath.isPresent()) { int spareHostCapacity = failurePath.get().hostsCausingFailure.size() - 1; @@ -96,7 +96,7 @@ public class SpareCapacityMaintainer extends NodeRepositoryMaintainer { } metric.set("spareHostCapacity", spareHostCapacity, null); } - return success; + return success ? 1.0 : 0.0; } private boolean execute(List<Move> mitigation, CapacityChecker.HostFailurePath failurePath) { diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/SwitchRebalancer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/SwitchRebalancer.java index cfab980570d..44890f2f5af 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/SwitchRebalancer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/SwitchRebalancer.java @@ -33,14 +33,14 @@ public class SwitchRebalancer extends NodeMover<Move> { } @Override - protected boolean maintain() { - if (!nodeRepository().nodes().isWorking()) return false; - if (!nodeRepository().zone().environment().isProduction()) return true; + protected double maintain() { + if (!nodeRepository().nodes().isWorking()) return 0.0; + if (!nodeRepository().zone().environment().isProduction()) return 1.0; NodeList allNodes = nodeRepository().nodes().list(); // Lockless as strong consistency is not needed - if (!zoneIsStable(allNodes)) return true; + if (!zoneIsStable(allNodes)) return 1.0; findBestMove(allNodes).execute(false, Agent.SwitchRebalancer, deployer, metric, nodeRepository()); - return true; + return 1.0; } @Override diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/Nodes.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/Nodes.java index 59cee0a469e..3afe5824af5 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/Nodes.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/Nodes.java @@ -248,7 +248,7 @@ public class Nodes { public List<Node> fail(List<Node> nodes, Agent agent, String reason) { NestedTransaction transaction = new NestedTransaction(); nodes = fail(nodes, agent, reason, transaction); - transaction.commit();; + transaction.commit(); return nodes; } @@ -305,10 +305,12 @@ public class Nodes { } public Node deallocate(Node node, Agent agent, String reason, NestedTransaction transaction) { - if (parkOnDeallocationOf(node, agent)) - return park(node.hostname(), false, agent, reason, transaction); - else + if (parkOnDeallocationOf(node, agent)) { + boolean keepAllocation = node.reports().getReport(Report.WANT_TO_ENCRYPT_ID).isPresent(); + return park(node.hostname(), keepAllocation, agent, reason, transaction); + } else { return db.writeTo(Node.State.dirty, List.of(node), agent, Optional.of(reason), transaction).get(0); + } } /** @@ -367,7 +369,7 @@ public class Nodes { return parked; } - public Node park(String hostname, boolean keepAllocation, Agent agent, String reason, NestedTransaction transaction) { + private Node park(String hostname, boolean keepAllocation, Agent agent, String reason, NestedTransaction transaction) { return move(hostname, Node.State.parked, agent, keepAllocation, Optional.of(reason), transaction); } @@ -573,14 +575,22 @@ public class Nodes { } /** - * Increases the restart generation of the active nodes matching the filter. + * Increases the restart generation of the active nodes matching given filter. + * + * @return the nodes in their new state + */ + public List<Node> restartActive(Predicate<Node> filter) { + return restart(StateFilter.from(Node.State.active).and(filter)); + } + + /** + * Increases the restart generation of the any nodes matching given filter. * * @return the nodes in their new state */ public List<Node> restart(Predicate<Node> filter) { - return performOn(StateFilter.from(Node.State.active).and(filter), - (node, lock) -> write(node.withRestart(node.allocation().get().restartGeneration().withIncreasedWanted()), - lock)); + return performOn(filter, (node, lock) -> write(node.withRestart(node.allocation().get().restartGeneration().withIncreasedWanted()), + lock)); } /** @@ -632,16 +642,22 @@ public class Nodes { List<Node> result; boolean wantToDeprovision = op == DecommissionOperation.deprovision; boolean wantToRebuild = op == DecommissionOperation.rebuild; + Optional<Report> wantToEncryptReport = op == DecommissionOperation.encrypt + ? Optional.of(Report.basicReport(Report.WANT_TO_ENCRYPT_ID, Report.Type.UNSPECIFIED, instant, "")) + : Optional.empty(); try (NodeMutex lock = nodeMutex.get(); Mutex allocationLock = lockUnallocated()) { // This takes allocationLock to prevent any further allocation of nodes on this host host = lock.node(); - result = performOn(list(allocationLock).childrenOf(host), - (node, nodeLock) -> write(node.withWantToRetire(true, wantToDeprovision, wantToRebuild, agent, instant), - nodeLock)); + result = performOn(list(allocationLock).childrenOf(host), (node, nodeLock) -> { + Node newNode = node.withWantToRetire(true, wantToDeprovision, wantToRebuild, agent, instant); + if (wantToEncryptReport.isPresent()) { + newNode = newNode.with(newNode.reports().withReport(wantToEncryptReport.get())); + } + return write(newNode, nodeLock); + }); Node newHost = host.withWantToRetire(true, wantToDeprovision, wantToRebuild, agent, instant); - if (op == DecommissionOperation.encrypt) { - Report report = Report.basicReport(Report.WANT_TO_ENCRYPT_ID, Report.Type.UNSPECIFIED, instant, ""); - newHost = newHost.with(newHost.reports().withReport(report)); + if (wantToEncryptReport.isPresent()) { + newHost = newHost.with(newHost.reports().withReport(wantToEncryptReport.get())); } result.add(write(newHost, lock)); } @@ -813,6 +829,7 @@ public class Nodes { .orElse(false); return node.status().wantToDeprovision() || node.status().wantToRebuild() || + node.reports().getReport(Report.WANT_TO_ENCRYPT_ID).isPresent() || retirementRequestedByOperator; } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/Report.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/Report.java index 4eb76828131..f5c5b9f2857 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/Report.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/Report.java @@ -28,6 +28,7 @@ public class Report { public static final String DESCRIPTION_FIELD = "description"; /** Known report IDs */ + // TODO(mpolden): Remove together with HostEncrypter public static final String WANT_TO_ENCRYPT_ID = "wantToEncrypt"; public static final String DISK_ENCRYPTED_ID = "diskEncrypted"; diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/ApplicationSerializer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/ApplicationSerializer.java index c8b928779b9..e3fb3379da1 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/ApplicationSerializer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/ApplicationSerializer.java @@ -10,6 +10,7 @@ import com.yahoo.slime.ObjectTraverser; import com.yahoo.slime.Slime; import com.yahoo.slime.SlimeUtils; import com.yahoo.vespa.hosted.provision.applications.Application; +import com.yahoo.vespa.hosted.provision.applications.AutoscalingStatus; import com.yahoo.vespa.hosted.provision.applications.Cluster; import com.yahoo.vespa.hosted.provision.applications.ScalingEvent; import com.yahoo.vespa.hosted.provision.applications.Status; @@ -55,6 +56,8 @@ public class ApplicationSerializer { private static final String nodeResourcesKey = "resources"; private static final String scalingEventsKey = "scalingEvents"; private static final String autoscalingStatusKey = "autoscalingStatus"; + private static final String autoscalingStatusObjectKey = "autoscalingStatusObject"; + private static final String descriptionKey = "description"; private static final String fromKey = "from"; private static final String toKey = "to"; private static final String generationKey = "generation"; @@ -118,7 +121,8 @@ public class ApplicationSerializer { cluster.suggestedResources().ifPresent(suggested -> toSlime(suggested, clusterObject.setObject(suggestedKey))); cluster.targetResources().ifPresent(target -> toSlime(target, clusterObject.setObject(targetResourcesKey))); scalingEventsToSlime(cluster.scalingEvents(), clusterObject.setArray(scalingEventsKey)); - clusterObject.setString(autoscalingStatusKey, cluster.autoscalingStatus()); + clusterObject.setString(autoscalingStatusKey, cluster.autoscalingStatus().description()); // TODO: Remove after June 2021 + toSlime(cluster.autoscalingStatus(), clusterObject.setObject(autoscalingStatusObjectKey)); } private static Cluster clusterFromSlime(String id, Inspector clusterObject) { @@ -129,7 +133,7 @@ public class ApplicationSerializer { optionalSuggestionFromSlime(clusterObject.field(suggestedKey)), optionalClusterResourcesFromSlime(clusterObject.field(targetResourcesKey)), scalingEventsFromSlime(clusterObject.field(scalingEventsKey)), - clusterObject.field(autoscalingStatusKey).asString()); + autoscalingStatusFromSlime(clusterObject.field(autoscalingStatusObjectKey), clusterObject)); } private static void toSlime(Cluster.Suggestion suggestion, Cursor suggestionObject) { @@ -188,6 +192,42 @@ public class ApplicationSerializer { optionalInstant(inspector.field(completionKey))); } + private static void toSlime(AutoscalingStatus status, Cursor object) { + object.setString(statusKey, toAutoscalingStatusCode(status.status())); + object.setString(descriptionKey, status.description()); + } + + private static AutoscalingStatus autoscalingStatusFromSlime(Inspector object, Inspector parent) { + // TODO: Remove this clause after June 2021 + if ( ! object.valid()) return new AutoscalingStatus(AutoscalingStatus.Status.unavailable, + parent.field(autoscalingStatusKey).asString()); + + return new AutoscalingStatus(fromAutoscalingStatusCode(object.field(statusKey).asString()), + object.field(descriptionKey).asString()); + } + + private static String toAutoscalingStatusCode(AutoscalingStatus.Status status) { + switch (status) { + case unavailable : return "unavailable"; + case waiting : return "waiting"; + case ideal : return "ideal"; + case insufficient : return "insufficient"; + case rescaling : return "rescaling"; + default : throw new IllegalArgumentException("Unknown autoscaling status " + status); + } + } + + private static AutoscalingStatus.Status fromAutoscalingStatusCode(String code) { + switch (code) { + case "unavailable" : return AutoscalingStatus.Status.unavailable; + case "waiting" : return AutoscalingStatus.Status.waiting; + case "ideal" : return AutoscalingStatus.Status.ideal; + case "insufficient" : return AutoscalingStatus.Status.insufficient; + case "rescaling" : return AutoscalingStatus.Status.rescaling; + default : throw new IllegalArgumentException("Unknown autoscaling status '" + code + "'"); + } + } + private static Optional<Instant> optionalInstant(Inspector inspector) { return inspector.valid() ? Optional.of(Instant.ofEpochMilli(inspector.asLong())) : Optional.empty(); } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/CuratorDatabaseClient.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/CuratorDatabaseClient.java index d1f881f8b7a..fb4799be29c 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/CuratorDatabaseClient.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/CuratorDatabaseClient.java @@ -11,7 +11,6 @@ import com.yahoo.config.provision.HostName; import com.yahoo.config.provision.NodeFlavors; import com.yahoo.config.provision.NodeType; import com.yahoo.config.provision.TenantName; -import com.yahoo.config.provision.Zone; import com.yahoo.path.Path; import com.yahoo.transaction.NestedTransaction; import com.yahoo.transaction.Transaction; @@ -77,13 +76,10 @@ public class CuratorDatabaseClient { private final NodeSerializer nodeSerializer; private final CuratorDatabase db; private final Clock clock; - private final Zone zone; private final CuratorCounter provisionIndexCounter; - public CuratorDatabaseClient(NodeFlavors flavors, Curator curator, Clock clock, Zone zone, boolean useCache, - long nodeCacheSize) { + public CuratorDatabaseClient(NodeFlavors flavors, Curator curator, Clock clock, boolean useCache, long nodeCacheSize) { this.nodeSerializer = new NodeSerializer(flavors, nodeCacheSize); - this.zone = zone; this.db = new CuratorDatabase(curator, root, useCache); this.clock = clock; this.provisionIndexCounter = new CuratorCounter(curator, root.append("provisionIndexCounter").getAbsolute()); diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/Activator.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/Activator.java index 6d06dc31a42..cb965e87739 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/Activator.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/Activator.java @@ -13,6 +13,7 @@ import com.yahoo.vespa.hosted.provision.NodeList; import com.yahoo.vespa.hosted.provision.NodeMutex; import com.yahoo.vespa.hosted.provision.NodeRepository; import com.yahoo.vespa.hosted.provision.applications.Application; +import com.yahoo.vespa.hosted.provision.applications.AutoscalingStatus; import com.yahoo.vespa.hosted.provision.applications.ScalingEvent; import com.yahoo.vespa.hosted.provision.node.Agent; import com.yahoo.vespa.hosted.provision.node.Allocation; @@ -117,7 +118,8 @@ class Activator { } if (cluster.targetResources().isPresent() && cluster.targetResources().get().justNumbers().equals(currentResources.justNumbers())) { - cluster = cluster.withAutoscalingStatus("Cluster is ideally scaled within configured limits"); + cluster = cluster.with(new AutoscalingStatus(AutoscalingStatus.Status.ideal, + "Cluster is ideally scaled within configured limits")); } if (cluster != modified.cluster(clusterEntry.getKey()).get()) modified = modified.with(cluster); diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/CapacityPolicies.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/CapacityPolicies.java index 711fe39d056..abd910485ac 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/CapacityPolicies.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/CapacityPolicies.java @@ -60,7 +60,7 @@ public class CapacityPolicies { return target; } - private NodeResources defaultNodeResources(ClusterSpec.Type clusterType) { + public NodeResources defaultNodeResources(ClusterSpec.Type clusterType) { if (clusterType == ClusterSpec.Type.admin) { if (zone.system() == SystemName.dev) { // Use small logserver in dev system diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/GroupPreparer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/GroupPreparer.java index def992a264b..5d45bed19e8 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/GroupPreparer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/GroupPreparer.java @@ -7,10 +7,6 @@ import com.yahoo.config.provision.ClusterSpec; import com.yahoo.config.provision.NodeType; import com.yahoo.config.provision.OutOfCapacityException; import com.yahoo.transaction.Mutex; -import com.yahoo.vespa.flags.FetchVector; -import com.yahoo.vespa.flags.FlagSource; -import com.yahoo.vespa.flags.Flags; -import com.yahoo.vespa.flags.StringFlag; import com.yahoo.vespa.hosted.provision.LockedNodeList; import com.yahoo.vespa.hosted.provision.Node; import com.yahoo.vespa.hosted.provision.NodeRepository; @@ -33,14 +29,10 @@ public class GroupPreparer { private final NodeRepository nodeRepository; private final Optional<HostProvisioner> hostProvisioner; - private final StringFlag allocateOsRequirementFlag; - public GroupPreparer(NodeRepository nodeRepository, - Optional<HostProvisioner> hostProvisioner, - FlagSource flagSource) { + public GroupPreparer(NodeRepository nodeRepository, Optional<HostProvisioner> hostProvisioner) { this.nodeRepository = nodeRepository; this.hostProvisioner = hostProvisioner; - this.allocateOsRequirementFlag = Flags.ALLOCATE_OS_REQUIREMENT.bindTo(flagSource); } /** @@ -60,17 +52,11 @@ public class GroupPreparer { // active config model which is changed on activate public List<Node> prepare(ApplicationId application, ClusterSpec cluster, NodeSpec requestedNodes, List<Node> surplusActiveNodes, NodeIndices indices, int wantedGroups) { - - String allocateOsRequirement = allocateOsRequirementFlag - .with(FetchVector.Dimension.APPLICATION_ID, application.serializedForm()) - .value(); - // Try preparing in memory without global unallocated lock. Most of the time there should be no changes and we // can return nodes previously allocated. { NodeAllocation probeAllocation = prepareAllocation(application, cluster, requestedNodes, surplusActiveNodes, - indices::probeNext, wantedGroups, PROBE_LOCK, - allocateOsRequirement); + indices::probeNext, wantedGroups, PROBE_LOCK); if (probeAllocation.fulfilledAndNoChanges()) { List<Node> acceptedNodes = probeAllocation.finalNodes(); surplusActiveNodes.removeAll(acceptedNodes); @@ -85,21 +71,18 @@ public class GroupPreparer { Mutex allocationLock = nodeRepository.nodes().lockUnallocated()) { NodeAllocation allocation = prepareAllocation(application, cluster, requestedNodes, surplusActiveNodes, - indices::next, wantedGroups, allocationLock, - allocateOsRequirement); + indices::next, wantedGroups, allocationLock); NodeType hostType = allocation.nodeType().hostType(); if (canProvisionDynamically(hostType)) { HostSharing sharing = hostSharing(requestedNodes, hostType); + Version osVersion = nodeRepository.osVersions().targetFor(hostType).orElse(Version.emptyVersion); List<ProvisionedHost> provisionedHosts = allocation.hostDeficit() - .map(deficit -> - hostProvisioner.get().provisionHosts(allocation.provisionIndices(deficit.count()), - hostType, - deficit.resources(), - application, - decideOsVersion(allocateOsRequirement, hostType), - sharing) - ) - .orElseGet(List::of); + .map(deficit -> hostProvisioner.get().provisionHosts(allocation.provisionIndices(deficit.count()), + hostType, + deficit.resources(), + application, osVersion, + sharing)) + .orElseGet(List::of); // At this point we have started provisioning of the hosts, the first priority is to make sure that // the returned hosts are added to the node-repo so that they are tracked by the provision maintainers @@ -131,7 +114,7 @@ public class GroupPreparer { private NodeAllocation prepareAllocation(ApplicationId application, ClusterSpec cluster, NodeSpec requestedNodes, List<Node> surplusActiveNodes, Supplier<Integer> nextIndex, int wantedGroups, - Mutex allocationLock, String allocateOsRequirement) { + Mutex allocationLock) { LockedNodeList allNodes = nodeRepository.nodes().list(allocationLock); NodeAllocation allocation = new NodeAllocation(allNodes, application, cluster, requestedNodes, nextIndex, nodeRepository); NodePrioritizer prioritizer = new NodePrioritizer(allNodes, @@ -142,8 +125,7 @@ public class GroupPreparer { nodeRepository.zone().getCloud().dynamicProvisioning(), nodeRepository.nameResolver(), nodeRepository.resourcesCalculator(), - nodeRepository.spareCount(), - allocateOsRequirement); + nodeRepository.spareCount()); allocation.offer(prioritizer.collect(surplusActiveNodes)); return allocation; } @@ -161,13 +143,4 @@ public class GroupPreparer { return sharing; } - private Version decideOsVersion(String allocateOsRequirement, NodeType hostType) { - if (allocateOsRequirement.equals("rhel8")) - return new Version(8, Integer.MAX_VALUE /* always use latest 8 version */, 0); - else if (allocateOsRequirement.equals("rhel7")) - return new Version(7, Integer.MAX_VALUE /* always use latest 7 version */, 0); - else - return nodeRepository.osVersions().targetFor(hostType).orElse(Version.emptyVersion); - } - } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/LoadBalancerProvisioner.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/LoadBalancerProvisioner.java index e665ace6334..c114aa58a05 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/LoadBalancerProvisioner.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/LoadBalancerProvisioner.java @@ -182,7 +182,8 @@ public class LoadBalancerProvisioner { : loadBalancer.get().state(); newLoadBalancer = loadBalancer.get().with(instance).with(state, now); if (loadBalancer.get().state() != newLoadBalancer.state()) { - log.log(Level.FINE, () -> "Moving " + newLoadBalancer.id() + " to state " + newLoadBalancer.state()); + log.log(Level.INFO, () -> "Moving " + newLoadBalancer.id() + " from " + loadBalancer.get().state() + + " to " + newLoadBalancer.state()); } } @@ -209,13 +210,13 @@ public class LoadBalancerProvisioner { private Optional<LoadBalancerInstance> provisionInstance(LoadBalancerId id, Set<Real> reals, Optional<LoadBalancer> currentLoadBalancer) { if (hasReals(currentLoadBalancer, reals)) return currentLoadBalancer.get().instance(); - log.log(Level.FINE, () -> "Creating " + id + ", targeting: " + reals); + log.log(Level.INFO, () -> "Creating " + id + ", targeting: " + reals); try { return Optional.of(service.create(new LoadBalancerSpec(id.application(), id.cluster(), reals), allowEmptyReals(currentLoadBalancer))); } catch (Exception e) { - log.log(Level.WARNING, "Could not (re)configure " + id + ", targeting: " + - reals + ". The operation will be retried on next deployment", e); + log.log(Level.WARNING, e, () -> "Could not (re)configure " + id + ", targeting: " + + reals + ". The operation will be retried on next deployment"); } return Optional.empty(); } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodePrioritizer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodePrioritizer.java index 8eca4ff2d95..695f0dd8659 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodePrioritizer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodePrioritizer.java @@ -1,7 +1,6 @@ // Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.provision.provisioning; -import com.yahoo.component.Version; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.ClusterSpec; import com.yahoo.config.provision.NodeType; @@ -32,7 +31,6 @@ public class NodePrioritizer { private final List<NodeCandidate> nodes = new ArrayList<>(); private final LockedNodeList allNodes; - private final String allocateOsRequirement; private final HostCapacity capacity; private final NodeSpec requestedNodes; private final ApplicationId application; @@ -48,9 +46,8 @@ public class NodePrioritizer { public NodePrioritizer(LockedNodeList allNodes, ApplicationId application, ClusterSpec clusterSpec, NodeSpec nodeSpec, int wantedGroups, boolean dynamicProvisioning, NameResolver nameResolver, - HostResourcesCalculator hostResourcesCalculator, int spareCount, String allocateOsRequirement) { + HostResourcesCalculator hostResourcesCalculator, int spareCount) { this.allNodes = allNodes; - this.allocateOsRequirement = allocateOsRequirement; this.capacity = new HostCapacity(allNodes, hostResourcesCalculator); this.requestedNodes = nodeSpec; this.clusterSpec = clusterSpec; @@ -140,13 +137,6 @@ public class NodePrioritizer { if (host.reservedTo().isPresent() && !host.reservedTo().get().equals(application.tenant())) continue; if (host.reservedTo().isPresent() && application.instance().isTester()) continue; if (host.exclusiveTo().isPresent()) continue; // Never allocate new nodes to exclusive hosts - - if (host.status().osVersion().isBefore(new Version(8))) { - if (allocateOsRequirement.equals("rhel8")) continue; - } else { - if (allocateOsRequirement.equals("rhel7")) continue; - } - if (spareHosts.contains(host) && !canAllocateToSpareHosts) continue; if ( ! capacity.hasCapacity(host, requestedNodes.resources().get())) continue; if ( ! allNodes.childrenOf(host).owner(application).cluster(clusterSpec.id()).isEmpty()) continue; diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeRepositoryProvisioner.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeRepositoryProvisioner.java index ee8ce23a5c0..24d23f13bb5 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeRepositoryProvisioner.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeRepositoryProvisioner.java @@ -8,7 +8,6 @@ import com.yahoo.config.provision.ApplicationTransaction; import com.yahoo.config.provision.Capacity; import com.yahoo.config.provision.ClusterResources; import com.yahoo.config.provision.ClusterSpec; -import com.yahoo.config.provision.Flavor; import com.yahoo.config.provision.HostFilter; import com.yahoo.config.provision.HostSpec; import com.yahoo.config.provision.NodeResources; @@ -72,7 +71,6 @@ public class NodeRepositoryProvisioner implements Provisioner { .map(lbService -> new LoadBalancerProvisioner(nodeRepository, lbService)); this.nodeResourceLimits = new NodeResourceLimits(nodeRepository); this.preparer = new Preparer(nodeRepository, - flagSource, provisionServiceProvider.getHostProvisioner(), loadBalancerProvisioner); this.activator = new Activator(nodeRepository, loadBalancerProvisioner); @@ -122,7 +120,7 @@ public class NodeRepositoryProvisioner implements Provisioner { @Override public void restart(ApplicationId application, HostFilter filter) { - nodeRepository.nodes().restart(ApplicationFilter.from(application).and(NodeHostFilter.from(filter))); + nodeRepository.nodes().restartActive(ApplicationFilter.from(application).and(NodeHostFilter.from(filter))); } @Override diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/Preparer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/Preparer.java index 97f935d273b..3fa44d4c091 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/Preparer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/Preparer.java @@ -5,7 +5,6 @@ import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.ClusterMembership; import com.yahoo.config.provision.ClusterSpec; import com.yahoo.config.provision.OutOfCapacityException; -import com.yahoo.vespa.flags.FlagSource; import com.yahoo.vespa.hosted.provision.Node; import com.yahoo.vespa.hosted.provision.NodeList; import com.yahoo.vespa.hosted.provision.NodeRepository; @@ -28,11 +27,11 @@ class Preparer { private final GroupPreparer groupPreparer; private final Optional<LoadBalancerProvisioner> loadBalancerProvisioner; - public Preparer(NodeRepository nodeRepository, FlagSource flagSource, Optional<HostProvisioner> hostProvisioner, + public Preparer(NodeRepository nodeRepository, Optional<HostProvisioner> hostProvisioner, Optional<LoadBalancerProvisioner> loadBalancerProvisioner) { this.nodeRepository = nodeRepository; this.loadBalancerProvisioner = loadBalancerProvisioner; - this.groupPreparer = new GroupPreparer(nodeRepository, hostProvisioner, flagSource); + this.groupPreparer = new GroupPreparer(nodeRepository, hostProvisioner); } /** Prepare all required resources for the given application and cluster */ diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/ApplicationSerializer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/ApplicationSerializer.java index 380003affb3..e800f6c9c84 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/ApplicationSerializer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/ApplicationSerializer.java @@ -15,6 +15,7 @@ import com.yahoo.vespa.hosted.provision.autoscale.MetricsDb; import java.net.URI; import java.util.List; +import java.util.Optional; /** * Serializes application information for nodes/v2/application responses @@ -61,8 +62,7 @@ public class ApplicationSerializer { NodeList nodes = applicationNodes.not().retired().cluster(cluster.id()); if (nodes.isEmpty()) return; ClusterResources currentResources = nodes.toResources(); - ClusterModel clusterModel = new ClusterModel(application, cluster, nodes.clusterSpec(), nodes, metricsDb, nodeRepository.clock()); - + Optional<ClusterModel> clusterModel = ClusterModel.create(application, cluster, nodes.clusterSpec(), nodes, metricsDb, nodeRepository.clock()); Cursor clusterObject = clustersObject.setObject(cluster.id().value()); clusterObject.setString("type", nodes.clusterSpec().type().name()); toSlime(cluster.minResources(), clusterObject.setObject("min")); @@ -71,12 +71,13 @@ public class ApplicationSerializer { if (cluster.shouldSuggestResources(currentResources)) cluster.suggestedResources().ifPresent(suggested -> toSlime(suggested.resources(), clusterObject.setObject("suggested"))); cluster.targetResources().ifPresent(target -> toSlime(target, clusterObject.setObject("target"))); - clusterUtilizationToSlime(clusterModel, clusterObject.setObject("utilization")); + clusterModel.ifPresent(model -> clusterUtilizationToSlime(model, clusterObject.setObject("utilization"))); scalingEventsToSlime(cluster.scalingEvents(), clusterObject.setArray("scalingEvents")); - clusterObject.setString("autoscalingStatus", cluster.autoscalingStatus()); - clusterObject.setLong("scalingDuration", clusterModel.scalingDuration().toMillis()); - clusterObject.setDouble("maxQueryGrowthRate", clusterModel.maxQueryGrowthRate()); - clusterObject.setDouble("currentQueryFractionOfMax", clusterModel.queryFractionOfMax()); + clusterObject.setString("autoscalingStatusCode", cluster.autoscalingStatus().status().name()); + clusterObject.setString("autoscalingStatus", cluster.autoscalingStatus().description()); + clusterModel.ifPresent(model -> clusterObject.setLong("scalingDuration", model.scalingDuration().toMillis())); + clusterModel.ifPresent(model -> clusterObject.setDouble("maxQueryGrowthRate", model.maxQueryGrowthRate())); + clusterModel.ifPresent(model -> clusterObject.setDouble("currentQueryFractionOfMax", model.queryFractionOfMax())); } private static void toSlime(ClusterResources resources, Cursor clusterResourcesObject) { diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/NodeSerializer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/NodeSerializer.java index b72d021e4f5..706c36b35ac 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/NodeSerializer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/NodeSerializer.java @@ -1,9 +1,7 @@ // Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.provision.restapi; -import com.yahoo.config.provision.NodeResources; import com.yahoo.config.provision.NodeType; -import com.yahoo.slime.Cursor; import com.yahoo.vespa.hosted.provision.Node; /** diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/NodesV2ApiHandler.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/NodesV2ApiHandler.java index c7bd12204a6..24e297de179 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/NodesV2ApiHandler.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/NodesV2ApiHandler.java @@ -195,7 +195,7 @@ public class NodesV2ApiHandler extends LoggingRequestHandler { private HttpResponse handlePOST(HttpRequest request) { Path path = new Path(request.getUri()); if (path.matches("/nodes/v2/command/restart")) { - int restartCount = nodeRepository.nodes().restart(toNodeFilter(request)).size(); + int restartCount = nodeRepository.nodes().restartActive(toNodeFilter(request)).size(); return new MessageResponse("Scheduled restart of " + restartCount + " matching nodes"); } if (path.matches("/nodes/v2/command/reboot")) { diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/AutoscalingTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/AutoscalingTest.java index 98bfa2170a5..41a399c5e2f 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/AutoscalingTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/AutoscalingTest.java @@ -15,6 +15,7 @@ import com.yahoo.config.provision.SystemName; import com.yahoo.config.provision.Zone; import com.yahoo.vespa.hosted.provision.NodeRepository; import com.yahoo.vespa.hosted.provision.Nodelike; +import com.yahoo.vespa.hosted.provision.provisioning.CapacityPolicies; import com.yahoo.vespa.hosted.provision.provisioning.HostResourcesCalculator; import org.junit.Test; @@ -219,6 +220,32 @@ public class AutoscalingTest { } @Test + public void autoscaling_with_unspecified_resources_use_defaults() { + NodeResources hostResources = new NodeResources(6, 100, 100, 1); + ClusterResources min = new ClusterResources( 2, 1, NodeResources.unspecified()); + ClusterResources max = new ClusterResources( 6, 1, NodeResources.unspecified()); + AutoscalingTester tester = new AutoscalingTester(hostResources); + + ApplicationId application1 = tester.applicationId("application1"); + ClusterSpec cluster1 = tester.clusterSpec(ClusterSpec.Type.container, "cluster1"); + + NodeResources defaultResources = + new CapacityPolicies(tester.nodeRepository()).defaultNodeResources(cluster1.type()); + + // deploy + tester.deploy(application1, cluster1, Capacity.from(min, max)); + tester.assertResources("Min number of nodes and default resources", + 2, 1, defaultResources, + Optional.of(tester.nodeRepository().nodes().list().owner(application1).toResources())); + tester.addMeasurements(0.25f, 0.95f, 0.95f, 0, 120, application1); + tester.clock().advance(Duration.ofMinutes(-10 * 5)); + tester.addQueryRateMeasurements(application1, cluster1.id(), 10, t -> t == 0 ? 20.0 : 10.0); // Query traffic only + tester.assertResources("Scaling up to limit since resource usage is too high", + 4, 1, defaultResources, + tester.autoscale(application1, cluster1.id(), min, max).target()); + } + + @Test public void autoscaling_respects_group_limit() { NodeResources hostResources = new NodeResources(30.0, 100, 100, 1); ClusterResources min = new ClusterResources( 2, 2, new NodeResources(1, 1, 1, 1)); diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/AutoscalingTester.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/AutoscalingTester.java index 5fe6023e5af..f96679b7195 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/AutoscalingTester.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/AutoscalingTester.java @@ -267,7 +267,7 @@ class AutoscalingTester { } public Autoscaler.Advice autoscale(ApplicationId applicationId, ClusterSpec.Id clusterId, - ClusterResources min, ClusterResources max) { + ClusterResources min, ClusterResources max) { Application application = nodeRepository().applications().get(applicationId).orElse(Application.empty(applicationId)) .withCluster(clusterId, false, min, max); try (Mutex lock = nodeRepository().nodes().lock(applicationId)) { @@ -278,7 +278,7 @@ class AutoscalingTester { } public Autoscaler.Advice suggest(ApplicationId applicationId, ClusterSpec.Id clusterId, - ClusterResources min, ClusterResources max) { + ClusterResources min, ClusterResources max) { Application application = nodeRepository().applications().get(applicationId).orElse(Application.empty(applicationId)) .withCluster(clusterId, false, min, max); try (Mutex lock = nodeRepository().nodes().lock(applicationId)) { @@ -290,6 +290,15 @@ class AutoscalingTester { public ClusterResources assertResources(String message, int nodeCount, int groupCount, + NodeResources expectedResources, + Optional<ClusterResources> resources) { + return assertResources(message, nodeCount, groupCount, + expectedResources.vcpu(), expectedResources.memoryGb(), expectedResources.diskGb(), + resources); + } + + public ClusterResources assertResources(String message, + int nodeCount, int groupCount, double approxCpu, double approxMemory, double approxDisk, Optional<ClusterResources> resources) { double delta = 0.0000000001; diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/ClusterModelTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/ClusterModelTest.java index 550ecceee23..af1bd2aa231 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/ClusterModelTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/ClusterModelTest.java @@ -7,6 +7,7 @@ import com.yahoo.config.provision.ClusterSpec; import com.yahoo.config.provision.NodeResources; import com.yahoo.test.ManualClock; import com.yahoo.vespa.hosted.provision.applications.Application; +import com.yahoo.vespa.hosted.provision.applications.AutoscalingStatus; import com.yahoo.vespa.hosted.provision.applications.Cluster; import com.yahoo.vespa.hosted.provision.applications.Status; import org.junit.Test; @@ -36,13 +37,15 @@ public class ClusterModelTest { // No current traffic share: Ideal load is low but capped var model1 = new ClusterModel(application.with(new Status(0.0, 1.0)), cluster, clock, Duration.ofMinutes(10), - timeseries(cluster,100, t -> t == 0 ? 10000.0 : 0.0, t -> 0.0, clock)); + timeseries(cluster,100, t -> t == 0 ? 10000.0 : 0.0, t -> 0.0, clock), + ClusterNodesTimeseries.empty()); assertEquals(0.131, model1.idealLoad().cpu(), delta); // Almost no current traffic share: Ideal load is low but capped var model2 = new ClusterModel(application.with(new Status(0.0001, 1.0)), cluster, clock, Duration.ofMinutes(10), - timeseries(cluster,100, t -> t == 0 ? 10000.0 : 0.0, t -> 0.0, clock)); + timeseries(cluster,100, t -> t == 0 ? 10000.0 : 0.0, t -> 0.0, clock), + ClusterNodesTimeseries.empty()); assertEquals(0.131, model2.idealLoad().cpu(), delta); } @@ -57,13 +60,15 @@ public class ClusterModelTest { // No current traffic: Ideal load is low but capped var model1 = new ClusterModel(application, cluster, clock, Duration.ofMinutes(10), - timeseries(cluster,100, t -> t == 0 ? 10000.0 : 0.0, t -> 0.0, clock)); + timeseries(cluster,100, t -> t == 0 ? 10000.0 : 0.0, t -> 0.0, clock), + ClusterNodesTimeseries.empty()); assertEquals(0.275, model1.idealLoad().cpu(), delta); // Almost no current traffic: Ideal load is low but capped var model2 = new ClusterModel(application.with(new Status(0.0001, 1.0)), cluster, clock, Duration.ofMinutes(10), - timeseries(cluster,100, t -> t == 0 ? 10000.0 : 0.0001, t -> 0.0, clock)); + timeseries(cluster,100, t -> t == 0 ? 10000.0 : 0.0001, t -> 0.0, clock), + ClusterNodesTimeseries.empty()); assertEquals(0.040, model2.idealLoad().cpu(), delta); } @@ -75,7 +80,7 @@ public class ClusterModelTest { Optional.empty(), Optional.empty(), List.of(), - ""); + AutoscalingStatus.empty()); } /** Creates the given number of measurements, spaced 5 minutes between, using the given function */ diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/MetricsV2MetricsFetcherTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/MetricsV2MetricsFetcherTest.java index 5f1a36e7b56..07f8fa41d24 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/MetricsV2MetricsFetcherTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/MetricsV2MetricsFetcherTest.java @@ -56,9 +56,9 @@ public class MetricsV2MetricsFetcherTest { assertEquals(0.820, values.get(0).getSecond().load().disk(), delta); assertEquals("host-2.yahoo.com", values.get(1).getFirst()); - assertEquals(0.2, values.get(1).getSecond().load().cpu(), delta); - assertEquals(0.0, values.get(1).getSecond().load().memory(), delta); - assertEquals(0.4, values.get(1).getSecond().load().disk(), delta); + assertEquals(0.0, values.get(1).getSecond().load().cpu(), delta); + assertEquals(0.35, values.get(1).getSecond().load().memory(), delta); + assertEquals(0.45, values.get(1).getSecond().load().disk(), delta); assertEquals(45.0, values.get(1).getSecond().queryRate(), delta); } @@ -134,7 +134,7 @@ public class MetricsV2MetricsFetcherTest { " \"metrics\": [\n" + " {\n" + " \"values\": {\n" + - " \"cpu.util\": 20,\n" + + " \"mem.util\": 30,\n" + " \"disk.util\": 40\n" + " },\n" + " \"dimensions\": {\n" + @@ -161,6 +161,14 @@ public class MetricsV2MetricsFetcherTest { " },\n" + " {\n" + " \"values\": {\n" + + " \"content.proton.resource_usage.memory.average\": 0.35,\n" + + " \"content.proton.resource_usage.disk.average\": 0.45\n" + + " },\n" + + " \"dimensions\": {\n" + + " }\n" + + " },\n" + + " {\n" + + " \"values\": {\n" + " \"content.proton.documentdb.matching.queries.rate\": 13.5\n" + " },\n" + " \"dimensions\": {\n" + diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/HostEncrypterTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/HostEncrypterTest.java index b3c78aa4627..5cae181f87d 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/HostEncrypterTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/HostEncrypterTest.java @@ -22,22 +22,26 @@ import java.time.Instant; import java.util.Comparator; import java.util.List; import java.util.Optional; +import java.util.function.Consumer; import java.util.function.Supplier; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertSame; +import static org.junit.Assert.assertTrue; /** * @author mpolden */ public class HostEncrypterTest { + private final ApplicationId infraApplication = ApplicationId.from("hosted-vespa", "infra", "default"); private final ProvisioningTester tester = new ProvisioningTester.Builder().build(); + private final HostEncrypter encrypter = new HostEncrypter(tester.nodeRepository(), Duration.ofDays(1), new MockMetric()); @Test public void no_hosts_encrypted_with_default_flag_value() { provisionHosts(1); - HostEncrypter encrypter = new HostEncrypter(tester.nodeRepository(), Duration.ofDays(1), new MockMetric()); encrypter.maintain(); assertEquals(0, tester.nodeRepository().nodes().list().encrypting().size()); } @@ -46,7 +50,6 @@ public class HostEncrypterTest { public void encrypt_hosts() { tester.flagSource().withIntFlag(Flags.MAX_ENCRYPTING_HOSTS.id(), 3); Supplier<NodeList> hosts = () -> tester.nodeRepository().nodes().list().nodeType(NodeType.host); - HostEncrypter encrypter = new HostEncrypter(tester.nodeRepository(), Duration.ofDays(1), new MockMetric()); // Provision hosts and deploy applications int hostCount = 5; @@ -67,7 +70,7 @@ public class HostEncrypterTest { assertEquals(owners.size(), hostsEncrypting.size()); for (int i = 0; i < hostsEncrypting.size(); i++) { Optional<ApplicationId> owner = owners.get(i); - List<Node> retiringChildren = allNodes.childrenOf(hostsEncrypting.get(i)).retiring().asList(); + List<Node> retiringChildren = allNodes.childrenOf(hostsEncrypting.get(i)).retiring().encrypting().asList(); assertEquals(owner.isPresent() ? 1 : 0, retiringChildren.size()); assertEquals("Encrypting host of " + owner.map(ApplicationId::toString) .orElse("no application"), @@ -116,19 +119,24 @@ public class HostEncrypterTest { List<Node> provisionedHosts = tester.makeReadyNodes(hostCount, new NodeResources(48, 128, 2000, 10), NodeType.host, 10); // Set OS version supporting encryption tester.patchNodes(provisionedHosts, (host) -> host.with(host.status().withOsVersion(host.status().osVersion().withCurrent(Optional.of(Version.fromString("8.0")))))); - tester.activateTenantHosts(); + tester.prepareAndActivateInfraApplication(infraApplication, NodeType.host); } private void completeEncryptionOf(List<Node> nodes) { Instant now = tester.clock().instant(); - tester.patchNodes(nodes, (node) -> { - if (node.reports().getReport(Report.WANT_TO_ENCRYPT_ID).isEmpty()) throw new IllegalArgumentException(node + " is not requested to encrypt"); + // Redeploy to park retired hosts + replaceNodes(infraApplication, (application) -> tester.prepareAndActivateInfraApplication(application, NodeType.host)); + List<Node> patchedNodes = tester.patchNodes(nodes, (node) -> { + assertSame(Node.State.parked, node.state()); + assertTrue(node + " wants to encrypt", node.reports().getReport(Report.WANT_TO_ENCRYPT_ID).isPresent()); return node.with(node.reports().withReport(Report.basicReport(Report.DISK_ENCRYPTED_ID, Report.Type.UNSPECIFIED, now, - "Host is encrypted"))) - .withWantToRetire(false, Agent.system, now); + "Host is encrypted"))); }); + patchedNodes = tester.nodeRepository().nodes().deallocate(patchedNodes, Agent.system, getClass().getSimpleName()); + tester.nodeRepository().nodes().setReady(patchedNodes, Agent.system, getClass().getSimpleName()); + tester.activateTenantHosts(); } private void deployApplication(ApplicationId application) { @@ -138,14 +146,18 @@ public class HostEncrypterTest { } private void replaceNodes(ApplicationId application) { + replaceNodes(application, this::deployApplication); + } + + private void replaceNodes(ApplicationId application, Consumer<ApplicationId> deployer) { // Deploy to retire nodes - deployApplication(application); + deployer.accept(application); List<Node> retired = tester.nodeRepository().nodes().list().owner(application).retired().asList(); assertFalse("At least one node is retired", retired.isEmpty()); tester.nodeRepository().nodes().setRemovable(application, retired); // Redeploy to deactivate removable nodes and allocate new ones - deployApplication(application); + deployer.accept(application); tester.nodeRepository().nodes().list(Node.State.inactive).owner(application) .forEach(node -> tester.nodeRepository().nodes().removeRecursively(node, true)); } diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/NodeMetricsDbMaintainerTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/NodeMetricsDbMaintainerTest.java index 5b2f7ce91e8..cfe6e4d348d 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/NodeMetricsDbMaintainerTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/NodeMetricsDbMaintainerTest.java @@ -43,7 +43,7 @@ public class NodeMetricsDbMaintainerTest { fetcher, Duration.ofHours(1), new TestMetric()); - assertTrue(maintainer.maintain()); + assertEquals(maintainer.maintain(), 1.0, 0.0000001); List<NodeTimeseries> timeseriesList = tester.nodeRepository().metricsDb().getNodeTimeseries(Duration.ofDays(1), Set.of("host-1.yahoo.com", "host-2.yahoo.com")); assertEquals(2, timeseriesList.size()); diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/persistence/ApplicationSerializerTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/persistence/ApplicationSerializerTest.java index 9cac6430d6e..61ff494679f 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/persistence/ApplicationSerializerTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/persistence/ApplicationSerializerTest.java @@ -6,6 +6,7 @@ import com.yahoo.config.provision.ClusterResources; import com.yahoo.config.provision.ClusterSpec; import com.yahoo.config.provision.NodeResources; import com.yahoo.vespa.hosted.provision.applications.Application; +import com.yahoo.vespa.hosted.provision.applications.AutoscalingStatus; import com.yahoo.vespa.hosted.provision.applications.Cluster; import com.yahoo.vespa.hosted.provision.applications.ScalingEvent; import com.yahoo.vespa.hosted.provision.applications.Status; @@ -35,7 +36,7 @@ public class ApplicationSerializerTest { Optional.empty(), Optional.empty(), List.of(), - "")); + AutoscalingStatus.empty())); var minResources = new NodeResources(1, 2, 3, 4); clusters.add(new Cluster(ClusterSpec.Id.from("c2"), true, @@ -50,7 +51,7 @@ public class ApplicationSerializerTest { 7L, Instant.ofEpochMilli(12345L), Optional.of(Instant.ofEpochMilli(67890L)))), - "Autoscaling status")); + new AutoscalingStatus(AutoscalingStatus.Status.insufficient, "Autoscaling status"))); Application original = new Application(ApplicationId.from("myTenant", "myApplication", "myInstance"), Status.initial().withCurrentReadShare(0.3).withMaxReadShare(0.5), clusters); diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/persistence/CuratorDatabaseClientTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/persistence/CuratorDatabaseClientTest.java index f47fb7f23be..99f6ce4fb00 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/persistence/CuratorDatabaseClientTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/persistence/CuratorDatabaseClientTest.java @@ -1,4 +1,4 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.provision.persistence; import com.yahoo.config.provision.ApplicationId; @@ -6,7 +6,6 @@ import com.yahoo.config.provision.ApplicationName; import com.yahoo.config.provision.InstanceName; import com.yahoo.config.provision.NodeType; import com.yahoo.config.provision.TenantName; -import com.yahoo.config.provision.Zone; import com.yahoo.vespa.curator.Curator; import com.yahoo.vespa.curator.mock.MockCurator; import com.yahoo.vespa.hosted.provision.Node; @@ -25,7 +24,7 @@ public class CuratorDatabaseClientTest { private final Curator curator = new MockCurator(); private final CuratorDatabaseClient zkClient = new CuratorDatabaseClient( - FlavorConfigBuilder.createDummies("default"), curator, Clock.systemUTC(), Zone.defaultZone(), true, 1000); + FlavorConfigBuilder.createDummies("default"), curator, Clock.systemUTC(), true, 1000); @Test public void can_read_stored_host_information() throws Exception { diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/application1.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/application1.json index 1083930e294..ca0c548c1be 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/application1.json +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/application1.json @@ -104,6 +104,7 @@ "at" : 123 } ], + "autoscalingStatusCode": "unavailable", "autoscalingStatus": "", "scalingDuration": 600000, "maxQueryGrowthRate": 0.1, diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/application2.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/application2.json index 61e0569d349..c7eaa4af974 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/application2.json +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/application2.json @@ -80,6 +80,7 @@ "at" : 123 } ], + "autoscalingStatusCode": "unavailable", "autoscalingStatus" : "", "scalingDuration": 43200000, "maxQueryGrowthRate": 0.1, diff --git a/searchcore/src/tests/proton/verify_ranksetup/verify_ranksetup_test.cpp b/searchcore/src/tests/proton/verify_ranksetup/verify_ranksetup_test.cpp index 31557f13a54..fc70bafed7f 100644 --- a/searchcore/src/tests/proton/verify_ranksetup/verify_ranksetup_test.cpp +++ b/searchcore/src/tests/proton/verify_ranksetup/verify_ranksetup_test.cpp @@ -98,7 +98,9 @@ struct Setup { property(fmt("rankingExpression(%s).rankingScript", name.c_str()), expr); } void ext_rank_expr(const std::string &name, const std::string &file) { - ranking_expressions.insert_or_assign(name, TEST_PATH(file)); + auto expr_name = fmt("my_expr_%s", name.c_str()); + property(fmt("rankingExpression(%s).expressionName", name.c_str()), expr_name); + ranking_expressions.insert_or_assign(expr_name, TEST_PATH(file)); } void first_phase(const std::string &feature) { property(rank::FirstPhase::NAME, feature); diff --git a/searchcore/src/vespa/searchcore/proton/matching/ranking_expressions.cpp b/searchcore/src/vespa/searchcore/proton/matching/ranking_expressions.cpp index 9b74f76aa6e..49cf08d76c0 100644 --- a/searchcore/src/vespa/searchcore/proton/matching/ranking_expressions.cpp +++ b/searchcore/src/vespa/searchcore/proton/matching/ranking_expressions.cpp @@ -40,8 +40,7 @@ RankingExpressions::loadExpression(const vespalib::string &name) const { auto pos = _expressions.find(name); if (pos == _expressions.end()) { - // not warning about missing expression here since what we - // think is a name might be an expression itself. + LOG(warning, "no such ranking expression: '%s'", name.c_str()); return {}; } auto path = pos->second; diff --git a/searchlib/abi-spec.json b/searchlib/abi-spec.json index 9e958dd4d4c..2d006bbd973 100644 --- a/searchlib/abi-spec.json +++ b/searchlib/abi-spec.json @@ -333,6 +333,7 @@ "public int hashCode()", "public boolean equals(java.lang.Object)", "public java.lang.String toString()", + "public java.util.Map getRankProperties(com.yahoo.searchlib.rankingexpression.rule.SerializationContext)", "public java.util.Map getRankProperties(java.util.List)", "public static java.lang.String propertyName(java.lang.String)", "public com.yahoo.tensor.TensorType type(com.yahoo.tensor.evaluation.TypeContext)", @@ -1330,6 +1331,7 @@ "public int hashCode()", "public final boolean equals(java.lang.Object)", "public final java.lang.String toString()", + "public final java.lang.StringBuilder toString(com.yahoo.searchlib.rankingexpression.rule.SerializationContext)", "public abstract java.lang.StringBuilder toString(java.lang.StringBuilder, com.yahoo.searchlib.rankingexpression.rule.SerializationContext, java.util.Deque, com.yahoo.searchlib.rankingexpression.rule.CompositeNode)", "public abstract com.yahoo.tensor.TensorType type(com.yahoo.tensor.evaluation.TypeContext)", "public abstract com.yahoo.searchlib.rankingexpression.evaluation.Value evaluate(com.yahoo.searchlib.rankingexpression.evaluation.Context)" @@ -1581,6 +1583,7 @@ "public com.yahoo.searchlib.rankingexpression.rule.SerializationContext withBindings(java.util.Map)", "public com.yahoo.searchlib.rankingexpression.rule.SerializationContext withoutBindings()", "public java.util.Map serializedFunctions()", + "public boolean needSerialization(java.lang.String)", "public bridge synthetic com.yahoo.searchlib.rankingexpression.rule.FunctionReferenceContext withoutBindings()", "public bridge synthetic com.yahoo.searchlib.rankingexpression.rule.FunctionReferenceContext withBindings(java.util.Map)" ], diff --git a/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/RankingExpression.java b/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/RankingExpression.java index 1f27bc8750e..3eb4f16a9dd 100755 --- a/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/RankingExpression.java +++ b/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/RankingExpression.java @@ -19,7 +19,6 @@ import java.io.Reader; import java.io.Serializable; import java.io.StringReader; import java.util.Deque; -import java.util.HashMap; import java.util.LinkedList; import java.util.List; import java.util.Map; @@ -251,18 +250,22 @@ public class RankingExpression implements Serializable { /** * Creates the necessary rank properties required to implement this expression. * - * @param functions the expression functions to expand + * @param context context for serialization * @return a list of named rank properties required to implement this expression */ - public Map<String, String> getRankProperties(List<ExpressionFunction> functions) { + public Map<String, String> getRankProperties(SerializationContext context) { Deque<String> path = new LinkedList<>(); - SerializationContext context = new SerializationContext(functions); String serializedRoot = root.toString(new StringBuilder(), context, path, null).toString(); Map<String, String> serializedExpressions = context.serializedFunctions(); serializedExpressions.put(propertyName(name), serializedRoot); return serializedExpressions; } + @Deprecated + public Map<String, String> getRankProperties(List<ExpressionFunction> functions) { + return getRankProperties(new SerializationContext(functions)); + } + /** * Returns the rank-property name for a given expression name. * diff --git a/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/rule/ExpressionNode.java b/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/rule/ExpressionNode.java index dfcdf1e2662..151b70d763d 100755 --- a/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/rule/ExpressionNode.java +++ b/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/rule/ExpressionNode.java @@ -30,7 +30,10 @@ public abstract class ExpressionNode implements Serializable { @Override public final String toString() { - return toString(new StringBuilder(), new SerializationContext(), null, null).toString(); + return toString(new SerializationContext()).toString(); + } + public final StringBuilder toString(SerializationContext context) { + return toString(new StringBuilder(), context, null, null); } /** diff --git a/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/rule/ReferenceNode.java b/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/rule/ReferenceNode.java index 335d3861d9d..e96ba021750 100755 --- a/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/rule/ReferenceNode.java +++ b/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/rule/ReferenceNode.java @@ -85,10 +85,9 @@ public final class ReferenceNode extends CompositeNode { path.addLast(myPath); String functionName = getName(); - String functionPropertyName = RankingExpression.propertyName(functionName); - boolean alreadySerialized = context.serializedFunctions().containsKey(functionPropertyName); + boolean needSerialization = (getArguments().size() > 0) || context.needSerialization(functionName); - if ( ! alreadySerialized || getArguments().size() > 0) { + if ( needSerialization ) { ExpressionFunction.Instance instance = function.expand(context, getArguments().expressions(), path); functionName = instance.getName(); diff --git a/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/rule/SerializationContext.java b/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/rule/SerializationContext.java index 92889d6607b..a6a64348452 100644 --- a/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/rule/SerializationContext.java +++ b/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/rule/SerializationContext.java @@ -3,6 +3,7 @@ package com.yahoo.searchlib.rankingexpression.rule; import com.google.common.collect.ImmutableMap; import com.yahoo.searchlib.rankingexpression.ExpressionFunction; +import com.yahoo.searchlib.rankingexpression.RankingExpression; import com.yahoo.tensor.TensorType; import java.util.Collection; @@ -103,4 +104,8 @@ public class SerializationContext extends FunctionReferenceContext { public Map<String, String> serializedFunctions() { return serializedFunctions; } + public boolean needSerialization(String functionName) { + return ! serializedFunctions().containsKey(RankingExpression.propertyName(functionName)); + } + } diff --git a/searchlib/src/test/java/com/yahoo/searchlib/rankingexpression/RankingExpressionTestCase.java b/searchlib/src/test/java/com/yahoo/searchlib/rankingexpression/RankingExpressionTestCase.java index ea09de32137..092faa1934e 100755 --- a/searchlib/src/test/java/com/yahoo/searchlib/rankingexpression/RankingExpressionTestCase.java +++ b/searchlib/src/test/java/com/yahoo/searchlib/rankingexpression/RankingExpressionTestCase.java @@ -9,6 +9,7 @@ import com.yahoo.searchlib.rankingexpression.rule.IfNode; import com.yahoo.searchlib.rankingexpression.rule.ExpressionNode; import com.yahoo.searchlib.rankingexpression.rule.FunctionNode; import com.yahoo.searchlib.rankingexpression.rule.ReferenceNode; +import com.yahoo.searchlib.rankingexpression.rule.SerializationContext; import com.yahoo.searchlib.rankingexpression.rule.TensorFunctionNode; import com.yahoo.tensor.functions.Reduce; import org.junit.Test; @@ -92,7 +93,7 @@ public class RankingExpressionTestCase { RankingExpression exp = new RankingExpression("foo"); try { - exp.getRankProperties(functions); + exp.getRankProperties(new SerializationContext(functions)); } catch (RuntimeException e) { assertEquals("Cycle in ranking expression function: [foo[]]", e.getMessage()); } @@ -100,13 +101,13 @@ public class RankingExpressionTestCase { @Test public void testFunctionCycleSerialization() throws ParseException { - List<ExpressionFunction> funnctions = new ArrayList<>(); - funnctions.add(new ExpressionFunction("foo", null, new RankingExpression("bar"))); - funnctions.add(new ExpressionFunction("bar", null, new RankingExpression("foo"))); + List<ExpressionFunction> functions = new ArrayList<>(); + functions.add(new ExpressionFunction("foo", null, new RankingExpression("bar"))); + functions.add(new ExpressionFunction("bar", null, new RankingExpression("foo"))); RankingExpression exp = new RankingExpression("foo"); try { - exp.getRankProperties(funnctions); + exp.getRankProperties(new SerializationContext(functions)); } catch (RuntimeException e) { assertEquals("Cycle in ranking expression function: [foo[], bar[]]", e.getMessage()); } @@ -350,7 +351,7 @@ public class RankingExpressionTestCase { try { RankingExpression expression = new RankingExpression(expressionString); // No functions -> expect one rank property - serializedExpression = expression.getRankProperties(Collections.emptyList()).values().iterator().next(); + serializedExpression = expression.getRankProperties(new SerializationContext()).values().iterator().next(); assertEquals(expectedSerialization, serializedExpression); } catch (ParseException e) { @@ -363,7 +364,7 @@ public class RankingExpressionTestCase { RankingExpression reparsedExpression = new RankingExpression(serializedExpression); // Serializing the primitivized expression should yield the same expression again String reserializedExpression = - reparsedExpression.getRankProperties(Collections.emptyList()).values().iterator().next(); + reparsedExpression.getRankProperties(new SerializationContext()).values().iterator().next(); assertEquals(expectedSerialization, reserializedExpression); } catch (ParseException e) { @@ -383,7 +384,7 @@ public class RankingExpressionTestCase { System.out.println("Parsing expression '" + expressionString + "':"); RankingExpression expression = new RankingExpression(expressionString); - Map<String, String> rankProperties = expression.getRankProperties(functions); + Map<String, String> rankProperties = expression.getRankProperties(new SerializationContext(functions)); if (print) { for (String key : rankProperties.keySet()) System.out.println(key + ": " + rankProperties.get(key)); diff --git a/searchlib/src/tests/attribute/attribute_test.cpp b/searchlib/src/tests/attribute/attribute_test.cpp index af1fcea2e21..79e120d0683 100644 --- a/searchlib/src/tests/attribute/attribute_test.cpp +++ b/searchlib/src/tests/attribute/attribute_test.cpp @@ -1670,8 +1670,8 @@ AttributeTest::testStatus() AttributePtr ptr = createAttribute("as", cfg); addDocs(ptr, numDocs); auto & sa = *(static_cast<StringAttribute *>(ptr.get())); - const size_t numUniq(16); - const size_t numValuesPerDoc(16); + const size_t numValuesPerDoc(values.size()); + const size_t numUniq(numValuesPerDoc); for (uint32_t i = 0; i < numDocs; ++i) { EXPECT_TRUE(appendToVector(sa, i, numValuesPerDoc, values)); } @@ -1680,11 +1680,11 @@ AttributeTest::testStatus() EXPECT_EQUAL(ptr->getStatus().getNumValues(), numDocs*numValuesPerDoc); EXPECT_EQUAL(ptr->getStatus().getNumUniqueValues(), numUniq); size_t expUsed = 0; - expUsed += 1 * InternalNodeSize + 1 * LeafNodeSize; // enum store tree - expUsed += numUniq * 32; // enum store (16 unique values, 32 bytes per entry) + expUsed += 1 * InternalNodeSize + 1 * LeafNodeSize; // Approximate enum store tree + expUsed += 272; // TODO Approximate... enum store (16 unique values, 17 bytes per entry) // multi value mapping (numdocs * sizeof(MappingIndex) + numvalues * sizeof(EnumIndex) + - // numdocs * sizeof(Array<EnumIndex>) (due to vector vector)) - expUsed += numDocs * sizeof(vespalib::datastore::EntryRef) + numDocs * numValuesPerDoc * sizeof(IEnumStore::Index) + ((numValuesPerDoc > 1024) ? numDocs * NestedVectorSize : 0); + // 32 + numdocs * sizeof(Array<EnumIndex>) (due to vector vector)) + expUsed += 32 + numDocs * sizeof(vespalib::datastore::EntryRef) + numDocs * numValuesPerDoc * sizeof(IEnumStore::Index) + ((numValuesPerDoc > 1024) ? numDocs * NestedVectorSize : 0); EXPECT_GREATER_EQUAL(ptr->getStatus().getUsed(), expUsed); EXPECT_GREATER_EQUAL(ptr->getStatus().getAllocated(), expUsed); } diff --git a/searchlib/src/tests/attribute/changevector/changevector_test.cpp b/searchlib/src/tests/attribute/changevector/changevector_test.cpp index ad33774e904..93a23630407 100644 --- a/searchlib/src/tests/attribute/changevector/changevector_test.cpp +++ b/searchlib/src/tests/attribute/changevector/changevector_test.cpp @@ -2,17 +2,27 @@ #include <vespa/vespalib/testkit/testapp.h> #include <vespa/searchlib/attribute/changevector.hpp> - +#include <vespa/vespalib/stllike/hash_set.h> using namespace search; +using Change = ChangeTemplate<NumericChangeData<long>>; +using CV = ChangeVectorT<Change>; + template <typename T> void verifyStrictOrdering(const T & v) { - long count(0); - for (const auto & c : v) { - count++; - EXPECT_EQUAL(count, c._data.get()); + vespalib::hash_set<uint32_t> complete; + uint32_t prev_doc(0); + uint32_t prev_value(0); + for (const auto & c : v.getDocIdInsertOrder()) { + if (prev_doc != c._doc) { + complete.insert(prev_doc); + EXPECT_FALSE(complete.contains(c._doc)); + prev_doc = c._doc; + } else { + EXPECT_GREATER(c._data, prev_value); + } + prev_value = c._data; } - EXPECT_EQUAL(v.size(), size_t(count)); } class Accessor { @@ -30,8 +40,6 @@ private: TEST("require insert ordering is preserved for same doc") { - typedef ChangeTemplate<NumericChangeData<long>> Change; - typedef ChangeVectorT<Change> CV; CV a; a.push_back(Change(Change::NOOP, 7, 1)); EXPECT_EQUAL(1u, a.size()); @@ -42,8 +50,6 @@ TEST("require insert ordering is preserved for same doc") TEST("require insert ordering is preserved ") { - typedef ChangeTemplate<NumericChangeData<long>> Change; - typedef ChangeVectorT<Change> CV; CV a; a.push_back(Change(Change::NOOP, 7, 1)); EXPECT_EQUAL(1u, a.size()); @@ -56,8 +62,6 @@ TEST("require insert ordering is preserved ") TEST("require insert ordering is preserved with mix") { - typedef ChangeTemplate<NumericChangeData<long>> Change; - typedef ChangeVectorT<Change> CV; CV a; a.push_back(Change(Change::NOOP, 7, 1)); EXPECT_EQUAL(1u, a.size()); @@ -77,8 +81,6 @@ TEST("require insert ordering is preserved with mix") } TEST("require that inserting empty vector does not affect the vector.") { - typedef ChangeTemplate<NumericChangeData<long>> Change; - typedef ChangeVectorT<Change> CV; CV a; std::vector<long> v; Accessor ac(v); @@ -86,4 +88,42 @@ TEST("require that inserting empty vector does not affect the vector.") { EXPECT_EQUAL(0u, a.size()); } +TEST("require that we have control over buffer construction size") { + CV a; + EXPECT_EQUAL(0u, a.size()); + EXPECT_EQUAL(256u, a.capacity()); + a.clear(); + EXPECT_EQUAL(0u, a.size()); + EXPECT_EQUAL(256u, a.capacity()); +} + +TEST("require that buffer can grow some") { + CV a; + for (size_t i(0); i < 1024; i++) { + a.push_back(Change(Change::NOOP, i, i)); + } + EXPECT_EQUAL(1024u, a.size()); + EXPECT_EQUAL(1024u, a.capacity()); + a.clear(); + EXPECT_EQUAL(0u, a.size()); + EXPECT_EQUAL(1024u, a.capacity()); +} + +TEST("require that buffer can grow some, but not unbound") { + CV a; + for (size_t i(0); i < 1025; i++) { + a.push_back(Change(Change::NOOP, i, i)); + } + EXPECT_EQUAL(1025u, a.size()); + EXPECT_EQUAL(2048u, a.capacity()); + a.clear(); + EXPECT_EQUAL(0u, a.size()); + EXPECT_EQUAL(256u, a.capacity()); +} + +TEST("Control Change size") { + EXPECT_EQUAL(32u, sizeof(ChangeTemplate<NumericChangeData<long>>)); + EXPECT_EQUAL(88u, sizeof(ChangeTemplate<StringChangeData>)); +} + TEST_MAIN() { TEST_RUN_ALL(); } diff --git a/searchlib/src/tests/attribute/tensorattribute/tensorattribute_test.cpp b/searchlib/src/tests/attribute/tensorattribute/tensorattribute_test.cpp index 9621b93fd37..b27c26d1139 100644 --- a/searchlib/src/tests/attribute/tensorattribute/tensorattribute_test.cpp +++ b/searchlib/src/tests/attribute/tensorattribute/tensorattribute_test.cpp @@ -572,12 +572,6 @@ Fixture::testSaveLoad() void Fixture::testCompaction() { - if ((_traits.use_dense_tensor_attribute && _denseTensors) || - ! _traits.use_dense_tensor_attribute) - { - LOG(info, "Skipping compaction test for tensor '%s' which is using free-lists", _cfg.tensorType().to_spec().c_str()); - return; - } ensureSpace(4); TensorSpec empty_xy_tensor(sparseSpec); TensorSpec simple_tensor = TensorSpec(sparseSpec) @@ -596,11 +590,19 @@ Fixture::testCompaction() set_tensor(2, fill_tensor); search::attribute::Status oldStatus = getStatus(); search::attribute::Status newStatus = oldStatus; - uint64_t iter = 0; + auto guard = _attr->makeReadGuard(false); + uint64_t iter = 2049; uint64_t iterLimit = 100000; for (; iter < iterLimit; ++iter) { clearTensor(2); set_tensor(2, fill_tensor); + if ((iter & (iter - 1)) == 0) { + // Temporarily drop read guard when iter crosses a power of 2. + guard.reset(); + _attr->commit(true); + _attr->commit(true); + guard = _attr->makeReadGuard(false); + } newStatus = getStatus(); if (newStatus.getUsed() < oldStatus.getUsed()) { break; @@ -609,7 +611,7 @@ Fixture::testCompaction() } EXPECT_GREATER(iterLimit, iter); LOG(info, - "iter = %" PRIu64 ", memory usage %" PRIu64 ", -> %" PRIu64, + "iter = %" PRIu64 ", memory usage %" PRIu64 " -> %" PRIu64, iter, oldStatus.getUsed(), newStatus.getUsed()); TEST_DO(assertGetNoTensor(1)); TEST_DO(assertGetTensor(fill_tensor, 2)); diff --git a/searchlib/src/tests/features/ranking_expression/ranking_expression_test.cpp b/searchlib/src/tests/features/ranking_expression/ranking_expression_test.cpp index 6baa6581edf..3bc70ee7798 100644 --- a/searchlib/src/tests/features/ranking_expression/ranking_expression_test.cpp +++ b/searchlib/src/tests/features/ranking_expression/ranking_expression_test.cpp @@ -68,13 +68,13 @@ struct SetupResult { DummyDependencyHandler deps; bool setup_ok; SetupResult(const TypeMap &object_inputs, const vespalib::string &expression, - bool external_expression = false); + const vespalib::string &expression_name = ""); ~SetupResult(); }; SetupResult::SetupResult(const TypeMap &object_inputs, const vespalib::string &expression, - bool external_expression) + const vespalib::string &expression_name) : stash(), index_env(), query_env(&index_env), rank(make_replacer()), deps(rank), setup_ok(false) { rank.setName("self"); @@ -82,11 +82,11 @@ SetupResult::SetupResult(const TypeMap &object_inputs, deps.define_object_input(input.first, ValueType::from_spec(input.second)); } std::vector<vespalib::string> params; - if (external_expression) { - params.push_back("my_expr"); - index_env.addRankingExpression("my_expr", expression); - } else { + if (expression_name.empty()) { index_env.getProperties().add("self.rankingScript", expression); + } else { + index_env.addRankingExpression(expression_name, expression); + index_env.getProperties().add("self.expressionName", expression_name); } Blueprint &bp = rank; setup_ok = bp.setup(index_env, params); @@ -96,9 +96,9 @@ SetupResult::~SetupResult() = default; void verify_output_type(const TypeMap &object_inputs, const vespalib::string &expression, const FeatureType &expect, - bool external_expression = false) + const vespalib::string &expression_name = "") { - SetupResult result(object_inputs, expression, external_expression); + SetupResult result(object_inputs, expression, expression_name); EXPECT_TRUE(result.setup_ok); EXPECT_EQUAL(1u, result.deps.output.size()); ASSERT_EQUAL(1u, result.deps.output_type.size()); @@ -137,10 +137,10 @@ TEST("require that ranking expression can resolve to concrete complex type") { } TEST("require that ranking expression can be external") { - TEST_DO(verify_output_type({}, "a*b", FeatureType::number(), true)); - TEST_DO(verify_output_type({{"b", "double"}}, "a*b", FeatureType::object(ValueType::double_type()), true)); + TEST_DO(verify_output_type({}, "a*b", FeatureType::number(), "my_expr")); + TEST_DO(verify_output_type({{"b", "double"}}, "a*b", FeatureType::object(ValueType::double_type()), "my_expr")); TEST_DO(verify_output_type({{"a", "tensor(x{},y{})"}, {"b", "tensor(y{},z{})"}}, "a*b", - FeatureType::object(ValueType::from_spec("tensor(x{},y{},z{})")), true)); + FeatureType::object(ValueType::from_spec("tensor(x{},y{},z{})")), "my_expr")); } TEST("require that setup fails for incompatible types") { diff --git a/searchlib/src/vespa/searchlib/attribute/attributevector.hpp b/searchlib/src/vespa/searchlib/attribute/attributevector.hpp index efc96bc57c2..616096e9091 100644 --- a/searchlib/src/vespa/searchlib/attribute/attributevector.hpp +++ b/searchlib/src/vespa/searchlib/attribute/attributevector.hpp @@ -89,40 +89,36 @@ AttributeVector::adjustWeight(ChangeVectorT< ChangeTemplate<T> >& changes, DocId template<typename T> bool -AttributeVector::applyArithmetic(ChangeVectorT< ChangeTemplate<T> > & changes, DocId doc, const T & v, +AttributeVector::applyArithmetic(ChangeVectorT< ChangeTemplate<T> > & changes, DocId doc, const T &, const ArithmeticValueUpdate & arithm) { - (void) v; - bool retval(!hasMultiValue() && (doc < getNumDocs())); - if (retval) { - size_t oldSz(changes.size()); - ArithmeticValueUpdate::Operator op(arithm.getOperator()); - double aop = arithm.getOperand(); - if (op == ArithmeticValueUpdate::Add) { - changes.push_back(ChangeTemplate<T>(ChangeBase::ADD, doc, 0, 0)); - } else if (op == ArithmeticValueUpdate::Sub) { - changes.push_back(ChangeTemplate<T>(ChangeBase::SUB, doc, 0, 0)); - } else if (op == ArithmeticValueUpdate::Mul) { - changes.push_back(ChangeTemplate<T>(ChangeBase::MUL, doc, 0, 0)); - } else if (op == ArithmeticValueUpdate::Div) { - if (this->getClass().inherits(IntegerAttribute::classId) && aop == 0) { - divideByZeroWarning(); - } else { - changes.push_back(ChangeTemplate<T>(ChangeBase::DIV, doc, 0, 0)); - } + if (hasMultiValue() || (doc >= getNumDocs())) return false; + + size_t oldSz(changes.size()); + ArithmeticValueUpdate::Operator op(arithm.getOperator()); + double aop = arithm.getOperand(); + if (op == ArithmeticValueUpdate::Add) { + changes.push_back(ChangeTemplate<T>(ChangeBase::ADD, doc, 0, 0)); + } else if (op == ArithmeticValueUpdate::Sub) { + changes.push_back(ChangeTemplate<T>(ChangeBase::SUB, doc, 0, 0)); + } else if (op == ArithmeticValueUpdate::Mul) { + changes.push_back(ChangeTemplate<T>(ChangeBase::MUL, doc, 0, 0)); + } else if (op == ArithmeticValueUpdate::Div) { + if (this->getClass().inherits(IntegerAttribute::classId) && aop == 0) { + divideByZeroWarning(); } else { - retval = false; - } - if (retval) { - const size_t diff = changes.size() - oldSz; - _status.incNonIdempotentUpdates(diff); - _status.incUpdates(diff); - if (diff > 0) { - changes.back()._arithOperand = aop; - } + changes.push_back(ChangeTemplate<T>(ChangeBase::DIV, doc, 0, 0)); } + } else { + return false; } - return retval; + const size_t diff = changes.size() - oldSz; + _status.incNonIdempotentUpdates(diff); + _status.incUpdates(diff); + if (diff > 0) { + changes.back()._arithOperand = aop; + } + return true; } template<typename T> diff --git a/searchlib/src/vespa/searchlib/attribute/changevector.h b/searchlib/src/vespa/searchlib/attribute/changevector.h index c3b4d0ce3b0..d63ef2e2b35 100644 --- a/searchlib/src/vespa/searchlib/attribute/changevector.h +++ b/searchlib/src/vespa/searchlib/attribute/changevector.h @@ -2,8 +2,8 @@ #pragma once -#include <vespa/vespalib/stllike/hash_map.h> #include <vespa/searchcommon/common/undefinedvalues.h> +#include <vespa/vespalib/stllike/allocator.h> #include <vector> namespace vespalib { class MemoryUsage; } @@ -26,11 +26,10 @@ struct ChangeBase { DIV, CLEARDOC }; - enum {TAIL=0, UNSET_ENUM = 0xffffffffu}; + enum {UNSET_ENUM = 0xffffffffu}; ChangeBase() : _type(NOOP), - _next(TAIL), _doc(0), _weight(1), _enumScratchPad(UNSET_ENUM), @@ -39,7 +38,6 @@ struct ChangeBase { ChangeBase(Type type, uint32_t d, int32_t w = 1) : _type(type), - _next(TAIL), _doc(d), _weight(w), _enumScratchPad(UNSET_ENUM), @@ -48,18 +46,11 @@ struct ChangeBase { int cmp(const ChangeBase &b) const { int diff(_doc - b._doc); return diff; } bool operator <(const ChangeBase & b) const { return cmp(b) < 0; } - bool isAtEnd() const { return _next == TAIL; } - uint32_t getNext() const { return _next; } - void setNext(uint32_t next) { _next = next; } uint32_t getEnum() const { return _enumScratchPad; } void setEnum(uint32_t value) const { _enumScratchPad = value; } bool isEnumValid() const { return _enumScratchPad != UNSET_ENUM; } - void invalidateEnum() const { _enumScratchPad = UNSET_ENUM; } Type _type; -private: - uint32_t _next; -public: uint32_t _doc; int32_t _weight; mutable uint32_t _enumScratchPad; @@ -108,7 +99,7 @@ struct ChangeTemplate : public ChangeBase { ChangeBase(type, d, w), _data(v) { } - T _data; + T _data; }; template <> @@ -131,54 +122,66 @@ NumericChangeData<double>::operator<(const NumericChangeData<double> &rhs) const return _v < rhs._v; } -class ChangeVectorBase { -protected: -}; - /** - * Maintains a list of changes where changes to the same docid are adjacent, but ordered by insertion order. - * Apart from that no ordering by docid. + * Maintains a list of changes. + * You can select to view the in insert order, + * or unordered, but changes to the same docid are adjacent and ordered by insertion order. */ template <typename T> -class ChangeVectorT : public ChangeVectorBase { +class ChangeVectorT { private: - using Map = vespalib::hash_map<uint32_t, uint32_t>; - using Vector = std::vector<T>; + using Vector = std::vector<T, vespalib::allocator_large<T>>; public: + using const_iterator = typename Vector::const_iterator; ChangeVectorT(); ~ChangeVectorT(); - class const_iterator { - public: - const_iterator(const Vector & vector, uint32_t next) : _v(&vector), _next(next) { } - bool operator == (const const_iterator & rhs) const { return _v == rhs._v && _next == rhs._next; } - bool operator != (const const_iterator & rhs) const { return _v != rhs._v || _next != rhs._next; } - const_iterator& operator++() { advance(); return *this; } - const_iterator operator++(int) { const_iterator other(*this); advance(); return other; } - const T & operator * () const { return v()[_next]; } - const T * operator -> () const { return &v()[_next]; } - private: - void advance() { _next = v()[_next].getNext(); } - const Vector & v() const { return *_v; } - const Vector * _v; - uint32_t _next; - }; - void push_back(const T & c); template <typename Accessor> void push_back(uint32_t doc, Accessor & ac); - const T & back() const { return _v.back(); } T & back() { return _v.back(); } size_t size() const { return _v.size(); } + size_t capacity() const { return _v.capacity(); } bool empty() const { return _v.empty(); } void clear(); - const_iterator begin() const { return const_iterator(_v, 0); } - const_iterator end() const { return const_iterator(_v, size()); } + class InsertOrder { + public: + InsertOrder(const Vector & v) : _v(v) { } + const_iterator begin() const { return _v.begin(); } + const_iterator end() const { return _v.end(); } + private: + const Vector &_v; + }; + class DocIdInsertOrder { + using AdjacentDocIds = std::vector<uint64_t, vespalib::allocator_large<uint64_t>>; + public: + class const_iterator { + public: + const_iterator(const Vector & vector, const AdjacentDocIds & order, uint32_t cur) + : _v(&vector), _o(&order), _cur(cur) { } + bool operator == (const const_iterator & rhs) const { return _v == rhs._v && _cur == rhs._cur; } + bool operator != (const const_iterator & rhs) const { return _v != rhs._v || _cur != rhs._cur; } + const_iterator& operator++() { _cur++; return *this; } + const_iterator operator++(int) { const_iterator other(*this); _cur++; return other; } + const T & operator * () const { return v(); } + const T * operator -> () const { return &v(); } + private: + const T & v() const { return (*_v)[(*_o)[_cur] & 0xffffffff]; } + const Vector * _v; + const AdjacentDocIds * _o; + uint32_t _cur; + }; + DocIdInsertOrder(const Vector & v); + const_iterator begin() const { return const_iterator(_v, _adjacent, 0); } + const_iterator end() const { return const_iterator(_v, _adjacent, _v.size()); } + private: + const Vector &_v; + AdjacentDocIds _adjacent; + }; + InsertOrder getInsertOrder() const { return InsertOrder(_v); } + DocIdInsertOrder getDocIdInsertOrder() const { return DocIdInsertOrder(_v); } vespalib::MemoryUsage getMemoryUsage() const; private: - void linkIn(uint32_t doc, size_t index, size_t last); - Vector _v; - Map _docs; - uint32_t _tail; + Vector _v; }; } // namespace search diff --git a/searchlib/src/vespa/searchlib/attribute/changevector.hpp b/searchlib/src/vespa/searchlib/attribute/changevector.hpp index dcb31ebae73..5052f4b9a10 100644 --- a/searchlib/src/vespa/searchlib/attribute/changevector.hpp +++ b/searchlib/src/vespa/searchlib/attribute/changevector.hpp @@ -4,9 +4,12 @@ #include "changevector.h" #include <vespa/vespalib/util/memoryusage.h> +#include <vespa/vespalib/util/alloc.h> namespace search { +using vespalib::roundUp2inN; + namespace { // This number is selected to be large enough to hold bursts between commits @@ -16,11 +19,9 @@ constexpr size_t NUM_ELEMS_TO_RESERVE = 200; template <typename T> ChangeVectorT<T>::ChangeVectorT() - : _v(), - _docs(NUM_ELEMS_TO_RESERVE*2), - _tail(0) + : _v() { - _v.reserve(vespalib::roundUp2inN(NUM_ELEMS_TO_RESERVE, sizeof(T))); + _v.reserve(roundUp2inN<T>(NUM_ELEMS_TO_RESERVE)); } template <typename T> @@ -29,17 +30,21 @@ ChangeVectorT<T>::~ChangeVectorT() = default; template <typename T> void ChangeVectorT<T>::clear() { - _v.clear(); - _docs.clear(); + if (_v.capacity() > roundUp2inN<T>(NUM_ELEMS_TO_RESERVE * 5)) { + // Ensure we do not keep insanely large buffers over time, due to abnormal peaks + // caused by hickups else where. + _v = Vector(); + _v.reserve(roundUp2inN<T>(NUM_ELEMS_TO_RESERVE)); + } else { + _v.clear(); + } } template <typename T> void ChangeVectorT<T>::push_back(const T & c) { - size_t index(size()); _v.push_back(c); - linkIn(c._doc, index, index); } template <typename T> @@ -49,48 +54,32 @@ ChangeVectorT<T>::push_back(uint32_t doc, Accessor & ac) { if (ac.size() <= 0) { return; } - size_t index(size()); - _v.reserve(vespalib::roundUp2inN(index + ac.size(), sizeof(T))); + _v.reserve(roundUp2inN<T>(size() + ac.size())); for (size_t i(0), m(ac.size()); i < m; i++, ac.next()) { _v.push_back(T(ChangeBase::APPEND, doc, typename T::DataType(ac.value()), ac.weight())); - _v.back().setNext(index + i + 1); } - linkIn(doc, index, size() - 1); } template <typename T> -void -ChangeVectorT<T>::linkIn(uint32_t doc, size_t first, size_t last) +vespalib::MemoryUsage +ChangeVectorT<T>::getMemoryUsage() const { - if (first != 0 && (_v[_tail]._doc == doc)) { - _v[_tail].setNext(first); - _tail = last; - } else { - Map::iterator found(_docs.find(doc)); - if (found == _docs.end()) { - _docs[doc] = last; - if (_tail != first) { - _v[_tail].setNext(first); - } - _tail = last; - } else { - uint32_t prev(found->second); - for (; _v[_v[prev].getNext()]._doc == doc; prev = _v[prev].getNext()); - _v[last].setNext(_v[prev].getNext()); - _v[prev].setNext(first); - found->second = last; - } - } - _v[_tail].setNext(size()); + size_t usedBytes = _v.size() * sizeof(T); + size_t allocBytes = _v.capacity() * sizeof(T); + return vespalib::MemoryUsage(allocBytes, usedBytes, 0, 0); } template <typename T> -vespalib::MemoryUsage -ChangeVectorT<T>::getMemoryUsage() const +ChangeVectorT<T>::DocIdInsertOrder::DocIdInsertOrder(const Vector & v) + : _v(v), + _adjacent() { - size_t usedBytes = _v.size() * sizeof(T) + _docs.getMemoryUsed(); - size_t allocBytes = _v.capacity() * sizeof(T) + _docs.getMemoryConsumption(); - return vespalib::MemoryUsage(allocBytes, usedBytes, 0, 0); + _adjacent.reserve(v.size()); + uint32_t index(0); + for (const auto & c : _v) { + _adjacent.push_back((uint64_t(c._doc) << 32) | index++); + } + std::sort(_adjacent.begin(), _adjacent.end()); } } // namespace search diff --git a/searchlib/src/vespa/searchlib/attribute/enumattribute.hpp b/searchlib/src/vespa/searchlib/attribute/enumattribute.hpp index fd576b3a9ba..164bb411061 100644 --- a/searchlib/src/vespa/searchlib/attribute/enumattribute.hpp +++ b/searchlib/src/vespa/searchlib/attribute/enumattribute.hpp @@ -63,7 +63,7 @@ void EnumAttribute<B>::insertNewUniqueValues(EnumStoreBatchUpdater& updater) { // find and insert new unique strings - for (const auto & data : this->_changes) { + for (const auto & data : this->_changes.getInsertOrder()) { considerAttributeChange(data, updater); } } diff --git a/searchlib/src/vespa/searchlib/attribute/enumstore.hpp b/searchlib/src/vespa/searchlib/attribute/enumstore.hpp index 90bcf92a103..9885613f4e3 100644 --- a/searchlib/src/vespa/searchlib/attribute/enumstore.hpp +++ b/searchlib/src/vespa/searchlib/attribute/enumstore.hpp @@ -207,7 +207,7 @@ template <typename EntryT> vespalib::MemoryUsage EnumStoreT<EntryT>::update_stat() { - auto &store = _store.get_allocator().get_data_store(); + auto &store = _store.get_data_store(); _cached_values_memory_usage = store.getMemoryUsage(); _cached_values_address_space_usage = store.getAddressSpaceUsage(); _cached_dictionary_btree_usage = _dict->get_btree_memory_usage(); diff --git a/searchlib/src/vespa/searchlib/attribute/multienumattribute.hpp b/searchlib/src/vespa/searchlib/attribute/multienumattribute.hpp index 8475451ba60..072398abcf9 100644 --- a/searchlib/src/vespa/searchlib/attribute/multienumattribute.hpp +++ b/searchlib/src/vespa/searchlib/attribute/multienumattribute.hpp @@ -25,7 +25,7 @@ template <typename B, typename M> bool MultiValueEnumAttribute<B, M>::extractChangeData(const Change & c, EnumIndex & idx) { - if (c._enumScratchPad == Change::UNSET_ENUM) { + if ( ! c.isEnumValid() ) { return this->_enumStore.find_index(c._data.raw(), idx); } idx = EnumIndex(vespalib::datastore::EntryRef(c._enumScratchPad)); diff --git a/searchlib/src/vespa/searchlib/attribute/multivalueattribute.h b/searchlib/src/vespa/searchlib/attribute/multivalueattribute.h index 66ca6bd2eac..d36777a25a9 100644 --- a/searchlib/src/vespa/searchlib/attribute/multivalueattribute.h +++ b/searchlib/src/vespa/searchlib/attribute/multivalueattribute.h @@ -20,7 +20,6 @@ protected: typedef typename B::DocId DocId; typedef typename B::Change Change; typedef typename B::ChangeVector ChangeVector; - typedef typename B::ChangeVector::const_iterator ChangeVectorIterator; using MultiValueType = M; using MultiValueMapping = attribute::MultiValueMapping<MultiValueType>; diff --git a/searchlib/src/vespa/searchlib/attribute/multivalueattribute.hpp b/searchlib/src/vespa/searchlib/attribute/multivalueattribute.hpp index 0cd2e0bbc27..2e73909ea1e 100644 --- a/searchlib/src/vespa/searchlib/attribute/multivalueattribute.hpp +++ b/searchlib/src/vespa/searchlib/attribute/multivalueattribute.hpp @@ -78,11 +78,12 @@ void MultiValueAttribute<B, M>::apply_attribute_changes_to_array(DocumentValues& docValues) { // compute new values for each document with changes - for (ChangeVectorIterator current(this->_changes.begin()), end(this->_changes.end()); (current != end); ) { + auto iterable = this->_changes.getDocIdInsertOrder(); + for (auto current(iterable.begin()), end(iterable.end()); (current != end); ) { DocId doc = current->_doc; // find last clear doc - ChangeVectorIterator last_clear_doc = end; - for (ChangeVectorIterator iter = current; (iter != end) && (iter->_doc == doc); ++iter) { + auto last_clear_doc = end; + for (auto iter = current; (iter != end) && (iter->_doc == doc); ++iter) { if (iter->_type == ChangeBase::CLEARDOC) { last_clear_doc = iter; } @@ -137,12 +138,13 @@ void MultiValueAttribute<B, M>::apply_attribute_changes_to_wset(DocumentValues& docValues) { // compute new values for each document with changes - for (ChangeVectorIterator current(this->_changes.begin()), end(this->_changes.end()); (current != end); ) { + auto iterable = this->_changes.getDocIdInsertOrder(); + for (auto current(iterable.begin()), end(iterable.end()); (current != end); ) { const DocId doc = current->_doc; // find last clear doc - ChangeVectorIterator last_clear_doc = end; + auto last_clear_doc = end; size_t max_elems_inserted = 0; - for (ChangeVectorIterator iter = current; (iter != end) && (iter->_doc == doc); ++iter) { + for (auto iter = current; (iter != end) && (iter->_doc == doc); ++iter) { if (iter->_type == ChangeBase::CLEARDOC) { last_clear_doc = iter; } diff --git a/searchlib/src/vespa/searchlib/attribute/postingstore.cpp b/searchlib/src/vespa/searchlib/attribute/postingstore.cpp index 6c62e650345..477917debf0 100644 --- a/searchlib/src/vespa/searchlib/attribute/postingstore.cpp +++ b/searchlib/src/vespa/searchlib/attribute/postingstore.cpp @@ -696,7 +696,10 @@ PostingStore<DataT>::move(EntryRef ref) if (!_store.getCompacting(ref)) { return ref; } - return allocBitVectorCopy(*bve).ref; + auto new_ref = allocBitVectorCopy(*bve).ref; + _bvs.erase(ref.ref()); + _bvs.insert(new_ref.ref()); + return new_ref; } else { if (!_store.getCompacting(ref)) { return ref; diff --git a/searchlib/src/vespa/searchlib/attribute/singleboolattribute.cpp b/searchlib/src/vespa/searchlib/attribute/singleboolattribute.cpp index 2b40150f87b..6c8edea13cf 100644 --- a/searchlib/src/vespa/searchlib/attribute/singleboolattribute.cpp +++ b/searchlib/src/vespa/searchlib/attribute/singleboolattribute.cpp @@ -59,7 +59,7 @@ SingleBoolAttribute::onCommit() { if ( ! _changes.empty()) { // apply updates ValueModifier valueGuard(getValueModifier()); - for (const auto & change : _changes) { + for (const auto & change : _changes.getInsertOrder()) { if (change._type == ChangeBase::UPDATE) { std::atomic_thread_fence(std::memory_order_release); setBit(change._doc, change._data != 0); diff --git a/searchlib/src/vespa/searchlib/attribute/singleenumattribute.hpp b/searchlib/src/vespa/searchlib/attribute/singleenumattribute.hpp index b39bdeb3b00..bf75400b157 100644 --- a/searchlib/src/vespa/searchlib/attribute/singleenumattribute.hpp +++ b/searchlib/src/vespa/searchlib/attribute/singleenumattribute.hpp @@ -175,7 +175,7 @@ void SingleValueEnumAttribute<B>::applyValueChanges(EnumStoreBatchUpdater& updater) { ValueModifier valueGuard(this->getValueModifier()); - for (const auto& change : this->_changes) { + for (const auto& change : this->_changes.getInsertOrder()) { if (change._type == ChangeBase::UPDATE) { applyUpdateValueChange(change, updater); } else if (change._type >= ChangeBase::ADD && change._type <= ChangeBase::DIV) { diff --git a/searchlib/src/vespa/searchlib/attribute/singlenumericattribute.hpp b/searchlib/src/vespa/searchlib/attribute/singlenumericattribute.hpp index fd913f34c3a..671bdc44e22 100644 --- a/searchlib/src/vespa/searchlib/attribute/singlenumericattribute.hpp +++ b/searchlib/src/vespa/searchlib/attribute/singlenumericattribute.hpp @@ -37,7 +37,7 @@ SingleValueNumericAttribute<B>::onCommit() { // apply updates typename B::ValueModifier valueGuard(this->getValueModifier()); - for (const auto & change : this->_changes) { + for (const auto & change : this->_changes.getInsertOrder()) { if (change._type == ChangeBase::UPDATE) { std::atomic_thread_fence(std::memory_order_release); _data[change._doc] = change._data; diff --git a/searchlib/src/vespa/searchlib/attribute/singlenumericpostattribute.hpp b/searchlib/src/vespa/searchlib/attribute/singlenumericpostattribute.hpp index f5ab855565c..e1c2a817af7 100644 --- a/searchlib/src/vespa/searchlib/attribute/singlenumericpostattribute.hpp +++ b/searchlib/src/vespa/searchlib/attribute/singlenumericpostattribute.hpp @@ -84,7 +84,7 @@ SingleValueNumericPostingAttribute<B>::applyValueChanges(EnumStoreBatchUpdater& // used to make sure several arithmetic operations on the same document in a single commit works std::map<DocId, EnumIndex> currEnumIndices; - for (const auto& change : this->_changes) { + for (const auto& change : this->_changes.getInsertOrder()) { auto enumIter = currEnumIndices.find(change._doc); EnumIndex oldIdx; if (enumIter != currEnumIndices.end()) { diff --git a/searchlib/src/vespa/searchlib/attribute/singlesmallnumericattribute.cpp b/searchlib/src/vespa/searchlib/attribute/singlesmallnumericattribute.cpp index f1d0da42165..8d460b5c661 100644 --- a/searchlib/src/vespa/searchlib/attribute/singlesmallnumericattribute.cpp +++ b/searchlib/src/vespa/searchlib/attribute/singlesmallnumericattribute.cpp @@ -53,7 +53,7 @@ SingleValueSmallNumericAttribute::onCommit() { // apply updates B::ValueModifier valueGuard(getValueModifier()); - for (const auto & change : _changes) { + for (const auto & change : _changes.getInsertOrder()) { if (change._type == ChangeBase::UPDATE) { std::atomic_thread_fence(std::memory_order_release); set(change._doc, change._data); diff --git a/searchlib/src/vespa/searchlib/attribute/singlestringpostattribute.hpp b/searchlib/src/vespa/searchlib/attribute/singlestringpostattribute.hpp index 39ad8d71021..4432acf2c55 100644 --- a/searchlib/src/vespa/searchlib/attribute/singlestringpostattribute.hpp +++ b/searchlib/src/vespa/searchlib/attribute/singlestringpostattribute.hpp @@ -85,7 +85,7 @@ SingleValueStringPostingAttributeT<B>::applyValueChanges(EnumStoreBatchUpdater& // used to make sure several arithmetic operations on the same document in a single commit works std::map<DocId, EnumIndex> currEnumIndices; - for (const auto& change : this->_changes) { + for (const auto& change : this->_changes.getInsertOrder()) { auto enumIter = currEnumIndices.find(change._doc); EnumIndex oldIdx; if (enumIter != currEnumIndices.end()) { diff --git a/searchlib/src/vespa/searchlib/common/bitvector.cpp b/searchlib/src/vespa/searchlib/common/bitvector.cpp index 8953710e8c7..fa63b846e17 100644 --- a/searchlib/src/vespa/searchlib/common/bitvector.cpp +++ b/searchlib/src/vespa/searchlib/common/bitvector.cpp @@ -6,6 +6,7 @@ #include "partialbitvector.h" #include <vespa/vespalib/hwaccelrated/iaccelrated.h> #include <vespa/vespalib/util/exceptions.h> +#include <vespa/vespalib/util/size_literals.h> #include <vespa/vespalib/objects/nbostream.h> #include <vespa/fastos/file.h> #include <cassert> @@ -32,6 +33,9 @@ void verifyInclusiveStart(const search::BitVector & a, const search::BitVector & } } +constexpr size_t MMAP_LIMIT = 32_Mi; +constexpr size_t DIRECTIO_ALIGNMENT = 4_Ki; + } ///////////////////////////////// @@ -49,7 +53,7 @@ BitVector::allocatePaddedAndAligned(Index start, Index end, Index capacity) uint32_t words = numActiveWords(start, capacity); words += (-words & 15); // Pad to 64 byte alignment const size_t sz(words * sizeof(Word)); - Alloc alloc = Alloc::alloc(sz); + Alloc alloc = Alloc::alloc(sz, MMAP_LIMIT); assert(alloc.size()/sizeof(Word) >= words); // Clear padding size_t usedBytes = numBytes(end - start); @@ -337,7 +341,7 @@ BitVector::create(Index numberOfElements, FastOS_FileInterface &file, size_t vectorsize = getFileBytes(numberOfElements); file.DirectIOPadding(offset, vectorsize, padbefore, padafter); assert((padbefore & (getAlignment() - 1)) == 0); - AllocatedBitVector::Alloc alloc = Alloc::alloc(padbefore + vectorsize + padafter, 0x1000000, 0x1000); + AllocatedBitVector::Alloc alloc = Alloc::alloc(padbefore + vectorsize + padafter, MMAP_LIMIT, DIRECTIO_ALIGNMENT); void * alignedBuffer = alloc.get(); file.ReadBuf(alignedBuffer, alloc.size(), offset - padbefore); bv = std::make_unique<AllocatedBitVector>(numberOfElements, std::move(alloc), padbefore); diff --git a/searchlib/src/vespa/searchlib/common/partialbitvector.h b/searchlib/src/vespa/searchlib/common/partialbitvector.h index f1d0716ed60..0c44378e78a 100644 --- a/searchlib/src/vespa/searchlib/common/partialbitvector.h +++ b/searchlib/src/vespa/searchlib/common/partialbitvector.h @@ -25,7 +25,7 @@ public: PartialBitVector(Index start, Index end); PartialBitVector(const BitVector & org, Index start, Index end); - virtual ~PartialBitVector(); + ~PartialBitVector() override; private: vespalib::alloc::Alloc _alloc; diff --git a/searchlib/src/vespa/searchlib/features/rankingexpressionfeature.cpp b/searchlib/src/vespa/searchlib/features/rankingexpressionfeature.cpp index 070d70997e6..15bd810e597 100644 --- a/searchlib/src/vespa/searchlib/features/rankingexpressionfeature.cpp +++ b/searchlib/src/vespa/searchlib/features/rankingexpressionfeature.cpp @@ -244,17 +244,15 @@ RankingExpressionBlueprint::setup(const fef::IIndexEnvironment &env, // Retrieve and concatenate whatever config is available. vespalib::string script = ""; fef::Property property = env.getProperties().lookup(getName(), "rankingScript"); + fef::Property expr_name = env.getProperties().lookup(getName(), "expressionName"); if (property.size() > 0) { for (uint32_t i = 0; i < property.size(); ++i) { script.append(property.getAt(i)); } - //LOG(debug, "Script from config: '%s'\n", script.c_str()); + } else if (expr_name.size() == 1) { + script = env.getRankingExpression(expr_name.get()); } else if (params.size() == 1) { - script = env.getRankingExpression(params[0].getValue()); - if (script.empty()) { - script = params[0].getValue(); - } - //LOG(debug, "Script from param: '%s'\n", script.c_str()); + script = params[0].getValue(); } else { return fail("No expression given."); } diff --git a/searchlib/src/vespa/searchlib/tensor/dense_tensor_attribute.cpp b/searchlib/src/vespa/searchlib/tensor/dense_tensor_attribute.cpp index 084dcee141b..be15967ab10 100644 --- a/searchlib/src/vespa/searchlib/tensor/dense_tensor_attribute.cpp +++ b/searchlib/src/vespa/searchlib/tensor/dense_tensor_attribute.cpp @@ -92,6 +92,16 @@ DenseTensorAttribute::consider_remove_from_index(DocId docid) } vespalib::MemoryUsage +DenseTensorAttribute::update_stat() +{ + vespalib::MemoryUsage result = TensorAttribute::update_stat(); + if (_index) { + result.merge(_index->memory_usage()); + } + return result; +} + +vespalib::MemoryUsage DenseTensorAttribute::memory_usage() const { vespalib::MemoryUsage result = TensorAttribute::memory_usage(); diff --git a/searchlib/src/vespa/searchlib/tensor/dense_tensor_attribute.h b/searchlib/src/vespa/searchlib/tensor/dense_tensor_attribute.h index 097c3c93dad..0884b776ca5 100644 --- a/searchlib/src/vespa/searchlib/tensor/dense_tensor_attribute.h +++ b/searchlib/src/vespa/searchlib/tensor/dense_tensor_attribute.h @@ -23,6 +23,7 @@ private: void internal_set_tensor(DocId docid, const vespalib::eval::Value& tensor); void consider_remove_from_index(DocId docid); + vespalib::MemoryUsage update_stat() override; vespalib::MemoryUsage memory_usage() const override; public: diff --git a/searchlib/src/vespa/searchlib/tensor/hnsw_index.cpp b/searchlib/src/vespa/searchlib/tensor/hnsw_index.cpp index 31ff3845d74..465793739ff 100644 --- a/searchlib/src/vespa/searchlib/tensor/hnsw_index.cpp +++ b/searchlib/src/vespa/searchlib/tensor/hnsw_index.cpp @@ -25,7 +25,7 @@ namespace { // TODO: Move this to MemoryAllocator, with name PAGE_SIZE. constexpr size_t small_page_size = 4_Ki; -constexpr size_t min_num_arrays_for_new_buffer = 8_Ki; +constexpr size_t min_num_arrays_for_new_buffer = 64_Ki; constexpr float alloc_grow_factor = 0.2; // TODO: Adjust these numbers to what we accept as max in config. constexpr size_t max_level_array_size = 16; diff --git a/searchlib/src/vespa/searchlib/tensor/tensor_attribute.cpp b/searchlib/src/vespa/searchlib/tensor/tensor_attribute.cpp index e8a25e0e23e..5044a989203 100644 --- a/searchlib/src/vespa/searchlib/tensor/tensor_attribute.cpp +++ b/searchlib/src/vespa/searchlib/tensor/tensor_attribute.cpp @@ -52,7 +52,8 @@ TensorAttribute::TensorAttribute(vespalib::stringref name, const Config &cfg, Te getGenerationHolder()), _tensorStore(tensorStore), _emptyTensor(createEmptyTensor(cfg.tensorType())), - _compactGeneration(0) + _compactGeneration(0), + _cached_tensor_store_memory_usage() { } @@ -84,9 +85,8 @@ TensorAttribute::onCommit() incGeneration(); if (getFirstUsedGeneration() > _compactGeneration) { // No data held from previous compact operation - Status &status = getStatus(); - size_t used = status.getUsed(); - size_t dead = status.getDead(); + size_t used = _cached_tensor_store_memory_usage.usedBytes(); + size_t dead = _cached_tensor_store_memory_usage.deadBytes(); if (getConfig().getCompactionStrategy().should_compact_memory(used, dead)) { compactWorst(); } @@ -96,7 +96,7 @@ TensorAttribute::onCommit() void TensorAttribute::onUpdateStat() { - vespalib::MemoryUsage total = memory_usage(); + vespalib::MemoryUsage total = update_stat(); this->updateStatistics(_refVector.size(), _refVector.size(), total.allocatedBytes(), @@ -161,6 +161,16 @@ TensorAttribute::setTensorRef(DocId docId, EntryRef ref) } vespalib::MemoryUsage +TensorAttribute::update_stat() +{ + vespalib::MemoryUsage result = _refVector.getMemoryUsage(); + _cached_tensor_store_memory_usage = _tensorStore.getMemoryUsage(); + result.merge(_cached_tensor_store_memory_usage); + result.mergeGenerationHeldBytes(getGenerationHolder().getHeldBytes()); + return result; +} + +vespalib::MemoryUsage TensorAttribute::memory_usage() const { vespalib::MemoryUsage result = _refVector.getMemoryUsage(); diff --git a/searchlib/src/vespa/searchlib/tensor/tensor_attribute.h b/searchlib/src/vespa/searchlib/tensor/tensor_attribute.h index 601e19e54d1..ea0f7391120 100644 --- a/searchlib/src/vespa/searchlib/tensor/tensor_attribute.h +++ b/searchlib/src/vespa/searchlib/tensor/tensor_attribute.h @@ -26,11 +26,13 @@ protected: TensorStore &_tensorStore; // data store for serialized tensors std::unique_ptr<vespalib::eval::Value> _emptyTensor; uint64_t _compactGeneration; // Generation when last compact occurred + vespalib::MemoryUsage _cached_tensor_store_memory_usage; template <typename RefType> void doCompactWorst(); void checkTensorType(const vespalib::eval::Value &tensor); void setTensorRef(DocId docId, EntryRef ref); + virtual vespalib::MemoryUsage update_stat(); virtual vespalib::MemoryUsage memory_usage() const; void populate_state(vespalib::slime::Cursor& object) const; diff --git a/security-utils/src/main/java/com/yahoo/security/X509CertificateUtils.java b/security-utils/src/main/java/com/yahoo/security/X509CertificateUtils.java index cefa8ab2f51..215dc311af3 100644 --- a/security-utils/src/main/java/com/yahoo/security/X509CertificateUtils.java +++ b/security-utils/src/main/java/com/yahoo/security/X509CertificateUtils.java @@ -18,13 +18,18 @@ import java.io.IOException; import java.io.StringReader; import java.io.StringWriter; import java.io.UncheckedIOException; +import java.math.BigInteger; import java.security.GeneralSecurityException; +import java.security.KeyPair; import java.security.PrivateKey; import java.security.PublicKey; import java.security.Signature; import java.security.SignatureException; import java.security.cert.CertificateException; import java.security.cert.X509Certificate; +import java.time.Duration; +import java.time.Instant; +import java.time.temporal.ChronoUnit; import java.util.ArrayList; import java.util.Collections; import java.util.List; @@ -161,4 +166,16 @@ public class X509CertificateUtils { } } + public static X509CertificateWithKey createSelfSigned(String cn, Duration duration) { + KeyPair keyPair = KeyUtils.generateKeypair(KeyAlgorithm.EC, 256); + X500Principal subject = new X500Principal(cn); + Instant now = Instant.now(); + X509Certificate cert = + X509CertificateBuilder.fromKeypair(keyPair, subject, now, + now.plus(duration), SignatureAlgorithm.SHA256_WITH_ECDSA, + BigInteger.ONE) + .setBasicConstraints(true, true) + .build(); + return new X509CertificateWithKey(cert, keyPair.getPrivate()); + } } diff --git a/slobrok/src/tests/mirrorapi/match_test.cpp b/slobrok/src/tests/mirrorapi/match_test.cpp index d350718a671..0686390565a 100644 --- a/slobrok/src/tests/mirrorapi/match_test.cpp +++ b/slobrok/src/tests/mirrorapi/match_test.cpp @@ -4,7 +4,7 @@ class MatchTester : public slobrok::api::IMirrorAPI { - SpecList lookup(const std::string &) const override { + SpecList lookup(vespalib::stringref ) const override { return SpecList(); } uint32_t updates() const override { return 0; } diff --git a/slobrok/src/vespa/slobrok/backoff.h b/slobrok/src/vespa/slobrok/backoff.h index 53193a9de6e..5d410635088 100644 --- a/slobrok/src/vespa/slobrok/backoff.h +++ b/slobrok/src/vespa/slobrok/backoff.h @@ -1,10 +1,9 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #pragma once -#include <stdio.h> +#include <cstdio> -namespace slobrok { -namespace api { +namespace slobrok::api { class BackOff { @@ -20,6 +19,4 @@ public: bool shouldWarn(); }; -} // namespace api -} // namespace slobrok - +} diff --git a/slobrok/src/vespa/slobrok/imirrorapi.h b/slobrok/src/vespa/slobrok/imirrorapi.h index 5cce9f1f49e..508847cbfeb 100644 --- a/slobrok/src/vespa/slobrok/imirrorapi.h +++ b/slobrok/src/vespa/slobrok/imirrorapi.h @@ -1,11 +1,10 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #pragma once -#include <string> +#include <vespa/vespalib/stllike/string.h> #include <vector> -namespace slobrok { -namespace api { +namespace slobrok::api { /** * @brief Defines an interface for the name server lookup. @@ -48,7 +47,7 @@ public: * @return a list of all matching services, with corresponding connect specs * @param pattern The pattern used for matching **/ - virtual SpecList lookup(const std::string & pattern) const = 0; + virtual SpecList lookup(vespalib::stringref pattern) const = 0; /** * Obtain the number of updates seen by this mirror. The value may wrap, but will never become 0 again. This can be @@ -62,6 +61,4 @@ public: virtual bool ready() const = 0; }; -} // namespace api -} // namespace slobrok - +} diff --git a/slobrok/src/vespa/slobrok/sbmirror.cpp b/slobrok/src/vespa/slobrok/sbmirror.cpp index 13680d16e68..8102e1fecbf 100644 --- a/slobrok/src/vespa/slobrok/sbmirror.cpp +++ b/slobrok/src/vespa/slobrok/sbmirror.cpp @@ -4,6 +4,7 @@ #include <vespa/fnet/frt/supervisor.h> #include <vespa/fnet/frt/target.h> #include <vespa/vespalib/util/exceptions.h> +#include <vespa/vespalib/stllike/hash_map.hpp> #include <vespa/log/log.h> LOG_SETUP(".slobrok.mirror"); @@ -25,7 +26,6 @@ MirrorAPI::MirrorAPI(FRT_Supervisor &orb, const ConfiguratorFactory & config) _configurator(config.create(_slobrokSpecs)), _currSlobrok(""), _rpc_ms(100), - _idx(0), _backOff(), _target(0), _req(0) @@ -53,14 +53,22 @@ MirrorAPI::~MirrorAPI() MirrorAPI::SpecList -MirrorAPI::lookup(const std::string & pattern) const +MirrorAPI::lookup(vespalib::stringref pattern) const { SpecList ret; + ret.reserve(1); + bool exact = pattern.find('*') == std::string::npos; std::lock_guard guard(_lock); - SpecList::const_iterator end = _specs.end(); - for (SpecList::const_iterator it = _specs.begin(); it != end; ++it) { - if (match(it->first.c_str(), pattern.c_str())) { - ret.push_back(*it); + if (exact) { + auto found = _specs.find(pattern); + if (found != _specs.end()) { + ret.emplace_back(found->first, found->second); + } + } else { + for (const auto & spec : _specs) { + if (match(spec.first.c_str(), pattern.data())) { + ret.emplace_back(spec.first, spec.second); + } } } return ret; @@ -77,8 +85,6 @@ MirrorAPI::lookup(const std::string & pattern) const bool IMirrorAPI::match(const char *name, const char *pattern) { - LOG_ASSERT(name != NULL); - LOG_ASSERT(pattern != NULL); while (*pattern != '\0') { if (*name == *pattern) { ++name; @@ -102,11 +108,11 @@ IMirrorAPI::match(const char *name, const char *pattern) void -MirrorAPI::updateTo(SpecList& newSpecs, uint32_t newGen) +MirrorAPI::updateTo(SpecMap newSpecs, uint32_t newGen) { { std::lock_guard guard(_lock); - std::swap(newSpecs, _specs); + _specs = std::move(newSpecs); _updates.add(); } _specsGen.setFromInt(newGen); @@ -167,34 +173,26 @@ MirrorAPI::handleIncrementalFetch() LOG(spam, "incremental diff [%u;%u] full dump, but numRemove=%u, numNames=%u", diff_from, diff_to, numRemove, numNames); } - SpecList specs; + SpecMap specs; for (uint32_t idx = 0; idx < numNames; idx++) { - specs.push_back( - std::make_pair(std::string(n[idx]._str), - std::string(s[idx]._str))); + specs[n[idx]._str] = s[idx]._str; } - updateTo(specs, diff_to); + updateTo(std::move(specs), diff_to); } else if (_specsGen == diff_from) { // incremental update - SpecList specs; - SpecList::const_iterator end = _specs.end(); - for (SpecList::const_iterator it = _specs.begin(); - it != end; - ++it) - { + SpecMap specs; + for (const auto & spec : _specs) { bool keep = true; for (uint32_t idx = 0; idx < numRemove; idx++) { - if (it->first == r[idx]._str) keep = false; + if (spec.first == r[idx]._str) keep = false; } for (uint32_t idx = 0; idx < numNames; idx++) { - if (it->first == n[idx]._str) keep = false; + if (spec.first == n[idx]._str) keep = false; } - if (keep) specs.push_back(*it); + if (keep) specs[spec.first] = spec.second; } for (uint32_t idx = 0; idx < numNames; idx++) { - specs.push_back( - std::make_pair(std::string(n[idx]._str), - std::string(s[idx]._str))); + specs[n[idx]._str] = s[idx]._str; } updateTo(specs, diff_to); } @@ -222,13 +220,7 @@ MirrorAPI::handleReqDone() if (_reqDone) { _reqDone = false; _reqPending = false; - bool reconn = (_target == 0); - - if (_req->IsError()) { - reconn = true; - } else { - reconn = handleIncrementalFetch(); - } + bool reconn = _req->IsError() ? true : handleIncrementalFetch(); if (reconn) { if (_target != 0) { diff --git a/slobrok/src/vespa/slobrok/sbmirror.h b/slobrok/src/vespa/slobrok/sbmirror.h index c1c7009ce12..ec1ce22194b 100644 --- a/slobrok/src/vespa/slobrok/sbmirror.h +++ b/slobrok/src/vespa/slobrok/sbmirror.h @@ -5,6 +5,7 @@ #include "backoff.h" #include "sblist.h" #include <vespa/vespalib/util/gencnt.h> +#include <vespa/vespalib/stllike/hash_map.h> #include <vespa/fnet/frt/invoker.h> class FRT_Target; @@ -40,6 +41,8 @@ public: * @param config how to get the connect spec list **/ MirrorAPI(FRT_Supervisor &orb, const ConfiguratorFactory & config); + MirrorAPI(const MirrorAPI &) = delete; + MirrorAPI &operator=(const MirrorAPI &) = delete; /** * @brief Clean up. @@ -47,7 +50,7 @@ public: ~MirrorAPI(); // Inherit doc from IMirrorAPI. - SpecList lookup(const std::string & pattern) const override; + SpecList lookup(vespalib::stringref pattern) const override; // Inherit doc from IMirrorAPI. uint32_t updates() const override { return _updates.getAsInt(); } @@ -67,16 +70,14 @@ public: bool ready() const override; private: - MirrorAPI(const MirrorAPI &); - MirrorAPI &operator=(const MirrorAPI &); - + using SpecMap = vespalib::hash_map<vespalib::string, vespalib::string>; /** from FNET_Task, polls slobrok **/ void PerformTask() override; /** from FRT_IRequestWait **/ void RequestDone(FRT_RPCRequest *req) override; - void updateTo(SpecList& newSpecs, uint32_t newGen); + void updateTo(SpecMap newSpecs, uint32_t newGen); bool handleIncrementalFetch(); @@ -93,14 +94,13 @@ private: bool _scheduled; bool _reqDone; bool _logOnSuccess; - SpecList _specs; + SpecMap _specs; vespalib::GenCnt _specsGen; vespalib::GenCnt _updates; SlobrokList _slobrokSpecs; Configurator::UP _configurator; std::string _currSlobrok; int _rpc_ms; - uint32_t _idx; BackOff _backOff; FRT_Target *_target; FRT_RPCRequest *_req; diff --git a/slobrok/src/vespa/slobrok/sbregister.cpp b/slobrok/src/vespa/slobrok/sbregister.cpp index 06a9fb2d79e..357e3fbd1ce 100644 --- a/slobrok/src/vespa/slobrok/sbregister.cpp +++ b/slobrok/src/vespa/slobrok/sbregister.cpp @@ -6,7 +6,6 @@ #include <vespa/vespalib/util/host_name.h> #include <vespa/vespalib/stllike/asciistream.h> #include <vespa/vespalib/util/exceptions.h> -#include <sstream> #include <vespa/log/log.h> LOG_SETUP(".slobrok.register"); diff --git a/slobrok/src/vespa/slobrok/server/sbenv.cpp b/slobrok/src/vespa/slobrok/server/sbenv.cpp index 579703a6f3f..2405c93b059 100644 --- a/slobrok/src/vespa/slobrok/server/sbenv.cpp +++ b/slobrok/src/vespa/slobrok/server/sbenv.cpp @@ -85,7 +85,7 @@ ConfigTask::PerformTask() } // namespace slobrok::<unnamed> SBEnv::SBEnv(const ConfigShim &shim) - : _transport(std::make_unique<FNET_Transport>()), + : _transport(std::make_unique<FNET_Transport>(TransportConfig().drop_empty_buffers(true))), _supervisor(std::make_unique<FRT_Supervisor>(_transport.get())), _configShim(shim), _configurator(shim.factory().create(*this)), diff --git a/staging_vespalib/src/vespa/vespalib/util/programoptions.cpp b/staging_vespalib/src/vespa/vespalib/util/programoptions.cpp index e38c54aaba0..8da7780f203 100644 --- a/staging_vespalib/src/vespa/vespalib/util/programoptions.cpp +++ b/staging_vespalib/src/vespa/vespalib/util/programoptions.cpp @@ -3,6 +3,7 @@ #include <vespa/vespalib/util/stringfmt.h> #include <vespa/vespalib/util/exceptions.h> #include <boost/lexical_cast.hpp> +#include <cassert> #include <vespa/log/log.h> LOG_SETUP(".programoptions"); diff --git a/storage/src/tests/common/CMakeLists.txt b/storage/src/tests/common/CMakeLists.txt index 400255964d6..4e7cecb9d9f 100644 --- a/storage/src/tests/common/CMakeLists.txt +++ b/storage/src/tests/common/CMakeLists.txt @@ -12,6 +12,7 @@ vespa_add_library(storage_testcommon TEST vespa_add_executable(storage_common_gtest_runner_app TEST SOURCES + bucket_stripe_utils_test.cpp bucket_utils_test.cpp global_bucket_space_distribution_converter_test.cpp gtest_runner.cpp diff --git a/storage/src/tests/common/bucket_stripe_utils_test.cpp b/storage/src/tests/common/bucket_stripe_utils_test.cpp new file mode 100644 index 00000000000..a654c4fe83e --- /dev/null +++ b/storage/src/tests/common/bucket_stripe_utils_test.cpp @@ -0,0 +1,49 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include <vespa/document/bucket/bucketid.h> +#include <vespa/storage/common/bucket_stripe_utils.h> +#include <vespa/vespalib/gtest/gtest.h> + +using document::BucketId; +using storage::calc_num_stripe_bits; +using storage::stripe_of_bucket_key; +using storage::adjusted_num_stripes; +constexpr uint8_t MUB = storage::spi::BucketLimits::MinUsedBits; + +TEST(BucketStripeUtilsTest, stripe_of_bucket_key) +{ + BucketId id(MUB, std::numeric_limits<uint64_t>::max()); + uint64_t key = id.stripUnused().toKey(); + EXPECT_EQ(0, stripe_of_bucket_key(key, 0)); + EXPECT_EQ(1, stripe_of_bucket_key(key, 1)); + EXPECT_EQ(3, stripe_of_bucket_key(key, 2)); + EXPECT_EQ(127, stripe_of_bucket_key(key, 7)); + EXPECT_EQ(255, stripe_of_bucket_key(key, 8)); +} + +TEST(BucketStripeUtilsTest, calc_num_stripe_bits) +{ + EXPECT_EQ(0, calc_num_stripe_bits(1)); + EXPECT_EQ(1, calc_num_stripe_bits(2)); + EXPECT_EQ(2, calc_num_stripe_bits(4)); + EXPECT_EQ(7, calc_num_stripe_bits(128)); + EXPECT_EQ(8, calc_num_stripe_bits(256)); +} + +TEST(BucketStripeUtilsTest, adjusted_num_stripes) +{ + EXPECT_EQ(0, adjusted_num_stripes(0)); + EXPECT_EQ(1, adjusted_num_stripes(1)); + EXPECT_EQ(2, adjusted_num_stripes(2)); + EXPECT_EQ(4, adjusted_num_stripes(3)); + EXPECT_EQ(256, adjusted_num_stripes(255)); + EXPECT_EQ(256, adjusted_num_stripes(256)); + EXPECT_EQ(256, adjusted_num_stripes(257)); +} + +TEST(BucketStripeUtilsTest, max_stripe_values) +{ + EXPECT_EQ(8, storage::MaxStripeBits); + EXPECT_EQ(256, storage::MaxStripes); +} + diff --git a/storage/src/tests/distributor/distributortest.cpp b/storage/src/tests/distributor/distributortest.cpp index 5a5ebb8c823..3d1c6165946 100644 --- a/storage/src/tests/distributor/distributortest.cpp +++ b/storage/src/tests/distributor/distributortest.cpp @@ -1255,7 +1255,7 @@ TEST_F(DistributorTest, wanted_split_bit_count_is_lower_bounded) { } TEST_F(DistributorTest, host_info_sent_immediately_once_all_stripes_first_reported) { - set_num_distributor_stripes(3); + set_num_distributor_stripes(4); createLinks(); getClock().setAbsoluteTimeInSeconds(1000); // TODO STRIPE can't call this currently since it touches the bucket DB updater directly: @@ -1265,6 +1265,7 @@ TEST_F(DistributorTest, host_info_sent_immediately_once_all_stripes_first_report EXPECT_EQ(0, explicit_node_state_reply_send_invocations()); // Nothing yet getDistributor().notify_stripe_wants_to_send_host_info(1); getDistributor().notify_stripe_wants_to_send_host_info(2); + getDistributor().notify_stripe_wants_to_send_host_info(3); tickDistributorNTimes(1); // Still nothing. Missing initial report from stripe 0 @@ -1283,11 +1284,11 @@ TEST_F(DistributorTest, host_info_sent_immediately_once_all_stripes_first_report // TODO STRIPE make delay configurable instead of hardcoded TEST_F(DistributorTest, non_bootstrap_host_info_send_request_delays_sending) { - set_num_distributor_stripes(3); + set_num_distributor_stripes(4); createLinks(); getClock().setAbsoluteTimeInSeconds(1000); - for (uint16_t i = 0; i < 3; ++i) { + for (uint16_t i = 0; i < 4; ++i) { getDistributor().notify_stripe_wants_to_send_host_info(i); } tickDistributorNTimes(1); diff --git a/storage/src/tests/distributor/mergeoperationtest.cpp b/storage/src/tests/distributor/mergeoperationtest.cpp index 7a90299bfb3..1026ea2855e 100644 --- a/storage/src/tests/distributor/mergeoperationtest.cpp +++ b/storage/src/tests/distributor/mergeoperationtest.cpp @@ -315,7 +315,7 @@ TEST_F(MergeOperationTest, allow_deleting_active_source_only_replica) { _sender.getLastCommand(true)); } -TEST_F(MergeOperationTest, MarkRedundantTrustedCopiesAsSourceOnly) { +TEST_F(MergeOperationTest, mark_redundant_trusted_copies_as_source_only) { // This test uses the same distribution as testGenerateNodeList(), i.e. // an ideal state sequence of [3, 5, 7, 6, 8, 0, 9, 2, 1, 4] @@ -415,6 +415,21 @@ TEST_F(MergeOperationTest, merge_operation_is_blocked_by_any_busy_target_node) { EXPECT_TRUE(op.isBlocked(*_pendingTracker, _operation_sequencer)); } + +TEST_F(MergeOperationTest, global_bucket_merges_are_not_blocked_by_busy_nodes) { + getClock().setAbsoluteTimeInSeconds(10); + document::BucketId bucket_id(16, 1); + addNodesToBucketDB(bucket_id, "0=10/1/1/t,1=20/1/1,2=10/1/1/t"); + enableDistributorClusterState("distributor:1 storage:3"); + document::Bucket global_bucket(document::FixedBucketSpaces::global_space(), bucket_id); + MergeOperation op(BucketAndNodes(global_bucket, toVector<uint16_t>(0, 1, 2))); + op.setIdealStateManager(&getIdealStateManager()); + + // Node 1 is included in operation node set but should not cause a block of global bucket merge + _pendingTracker->getNodeInfo().setBusy(0, std::chrono::seconds(10)); + EXPECT_FALSE(op.isBlocked(*_pendingTracker, _operation_sequencer)); +} + TEST_F(MergeOperationTest, merge_operation_is_blocked_by_locked_bucket) { getClock().setAbsoluteTimeInSeconds(10); addNodesToBucketDB(document::BucketId(16, 1), "0=10/1/1/t,1=20/1/1,2=10/1/1/t"); diff --git a/storage/src/tests/distributor/multi_thread_stripe_access_guard_test.cpp b/storage/src/tests/distributor/multi_thread_stripe_access_guard_test.cpp index 8513186d1e1..ba28396886f 100644 --- a/storage/src/tests/distributor/multi_thread_stripe_access_guard_test.cpp +++ b/storage/src/tests/distributor/multi_thread_stripe_access_guard_test.cpp @@ -1,22 +1,43 @@ // Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "mock_tickable_stripe.h" #include <vespa/document/bucket/fixed_bucket_spaces.h> +#include <vespa/persistence/spi/bucket_limits.h> #include <vespa/storage/distributor/distributor_stripe_pool.h> #include <vespa/storage/distributor/multi_threaded_stripe_access_guard.h> +#include <vespa/vdslib/distribution/distribution.h> #include <vespa/vdslib/state/clusterstate.h> #include <vespa/vespalib/gtest/gtest.h> using namespace ::testing; +using RawIdVector = std::vector<uint64_t>; + +constexpr uint8_t MUB = storage::spi::BucketLimits::MinUsedBits; namespace storage::distributor { struct AggregationTestingMockTickableStripe : MockTickableStripe { PotentialDataLossReport report; + std::vector<dbtransition::Entry> entries; PotentialDataLossReport remove_superfluous_buckets(document::BucketSpace, const lib::ClusterState&, bool) override { return report; } + void merge_entries_into_db(document::BucketSpace, api::Timestamp, const lib::Distribution&, + const lib::ClusterState&, const char*, const std::unordered_set<uint16_t>&, + const std::vector<dbtransition::Entry>& entries_in) override { + entries = entries_in; + } + + RawIdVector entries_as_raw_ids() const { + std::vector<uint64_t> result; + for (const auto& entry : entries) { + result.push_back(entry.bucket_id().withoutCountBits()); + } + std::sort(result.begin(), result.end()); + return result; + } + bool tick() override { return false; } @@ -25,6 +46,7 @@ struct AggregationTestingMockTickableStripe : MockTickableStripe { struct MultiThreadedStripeAccessGuardTest : Test { DistributorStripePool _pool; MultiThreadedStripeAccessor _accessor; + AggregationTestingMockTickableStripe _stripe0; AggregationTestingMockTickableStripe _stripe1; AggregationTestingMockTickableStripe _stripe2; AggregationTestingMockTickableStripe _stripe3; @@ -39,21 +61,77 @@ struct MultiThreadedStripeAccessGuardTest : Test { } void start_pool_with_stripes() { - _pool.start({{&_stripe1, &_stripe2, &_stripe3}}); + _pool.start({{&_stripe0, &_stripe1, &_stripe2, &_stripe3}}); } + + void start_pool_with_one_stripe() { + _pool.start({&_stripe0}); + } + + void merge_entries_into_db(const RawIdVector& raw_ids) { + std::vector<dbtransition::Entry> entries; + for (auto raw_id : raw_ids) { + entries.emplace_back(document::BucketId(MUB, raw_id), BucketCopy()); + } + std::sort(entries.begin(), entries.end()); + auto guard = _accessor.rendezvous_and_hold_all(); + guard->merge_entries_into_db(document::FixedBucketSpaces::default_space(), api::Timestamp(), + lib::Distribution(), lib::ClusterState(), "", {}, + entries); + } + }; TEST_F(MultiThreadedStripeAccessGuardTest, remove_superfluous_buckets_aggregates_reports_across_stripes) { - _stripe1.report = PotentialDataLossReport(20, 100); - _stripe2.report = PotentialDataLossReport(5, 200); - _stripe3.report = PotentialDataLossReport(7, 350); + _stripe0.report = PotentialDataLossReport(20, 100); + _stripe1.report = PotentialDataLossReport(5, 200); + _stripe2.report = PotentialDataLossReport(7, 350); + _stripe3.report = PotentialDataLossReport(3, 30); start_pool_with_stripes(); auto guard = _accessor.rendezvous_and_hold_all(); auto report = guard->remove_superfluous_buckets(document::FixedBucketSpaces::default_space(), lib::ClusterState(), false); - EXPECT_EQ(report.buckets, 32); - EXPECT_EQ(report.documents, 650); + EXPECT_EQ(report.buckets, 35); + EXPECT_EQ(report.documents, 680); +} + +TEST_F(MultiThreadedStripeAccessGuardTest, merge_entries_into_db_operates_across_all_stripes) { + start_pool_with_stripes(); + // Note: The bucket key is calculated by reversing the bits of the raw bucket id. + // We have 4 stripes and use 2 stripe bits. The 2 MSB of the bucket key is used to map to stripe. + // This gives the following mapping from raw bucket id to bucket key to stripe: + // raw id | key (8 MSB) | stripe + // 0x..0 | 00000000 | 0 + // 0x..1 | 10000000 | 2 + // 0x..2 | 01000000 | 1 + // 0x..3 | 11000000 | 3 + merge_entries_into_db({0x10,0x20,0x30,0x40,0x11,0x21,0x31,0x12,0x22,0x13}); + EXPECT_EQ(RawIdVector({0x10,0x20,0x30,0x40}), _stripe0.entries_as_raw_ids()); + EXPECT_EQ(RawIdVector({0x12,0x22}), _stripe1.entries_as_raw_ids()); + EXPECT_EQ(RawIdVector({0x11,0x21,0x31}), _stripe2.entries_as_raw_ids()); + EXPECT_EQ(RawIdVector({0x13}), _stripe3.entries_as_raw_ids()); +} + +TEST_F(MultiThreadedStripeAccessGuardTest, merge_entries_into_db_operates_across_subset_of_stripes) { + start_pool_with_stripes(); + merge_entries_into_db({0x12,0x22,0x13}); + EXPECT_EQ(RawIdVector(), _stripe0.entries_as_raw_ids()); + EXPECT_EQ(RawIdVector({0x12,0x22}), _stripe1.entries_as_raw_ids()); + EXPECT_EQ(RawIdVector(), _stripe2.entries_as_raw_ids()); + EXPECT_EQ(RawIdVector({0x13}), _stripe3.entries_as_raw_ids()); +} + +TEST_F(MultiThreadedStripeAccessGuardTest, merge_entries_into_db_operates_across_one_stripe) { + start_pool_with_one_stripe(); + merge_entries_into_db({0x10,0x11}); + EXPECT_EQ(RawIdVector({0x10,0x11}), _stripe0.entries_as_raw_ids()); +} + +TEST_F(MultiThreadedStripeAccessGuardTest, merge_entries_into_db_handles_empty_entries_vector) { + start_pool_with_one_stripe(); + merge_entries_into_db({}); + EXPECT_EQ(RawIdVector(), _stripe0.entries_as_raw_ids()); } } diff --git a/storage/src/tests/storageserver/mergethrottlertest.cpp b/storage/src/tests/storageserver/mergethrottlertest.cpp index 12ed9ead1b6..3a153fef9c3 100644 --- a/storage/src/tests/storageserver/mergethrottlertest.cpp +++ b/storage/src/tests/storageserver/mergethrottlertest.cpp @@ -1233,6 +1233,7 @@ TEST_F(MergeThrottlerTest, busy_returned_on_full_queue) { // Wait till we have maxPending replies and maxQueue queued _topLinks[0]->waitForMessages(maxPending, _messageWaitTime); + EXPECT_EQ(19, _throttlers[0]->getMetrics().queueSize.getMaximum()); waitUntilMergeQueueIs(*_throttlers[0], maxQueue, _messageWaitTime); // Clear all forwarded merges diff --git a/storage/src/tests/storageserver/rpc/caching_rpc_target_resolver_test.cpp b/storage/src/tests/storageserver/rpc/caching_rpc_target_resolver_test.cpp index 8f8eb84b0f2..1bbc1ec7aef 100644 --- a/storage/src/tests/storageserver/rpc/caching_rpc_target_resolver_test.cpp +++ b/storage/src/tests/storageserver/rpc/caching_rpc_target_resolver_test.cpp @@ -17,7 +17,7 @@ public: Mappings mappings; uint32_t gen; MockMirror() : mappings(), gen(1) {} - SpecList lookup(const std::string& pattern) const override { + SpecList lookup(vespalib::stringref pattern) const override { auto itr = mappings.find(pattern); if (itr != mappings.end()) { return itr->second; diff --git a/storage/src/vespa/storage/bucketdb/striped_btree_lockable_map.h b/storage/src/vespa/storage/bucketdb/striped_btree_lockable_map.h index b01f06c2b3f..73b5c37ab8b 100644 --- a/storage/src/vespa/storage/bucketdb/striped_btree_lockable_map.h +++ b/storage/src/vespa/storage/bucketdb/striped_btree_lockable_map.h @@ -27,7 +27,6 @@ public: using Decision = typename ParentType::Decision; using BucketId = document::BucketId; - constexpr static uint8_t MaxStripeBits = 8; private: using StripedDBType = BTreeLockableMap<T>; uint8_t _n_stripe_bits; diff --git a/storage/src/vespa/storage/bucketdb/striped_btree_lockable_map.hpp b/storage/src/vespa/storage/bucketdb/striped_btree_lockable_map.hpp index e4f6efa30a3..310a9d5154b 100644 --- a/storage/src/vespa/storage/bucketdb/striped_btree_lockable_map.hpp +++ b/storage/src/vespa/storage/bucketdb/striped_btree_lockable_map.hpp @@ -3,20 +3,13 @@ #include "striped_btree_lockable_map.h" #include "btree_lockable_map.hpp" +#include <vespa/storage/common/bucket_stripe_utils.h> #include <algorithm> #include <cassert> #include <queue> namespace storage::bucketdb { -namespace { - -constexpr uint8_t used_bits_of(uint64_t key) noexcept { - return static_cast<uint8_t>(key & 0b11'1111ULL); -} - -} - template <typename T> StripedBTreeLockableMap<T>::StripedBTreeLockableMap(uint8_t n_stripe_bits) : _n_stripe_bits(n_stripe_bits), @@ -37,9 +30,7 @@ StripedBTreeLockableMap<T>::~StripedBTreeLockableMap() = default; template <typename T> size_t StripedBTreeLockableMap<T>::stripe_of(key_type key) const noexcept { - assert(used_bits_of(key) >= _n_stripe_bits); - // Since bucket keys have count-bits at the LSB positions, we want to look at the MSBs instead. - return (key >> (64 - _n_stripe_bits)); + return stripe_of_bucket_key(key, _n_stripe_bits); } template <typename T> diff --git a/storage/src/vespa/storage/common/CMakeLists.txt b/storage/src/vespa/storage/common/CMakeLists.txt index 741d97f78ef..ae428929946 100644 --- a/storage/src/vespa/storage/common/CMakeLists.txt +++ b/storage/src/vespa/storage/common/CMakeLists.txt @@ -1,6 +1,7 @@ # Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. vespa_add_library(storage_common OBJECT SOURCES + bucket_stripe_utils.cpp bucketmessages.cpp content_bucket_space.cpp content_bucket_space_repo.cpp diff --git a/storage/src/vespa/storage/common/bucket_stripe_utils.cpp b/storage/src/vespa/storage/common/bucket_stripe_utils.cpp new file mode 100644 index 00000000000..10667e79678 --- /dev/null +++ b/storage/src/vespa/storage/common/bucket_stripe_utils.cpp @@ -0,0 +1,53 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include "bucket_stripe_utils.h" +#include <vespa/vespalib/util/alloc.h> +#include <cassert> + +namespace storage { + +namespace { + +constexpr uint8_t used_bits_of(uint64_t key) noexcept { + return static_cast<uint8_t>(key & 0b11'1111ULL); +} + +} + +size_t +stripe_of_bucket_key(uint64_t key, uint8_t n_stripe_bits) noexcept +{ + if (n_stripe_bits == 0) { + return 0; + } + assert(used_bits_of(key) >= n_stripe_bits); + // Since bucket keys have count-bits at the LSB positions, we want to look at the MSBs instead. + return (key >> (64 - n_stripe_bits)); +} + +uint8_t +calc_num_stripe_bits(uint32_t n_stripes) noexcept +{ + assert(n_stripes > 0); + if (n_stripes == 1) { + return 0; + } + assert(n_stripes == adjusted_num_stripes(n_stripes)); + + auto result = vespalib::Optimized::msbIdx(n_stripes); + assert(result <= MaxStripeBits); + return result; +} + +uint32_t adjusted_num_stripes(uint32_t n_stripes) noexcept +{ + if (n_stripes > 1) { + if (n_stripes > MaxStripes) { + return MaxStripes; + } + return vespalib::roundUp2inN(n_stripes); + } + return n_stripes; +} + +} diff --git a/storage/src/vespa/storage/common/bucket_stripe_utils.h b/storage/src/vespa/storage/common/bucket_stripe_utils.h new file mode 100644 index 00000000000..96f1247a09f --- /dev/null +++ b/storage/src/vespa/storage/common/bucket_stripe_utils.h @@ -0,0 +1,35 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#pragma once + +#include <vespa/persistence/spi/bucket_limits.h> +#include <cstdint> +#include <cstdlib> + +namespace storage { + +constexpr static uint8_t MaxStripeBits = spi::BucketLimits::MinUsedBits; +constexpr static uint32_t MaxStripes = (1ULL << MaxStripeBits); + +/** + * Returns the stripe in which the given bucket key belongs, + * when using the given number of stripe bits. + */ +size_t stripe_of_bucket_key(uint64_t key, uint8_t n_stripe_bits) noexcept; + +/** + * Returns the number of stripe bits used to represent the given number of stripes. + * + * This also asserts that the number of stripes is valid (power of 2 and within MaxStripes boundary). + */ +uint8_t calc_num_stripe_bits(uint32_t n_stripes) noexcept; + +/** + * Validates the number of stripes and returns the (potentially adjusted) value. + * + * This ensures the number of stripes is a power of 2 and within MaxStripes boundary. + */ +[[nodiscard]] uint32_t adjusted_num_stripes(uint32_t n_stripes) noexcept; + +} + diff --git a/storage/src/vespa/storage/distributor/distributor.cpp b/storage/src/vespa/storage/distributor/distributor.cpp index 47f7fee5873..65945b2c6ae 100644 --- a/storage/src/vespa/storage/distributor/distributor.cpp +++ b/storage/src/vespa/storage/distributor/distributor.cpp @@ -16,12 +16,15 @@ #include "ownership_transfer_safe_time_point_calculator.h" #include "throttlingoperationstarter.h" #include <vespa/document/bucket/fixed_bucket_spaces.h> +#include <vespa/storage/common/bucket_stripe_utils.h> #include <vespa/storage/common/global_bucket_space_distribution_converter.h> #include <vespa/storage/common/hostreporter/hostinfo.h> #include <vespa/storage/common/node_identity.h> #include <vespa/storage/common/nodestateupdater.h> #include <vespa/storage/config/distributorconfiguration.h> #include <vespa/storage/distributor/maintenance/simplebucketprioritydatabase.h> +#include <vespa/storageapi/message/persistence.h> +#include <vespa/storageapi/message/visitor.h> #include <vespa/storageframework/generic/status/xmlstatusreporter.h> #include <vespa/vdslib/distribution/distribution.h> #include <vespa/vespalib/util/memoryusage.h> @@ -59,6 +62,7 @@ Distributor::Distributor(DistributorComponentRegister& compReg, _metrics(std::make_shared<DistributorMetricSet>()), _messageSender(messageSender), _use_legacy_mode(num_distributor_stripes == 0), + _n_stripe_bits(0), _stripe(std::make_unique<DistributorStripe>(compReg, *_metrics, node_identity, threadPool, doneInitHandler, *this, *this, _use_legacy_mode)), _stripe_pool(stripe_pool), @@ -88,13 +92,20 @@ Distributor::Distributor(DistributorComponentRegister& compReg, _component.registerMetric(*_metrics); _component.registerMetricUpdateHook(_metricUpdateHook, framework::SecondTime(0)); if (!_use_legacy_mode) { - LOG(info, "Setting up distributor with %u stripes", num_distributor_stripes); // TODO STRIPE remove once legacy gone + assert(num_distributor_stripes == adjusted_num_stripes(num_distributor_stripes)); + _n_stripe_bits = calc_num_stripe_bits(num_distributor_stripes); + LOG(info, "Setting up distributor with %u stripes using %u stripe bits", + num_distributor_stripes, _n_stripe_bits); // TODO STRIPE remove once legacy gone _stripe_accessor = std::make_unique<MultiThreadedStripeAccessor>(_stripe_pool); _bucket_db_updater = std::make_unique<BucketDBUpdater>(_component, _component, *this, *this, _component.getDistribution(), *_stripe_accessor); _stripes.emplace_back(std::move(_stripe)); + for (size_t i = 1; i < num_distributor_stripes; ++i) { + _stripes.emplace_back(std::make_unique<DistributorStripe>(compReg, *_metrics, node_identity, threadPool, + doneInitHandler, *this, *this, _use_legacy_mode, i)); + } _stripe_scan_stats.resize(num_distributor_stripes); _distributorStatusDelegate.registerStatusPage(); _bucket_db_status_delegate = std::make_unique<StatusReporterDelegate>(compReg, *this, *_bucket_db_updater); @@ -114,14 +125,12 @@ Distributor::~Distributor() // TODO STRIPE remove DistributorStripe& Distributor::first_stripe() noexcept { - assert(_stripes.size() == 1); return *_stripes[0]; } // TODO STRIPE remove const DistributorStripe& Distributor::first_stripe() const noexcept { - assert(_stripes.size() == 1); return *_stripes[0]; } @@ -252,7 +261,10 @@ Distributor::onOpen() _threadPool.addThread(*this); _threadPool.start(_component.getThreadPool()); if (!_use_legacy_mode) { - std::vector<TickableStripe*> pool_stripes({_stripes[0].get()}); + std::vector<TickableStripe*> pool_stripes; + for (auto& stripe : _stripes) { + pool_stripes.push_back(stripe.get()); + } _stripe_pool.start(pool_stripes); } } else { @@ -319,6 +331,49 @@ bool should_be_handled_by_top_level_bucket_db_updater(const api::StorageMessage& } } +document::BucketId +get_bucket_id_for_striping(const api::StorageMessage& msg, const DistributorNodeContext& node_ctx) +{ + if (!msg.getBucketId().isSet()) { + // Calculate a bucket id (dependent on the message type) to dispatch the message to the correct distributor stripe. + switch (msg.getType().getId()) { + case api::MessageType::PUT_ID: + case api::MessageType::UPDATE_ID: + case api::MessageType::REMOVE_ID: + return node_ctx.bucket_id_factory().getBucketId(dynamic_cast<const api::TestAndSetCommand&>(msg).getDocumentId()); + case api::MessageType::REQUESTBUCKETINFO_REPLY_ID: + { + const auto& reply = dynamic_cast<const api::RequestBucketInfoReply&>(msg); + if (!reply.getBucketInfo().empty()) { + // Note: All bucket ids in this reply belong to the same distributor stripe, so we just use the first entry. + return reply.getBucketInfo()[0]._bucketId; + } else { + return reply.getBucketId(); + } + } + case api::MessageType::GET_ID: + return node_ctx.bucket_id_factory().getBucketId(dynamic_cast<const api::GetCommand&>(msg).getDocumentId()); + case api::MessageType::VISITOR_CREATE_ID: + return dynamic_cast<const api::CreateVisitorCommand&>(msg).super_bucket_id(); + case api::MessageType::VISITOR_CREATE_REPLY_ID: + return dynamic_cast<const api::CreateVisitorReply&>(msg).super_bucket_id(); + default: + return msg.getBucketId(); + } + } + return msg.getBucketId(); +} + +uint32_t +stripe_of_bucket_id(const document::BucketId& bucketd_id, uint8_t n_stripe_bits) +{ + if (!bucketd_id.isSet()) { + // TODO STRIPE: Messages with a non-set bucket id should be handled by the top-level distributor instead. + return 0; + } + return storage::stripe_of_bucket_key(bucketd_id.toKey(), n_stripe_bits); +} + } bool @@ -333,12 +388,13 @@ Distributor::onDown(const std::shared_ptr<api::StorageMessage>& msg) dispatch_to_main_distributor_thread_queue(msg); return true; } - assert(_stripes.size() == 1); - assert(_stripe_pool.stripe_count() == 1); - // TODO STRIPE correct routing with multiple stripes - bool handled = first_stripe().handle_or_enqueue_message(msg); + auto bucket_id = get_bucket_id_for_striping(*msg, _component); + uint32_t stripe_idx = stripe_of_bucket_id(bucket_id, _n_stripe_bits); + MBUS_TRACE(msg->getTrace(), 9, + vespalib::make_string("Distributor::onDown(): Dispatch message to stripe %u", stripe_idx)); + bool handled = _stripes[stripe_idx]->handle_or_enqueue_message(msg); if (handled) { - _stripe_pool.stripe_thread(0).notify_event_has_triggered(); + _stripe_pool.stripe_thread(stripe_idx).notify_event_has_triggered(); } return handled; } @@ -440,7 +496,6 @@ Distributor::propagateDefaultDistribution( } else { // Should only be called at ctor time, at which point the pool is not yet running. assert(_stripe_pool.stripe_count() == 0); - assert(_stripes.size() == 1); // TODO STRIPE all the stripes yes auto new_configs = BucketSpaceDistributionConfigs::from_default_distribution(std::move(distribution)); for (auto& stripe : _stripes) { stripe->update_distribution_config(new_configs); diff --git a/storage/src/vespa/storage/distributor/distributor.h b/storage/src/vespa/storage/distributor/distributor.h index 50bd2526ff4..61a1f06309d 100644 --- a/storage/src/vespa/storage/distributor/distributor.h +++ b/storage/src/vespa/storage/distributor/distributor.h @@ -201,6 +201,7 @@ private: ChainedMessageSender* _messageSender; const bool _use_legacy_mode; // TODO STRIPE multiple stripes...! This is for proof of concept of wiring. + uint8_t _n_stripe_bits; std::unique_ptr<DistributorStripe> _stripe; DistributorStripePool& _stripe_pool; std::vector<std::unique_ptr<DistributorStripe>> _stripes; diff --git a/storage/src/vespa/storage/distributor/distributor_stripe.cpp b/storage/src/vespa/storage/distributor/distributor_stripe.cpp index 5c6c529fe69..bf78707cfd9 100644 --- a/storage/src/vespa/storage/distributor/distributor_stripe.cpp +++ b/storage/src/vespa/storage/distributor/distributor_stripe.cpp @@ -41,7 +41,8 @@ DistributorStripe::DistributorStripe(DistributorComponentRegister& compReg, DoneInitializeHandler& doneInitHandler, ChainedMessageSender& messageSender, StripeHostInfoNotifier& stripe_host_info_notifier, - bool use_legacy_mode) + bool use_legacy_mode, + uint32_t stripe_index) : DistributorStripeInterface(), framework::StatusReporter("distributor", "Distributor"), _clusterStateBundle(lib::ClusterState()), @@ -57,7 +58,7 @@ DistributorStripe::DistributorStripe(DistributorComponentRegister& compReg, _bucketDBUpdater(_component, _component, *this, *this, use_legacy_mode), _distributorStatusDelegate(compReg, *this, *this), _bucketDBStatusDelegate(compReg, *this, _bucketDBUpdater), - _idealStateManager(*this, *_bucketSpaceRepo, *_readOnlyBucketSpaceRepo, compReg), + _idealStateManager(*this, *_bucketSpaceRepo, *_readOnlyBucketSpaceRepo, compReg, stripe_index), _messageSender(messageSender), _stripe_host_info_notifier(stripe_host_info_notifier), _externalOperationHandler(_component, _component, getMetrics(), getMessageSender(), @@ -86,7 +87,8 @@ DistributorStripe::DistributorStripe(DistributorComponentRegister& compReg, _last_db_memory_sample_time_point(), _inhibited_maintenance_tick_count(0), _must_send_updated_host_info(false), - _use_legacy_mode(use_legacy_mode) + _use_legacy_mode(use_legacy_mode), + _stripe_index(stripe_index) { if (use_legacy_mode) { _distributorStatusDelegate.registerStatusPage(); @@ -176,7 +178,7 @@ DistributorStripe::handle_or_enqueue_message(const std::shared_ptr<api::StorageM return true; } MBUS_TRACE(msg->getTrace(), 9, - "Distributor: Added to message queue. Thread state: " + vespalib::make_string("DistributorStripe[%u]: Added to message queue. Thread state: ", _stripe_index) + _threadPool.getStatus()); if (_use_legacy_mode) { // TODO STRIPE remove @@ -570,7 +572,8 @@ bool is_client_request(const api::StorageMessage& msg) noexcept { void DistributorStripe::handle_or_propagate_message(const std::shared_ptr<api::StorageMessage>& msg) { if (!handleMessage(msg)) { - MBUS_TRACE(msg->getTrace(), 9, "Distributor: Not handling it. Sending further down."); + MBUS_TRACE(msg->getTrace(), 9, + vespalib::make_string("DistributorStripe[%u]: Not handling it. Sending further down", _stripe_index)); _messageSender.sendDown(msg); } } @@ -578,10 +581,12 @@ void DistributorStripe::handle_or_propagate_message(const std::shared_ptr<api::S void DistributorStripe::startExternalOperations() { for (auto& msg : _fetchedMessages) { if (is_client_request(*msg)) { - MBUS_TRACE(msg->getTrace(), 9, "Distributor: adding to client request priority queue"); + MBUS_TRACE(msg->getTrace(), 9, + vespalib::make_string("DistributorStripe[%u]: Adding to client request priority queue", _stripe_index)); _client_request_priority_queue.emplace(std::move(msg)); } else { - MBUS_TRACE(msg->getTrace(), 9, "Distributor: Grabbed from queue to be processed."); + MBUS_TRACE(msg->getTrace(), 9, + vespalib::make_string("DistributorStripe[%u]: Grabbed from queue to be processed", _stripe_index)); handle_or_propagate_message(msg); } } @@ -589,8 +594,9 @@ void DistributorStripe::startExternalOperations() { const bool start_single_client_request = !_client_request_priority_queue.empty(); if (start_single_client_request) { const auto& msg = _client_request_priority_queue.top(); - MBUS_TRACE(msg->getTrace(), 9, "Distributor: Grabbed from " - "client request priority queue to be processed."); + MBUS_TRACE(msg->getTrace(), 9, + vespalib::make_string("DistributorStripe[%u]: Grabbed from " + "client request priority queue to be processed", _stripe_index)); handle_or_propagate_message(msg); _client_request_priority_queue.pop(); } diff --git a/storage/src/vespa/storage/distributor/distributor_stripe.h b/storage/src/vespa/storage/distributor/distributor_stripe.h index b82b5483bd3..347863b6d77 100644 --- a/storage/src/vespa/storage/distributor/distributor_stripe.h +++ b/storage/src/vespa/storage/distributor/distributor_stripe.h @@ -64,7 +64,8 @@ public: DoneInitializeHandler&, ChainedMessageSender& messageSender, StripeHostInfoNotifier& stripe_host_info_notifier, - bool use_legacy_mode); + bool use_legacy_mode, + uint32_t stripe_index = 0); ~DistributorStripe() override; @@ -363,6 +364,7 @@ private: size_t _inhibited_maintenance_tick_count; bool _must_send_updated_host_info; bool _use_legacy_mode; + uint32_t _stripe_index; }; } diff --git a/storage/src/vespa/storage/distributor/distributor_stripe_pool.cpp b/storage/src/vespa/storage/distributor/distributor_stripe_pool.cpp index 715d95e70fb..13162de4208 100644 --- a/storage/src/vespa/storage/distributor/distributor_stripe_pool.cpp +++ b/storage/src/vespa/storage/distributor/distributor_stripe_pool.cpp @@ -1,6 +1,7 @@ // Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "distributor_stripe_pool.h" #include "distributor_stripe_thread.h" +#include <vespa/storage/common/bucket_stripe_utils.h> #include <vespa/vespalib/util/size_literals.h> #include <cassert> @@ -8,6 +9,7 @@ namespace storage::distributor { DistributorStripePool::DistributorStripePool() : _thread_pool(512_Ki), + _n_stripe_bits(0), _stripes(), _threads(), _mutex(), @@ -48,6 +50,14 @@ void DistributorStripePool::unpark_all_threads() noexcept { _parker_cond.wait(lock, [this]{ return (_parked_threads == 0); }); } +const TickableStripe& DistributorStripePool::stripe_of_key(uint64_t key) const noexcept { + return stripe_thread(stripe_of_bucket_key(key, _n_stripe_bits)).stripe(); +} + +TickableStripe& DistributorStripePool::stripe_of_key(uint64_t key) noexcept { + return stripe_thread(stripe_of_bucket_key(key, _n_stripe_bits)).stripe(); +} + void DistributorStripePool::park_thread_until_released(DistributorStripeThread& thread) noexcept { std::unique_lock lock(_mutex); assert(_parked_threads < _threads.size()); @@ -67,6 +77,8 @@ void DistributorStripePool::park_thread_until_released(DistributorStripeThread& void DistributorStripePool::start(const std::vector<TickableStripe*>& stripes) { assert(!stripes.empty()); assert(_stripes.empty() && _threads.empty()); + assert(stripes.size() == adjusted_num_stripes(stripes.size())); + _n_stripe_bits = calc_num_stripe_bits(stripes.size()); _stripes.reserve(stripes.size()); _threads.reserve(stripes.size()); diff --git a/storage/src/vespa/storage/distributor/distributor_stripe_pool.h b/storage/src/vespa/storage/distributor/distributor_stripe_pool.h index 5e72cb47fc4..75328479296 100644 --- a/storage/src/vespa/storage/distributor/distributor_stripe_pool.h +++ b/storage/src/vespa/storage/distributor/distributor_stripe_pool.h @@ -38,6 +38,7 @@ class DistributorStripePool { using NativeThreadVector = std::vector<FastOS_ThreadInterface*>; FastOS_ThreadPool _thread_pool; + uint8_t _n_stripe_bits; StripeVector _stripes; NativeThreadVector _threads; std::mutex _mutex; @@ -57,7 +58,8 @@ public: // Set up the stripe pool with a 1-1 relationship between the provided // stripes and running threads. Can only be called once per pool. // - // Precondition: stripes.size() > 0 + // Precondition: stripes.size() > 0 && + // when stripes.size() > 1: is a power of 2 and within storage::MaxStripes boundary void start(const std::vector<TickableStripe*>& stripes); void stop_and_join(); @@ -76,6 +78,8 @@ public: [[nodiscard]] DistributorStripeThread& stripe_thread(size_t idx) noexcept { return *_stripes[idx]; } + [[nodiscard]] const TickableStripe& stripe_of_key(uint64_t key) const noexcept; + [[nodiscard]] TickableStripe& stripe_of_key(uint64_t key) noexcept; [[nodiscard]] size_t stripe_count() const noexcept { return _stripes.size(); } [[nodiscard]] bool is_stopped() const noexcept { return _stopped; } diff --git a/storage/src/vespa/storage/distributor/idealstatemanager.cpp b/storage/src/vespa/storage/distributor/idealstatemanager.cpp index 7bebe4c001a..013551b8505 100644 --- a/storage/src/vespa/storage/distributor/idealstatemanager.cpp +++ b/storage/src/vespa/storage/distributor/idealstatemanager.cpp @@ -28,13 +28,17 @@ IdealStateManager::IdealStateManager( DistributorStripeInterface& owner, DistributorBucketSpaceRepo& bucketSpaceRepo, DistributorBucketSpaceRepo& readOnlyBucketSpaceRepo, - DistributorComponentRegister& compReg) + DistributorComponentRegister& compReg, + uint32_t stripe_index) : _metrics(new IdealStateMetricSet), _distributorComponent(owner, bucketSpaceRepo, readOnlyBucketSpaceRepo, compReg, "Ideal state manager"), _bucketSpaceRepo(bucketSpaceRepo), _has_logged_phantom_replica_warning(false) { - _distributorComponent.registerMetric(*_metrics); + if (stripe_index == 0) { + // TODO STRIPE: Add proper handling of metrics across distributor stripes + _distributorComponent.registerMetric(*_metrics); + } LOG(debug, "Adding BucketStateStateChecker to state checkers"); _stateCheckers.push_back(StateChecker::SP(new BucketStateStateChecker())); diff --git a/storage/src/vespa/storage/distributor/idealstatemanager.h b/storage/src/vespa/storage/distributor/idealstatemanager.h index ab7a64142f6..041e009ee9f 100644 --- a/storage/src/vespa/storage/distributor/idealstatemanager.h +++ b/storage/src/vespa/storage/distributor/idealstatemanager.h @@ -36,7 +36,8 @@ public: IdealStateManager(DistributorStripeInterface& owner, DistributorBucketSpaceRepo& bucketSpaceRepo, DistributorBucketSpaceRepo& readOnlyBucketSpaceRepo, - DistributorComponentRegister& compReg); + DistributorComponentRegister& compReg, + uint32_t stripe_index = 0); ~IdealStateManager() override; diff --git a/storage/src/vespa/storage/distributor/multi_threaded_stripe_access_guard.cpp b/storage/src/vespa/storage/distributor/multi_threaded_stripe_access_guard.cpp index a5adf732824..80af41e57ad 100644 --- a/storage/src/vespa/storage/distributor/multi_threaded_stripe_access_guard.cpp +++ b/storage/src/vespa/storage/distributor/multi_threaded_stripe_access_guard.cpp @@ -84,9 +84,26 @@ MultiThreadedStripeAccessGuard::merge_entries_into_db(document::BucketSpace buck const std::unordered_set<uint16_t>& outdated_nodes, const std::vector<dbtransition::Entry>& entries) { - // TODO STRIPE multiple stripes - first_stripe().merge_entries_into_db(bucket_space, gathered_at_timestamp, distribution, - new_state, storage_up_states, outdated_nodes, entries); + if (entries.empty()) { + return; + } + std::vector<dbtransition::Entry> stripe_entries; + stripe_entries.reserve(entries.size() / _stripe_pool.stripe_count()); + auto* curr_stripe = &_stripe_pool.stripe_of_key(entries[0].bucket_key); + stripe_entries.push_back(entries[0]); + for (size_t i = 1; i < entries.size(); ++i) { + const auto& entry = entries[i]; + auto* next_stripe = &_stripe_pool.stripe_of_key(entry.bucket_key); + if (curr_stripe != next_stripe) { + curr_stripe->merge_entries_into_db(bucket_space, gathered_at_timestamp, distribution, + new_state, storage_up_states, outdated_nodes, stripe_entries); + stripe_entries.clear(); + } + curr_stripe = next_stripe; + stripe_entries.push_back(entry); + } + curr_stripe->merge_entries_into_db(bucket_space, gathered_at_timestamp, distribution, + new_state, storage_up_states, outdated_nodes, stripe_entries); } void MultiThreadedStripeAccessGuard::update_read_snapshot_before_db_pruning() { diff --git a/storage/src/vespa/storage/distributor/operations/idealstate/idealstateoperation.cpp b/storage/src/vespa/storage/distributor/operations/idealstate/idealstateoperation.cpp index 142ff72bc79..1a48df0fd7c 100644 --- a/storage/src/vespa/storage/distributor/operations/idealstate/idealstateoperation.cpp +++ b/storage/src/vespa/storage/distributor/operations/idealstate/idealstateoperation.cpp @@ -9,8 +9,8 @@ #include <vespa/log/log.h> LOG_SETUP(".distributor.operation"); -using namespace storage; -using namespace storage::distributor; +namespace storage::distributor { + using document::BucketSpace; const uint32_t IdealStateOperation::MAINTENANCE_MESSAGE_TYPES[] = @@ -85,7 +85,7 @@ IdealStateOperation::setIdealStateManager(IdealStateManager* manager) { void IdealStateOperation::done() { - if (_manager != NULL) { + if (_manager) { if (ok()) { _manager->getMetrics().operations[getType()]->ok.inc(1); } else { @@ -107,35 +107,6 @@ IdealStateOperation::setCommandMeta(api::MaintenanceCommand& cmd) const cmd.setReason(_detailedReason); } -std::string -IdealStateOperation::toXML(framework::Clock& clock) const -{ - std::ostringstream ost; - - ost << "<operation bucketid=\"" << getBucketId() - << "\" reason=\"" << _detailedReason << "\" operations=\""; - - ost << getName() << "["; - for (uint32_t j = 0; j < getNodes().size(); j++) { - if (j != 0) { - ost << ","; - } - ost << getNodes()[j]; - } - ost << "]"; - - if (getStartTime().isSet()) { - uint64_t timeSpent( - (clock.getTimeInMillis() - getStartTime()).getTime()); - ost << "\" runtime_secs=\"" << timeSpent << "\""; - } else { - ost << "\""; - } - - ost << "/>"; - return ost.str(); -} - namespace { class IdealStateOpChecker : public PendingMessageTracker::Checker @@ -277,3 +248,5 @@ IdealStateOperation::shouldBlockThisOperation(uint32_t messageType, return false; } + +} diff --git a/storage/src/vespa/storage/distributor/operations/idealstate/idealstateoperation.h b/storage/src/vespa/storage/distributor/operations/idealstate/idealstateoperation.h index 7906150d0cb..0e45d7f3b3a 100644 --- a/storage/src/vespa/storage/distributor/operations/idealstate/idealstateoperation.h +++ b/storage/src/vespa/storage/distributor/operations/idealstate/idealstateoperation.h @@ -208,8 +208,6 @@ public: */ void setCommandMeta(api::MaintenanceCommand& cmd) const; - std::string toXML(framework::Clock& clock) const; - std::string toString() const override; /** diff --git a/storage/src/vespa/storage/distributor/operations/idealstate/mergeoperation.cpp b/storage/src/vespa/storage/distributor/operations/idealstate/mergeoperation.cpp index 481506096eb..27e203a9060 100644 --- a/storage/src/vespa/storage/distributor/operations/idealstate/mergeoperation.cpp +++ b/storage/src/vespa/storage/distributor/operations/idealstate/mergeoperation.cpp @@ -1,5 +1,6 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "mergeoperation.h" +#include <vespa/document/bucket/fixed_bucket_spaces.h> #include <vespa/storage/distributor/idealstatemanager.h> #include <vespa/storage/distributor/distributor_bucket_space.h> #include <vespa/storage/distributor/pendingmessagetracker.h> @@ -325,13 +326,30 @@ bool MergeOperation::shouldBlockThisOperation(uint32_t messageType, uint8_t pri) bool MergeOperation::isBlocked(const PendingMessageTracker& pending_tracker, const OperationSequencer& op_seq) const { - const auto& node_info = pending_tracker.getNodeInfo(); - for (auto node : getNodes()) { - if (node_info.isBusy(node)) { - return true; + // To avoid starvation of high priority global bucket merges, we do not consider + // these for blocking due to a node being "busy" (usually caused by a full merge + // throttler queue). + // + // This is for two reasons: + // 1. When an ideal state op is blocked, it is still removed from the internal + // maintenance priority queue. This means a blocked high pri operation will + // not be retried until the next DB pass (at which point the node is likely + // to still be marked as busy when there's heavy merge traffic). + // 2. Global bucket merges have high priority and will most likely be allowed + // to enter the merge throttler queues, displacing lower priority merges. + if (!is_global_bucket_merge()) { + const auto& node_info = pending_tracker.getNodeInfo(); + for (auto node : getNodes()) { + if (node_info.isBusy(node)) { + return true; + } } } return IdealStateOperation::isBlocked(pending_tracker, op_seq); } +bool MergeOperation::is_global_bucket_merge() const noexcept { + return getBucket().getBucketSpace() == document::FixedBucketSpaces::global_space(); +} + } diff --git a/storage/src/vespa/storage/distributor/operations/idealstate/mergeoperation.h b/storage/src/vespa/storage/distributor/operations/idealstate/mergeoperation.h index 5df9421e815..11b5494fd9b 100644 --- a/storage/src/vespa/storage/distributor/operations/idealstate/mergeoperation.h +++ b/storage/src/vespa/storage/distributor/operations/idealstate/mergeoperation.h @@ -61,6 +61,7 @@ private: void deleteSourceOnlyNodes(const BucketDatabase::Entry& currentState, DistributorStripeMessageSender& sender); + bool is_global_bucket_merge() const noexcept; }; } diff --git a/storage/src/vespa/storage/storageserver/mergethrottler.cpp b/storage/src/vespa/storage/storageserver/mergethrottler.cpp index dc8457769a2..a9bd7ade270 100644 --- a/storage/src/vespa/storage/storageserver/mergethrottler.cpp +++ b/storage/src/vespa/storage/storageserver/mergethrottler.cpp @@ -71,6 +71,7 @@ MergeThrottler::ChainedMergeState::~ChainedMergeState() = default; MergeThrottler::Metrics::Metrics(metrics::MetricSet* owner) : metrics::MetricSet("mergethrottler", {}, "", owner), averageQueueWaitingTime("averagequeuewaitingtime", {}, "Average time a merge spends in the throttler queue", this), + queueSize("queuesize", {}, "Length of merge queue", this), bounced_due_to_back_pressure("bounced_due_to_back_pressure", {}, "Number of merges bounced due to resource exhaustion back-pressure", this), chaining("mergechains", this), local("locallyexecutedmerges", this) @@ -415,6 +416,7 @@ MergeThrottler::enqueueMerge( if (!validateNewMerge(mergeCmd, nodeSeq, msgGuard)) { return; } + _metrics->queueSize.set(_queue.size()); _queue.insert(MergePriorityQueue::value_type(msg, _queueSequence++)); } diff --git a/storage/src/vespa/storage/storageserver/mergethrottler.h b/storage/src/vespa/storage/storageserver/mergethrottler.h index e8815eee680..0c608f29196 100644 --- a/storage/src/vespa/storage/storageserver/mergethrottler.h +++ b/storage/src/vespa/storage/storageserver/mergethrottler.h @@ -57,12 +57,13 @@ public: MergeFailureMetrics failures; MergeOperationMetrics(const std::string& name, metrics::MetricSet* owner); - ~MergeOperationMetrics(); + ~MergeOperationMetrics() override; }; class Metrics : public metrics::MetricSet { public: metrics::DoubleAverageMetric averageQueueWaitingTime; + metrics::LongValueMetric queueSize; metrics::LongCountMetric bounced_due_to_back_pressure; MergeOperationMetrics chaining; MergeOperationMetrics local; diff --git a/storage/src/vespa/storage/storageserver/rpc/caching_rpc_target_resolver.cpp b/storage/src/vespa/storage/storageserver/rpc/caching_rpc_target_resolver.cpp index 39959c24968..743225c6209 100644 --- a/storage/src/vespa/storage/storageserver/rpc/caching_rpc_target_resolver.cpp +++ b/storage/src/vespa/storage/storageserver/rpc/caching_rpc_target_resolver.cpp @@ -92,14 +92,13 @@ CachingRpcTargetResolver::insert_new_target_mapping(const api::StorageMessageAdd } std::shared_ptr<RpcTarget> -CachingRpcTargetResolver::resolve_rpc_target(const api::StorageMessageAddress& address, - uint64_t bucket_id) { +CachingRpcTargetResolver::resolve_rpc_target(const api::StorageMessageAddress& address, uint64_t bucket_id) { const uint32_t curr_slobrok_gen = _slobrok_mirror.updates(); if (auto result = lookup_target(address, bucket_id, curr_slobrok_gen)) { return result; } auto slobrok_id = address_to_slobrok_id(address); - auto specs = _slobrok_mirror.lookup(slobrok_id); // FIXME string type mismatch; implicit conv! + auto specs = _slobrok_mirror.lookup(slobrok_id); if (specs.empty()) { LOG(debug, "Found no mapping for '%s'", slobrok_id.c_str()); // TODO return potentially stale existing target if no longer existing in SB? diff --git a/storageapi/src/vespa/storageapi/message/visitor.cpp b/storageapi/src/vespa/storageapi/message/visitor.cpp index d87f65a72cf..ec7fb1dc2d4 100644 --- a/storageapi/src/vespa/storageapi/message/visitor.cpp +++ b/storageapi/src/vespa/storageapi/message/visitor.cpp @@ -74,6 +74,16 @@ CreateVisitorCommand::getBucket() const return document::Bucket(_bucketSpace, document::BucketId()); } +document::BucketId +CreateVisitorCommand::super_bucket_id() const +{ + if (_buckets.empty()) { + // TODO STRIPE: Is this actually an error situation? Should be fixed elsewhere. + return document::BucketId(); + } + return _buckets[0]; +} + void CreateVisitorCommand::print(std::ostream& out, bool verbose, const std::string& indent) const @@ -120,6 +130,7 @@ CreateVisitorCommand::print(std::ostream& out, bool verbose, CreateVisitorReply::CreateVisitorReply(const CreateVisitorCommand& cmd) : StorageReply(cmd), + _super_bucket_id(cmd.super_bucket_id()), _lastBucket(document::BucketId(INT_MAX)) { } diff --git a/storageapi/src/vespa/storageapi/message/visitor.h b/storageapi/src/vespa/storageapi/message/visitor.h index 8440591ecde..1313b275e95 100644 --- a/storageapi/src/vespa/storageapi/message/visitor.h +++ b/storageapi/src/vespa/storageapi/message/visitor.h @@ -79,6 +79,7 @@ public: uint32_t getVisitorCmdId() const { return _visitorCmdId; } document::BucketSpace getBucketSpace() const { return _bucketSpace; } document::Bucket getBucket() const override; + document::BucketId super_bucket_id() const; const vespalib::string & getLibraryName() const { return _libName; } const vespalib::string & getInstanceId() const { return _instanceId; } const vespalib::string & getControlDestination() const { return _controlDestination; } @@ -114,6 +115,7 @@ public: */ class CreateVisitorReply : public StorageReply { private: + document::BucketId _super_bucket_id; document::BucketId _lastBucket; vdslib::VisitorStatistics _visitorStatistics; @@ -124,6 +126,7 @@ public: void setLastBucket(const document::BucketId& lastBucket) { _lastBucket = lastBucket; } + const document::BucketId& super_bucket_id() const { return _super_bucket_id; } const document::BucketId& getLastBucket() const { return _lastBucket; } void setVisitorStatistics(const vdslib::VisitorStatistics& stats) { _visitorStatistics = stats; } diff --git a/storageserver/src/apps/storaged/storage.cpp b/storageserver/src/apps/storaged/storage.cpp index 8098ee138db..6fd672574c6 100644 --- a/storageserver/src/apps/storaged/storage.cpp +++ b/storageserver/src/apps/storaged/storage.cpp @@ -65,9 +65,7 @@ public: ~StorageApp() override; void handleSignal(int signal) { - LOG(info, "Got signal %d, waiting for lock", signal); - std::lock_guard sync(_signalLock); - LOG(info, "Got lock for signal %d", signal); + LOG(info, "Got signal %d", signal); _lastSignal = signal; _signalCond.notify_one(); } diff --git a/storageserver/src/vespa/storageserver/app/distributorprocess.cpp b/storageserver/src/vespa/storageserver/app/distributorprocess.cpp index ede7fd1c9c0..45802df8866 100644 --- a/storageserver/src/vespa/storageserver/app/distributorprocess.cpp +++ b/storageserver/src/vespa/storageserver/app/distributorprocess.cpp @@ -1,9 +1,10 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "distributorprocess.h" -#include <vespa/storage/common/storagelink.h> -#include <vespa/storage/common/i_storage_chain_builder.h> #include <vespa/config/helper/configgetter.hpp> +#include <vespa/storage/common/bucket_stripe_utils.h> +#include <vespa/storage/common/i_storage_chain_builder.h> +#include <vespa/storage/common/storagelink.h> #include <vespa/log/log.h> LOG_SETUP(".process.distributor"); @@ -32,6 +33,21 @@ DistributorProcess::shutdown() _node.reset(); } +namespace { + +uint32_t +adjusted_num_distributor_stripes(uint32_t cfg_n_stripes) +{ + uint32_t adjusted_n_stripes = storage::adjusted_num_stripes(cfg_n_stripes); + if (adjusted_n_stripes != cfg_n_stripes) { + LOG(warning, "Configured number of distributor stripes (%u) is not valid. Adjusting to a valid value (%u)", + cfg_n_stripes, adjusted_n_stripes); + } + return adjusted_n_stripes; +} + +} + void DistributorProcess::setupConfig(milliseconds subscribeTimeout) { @@ -40,7 +56,7 @@ DistributorProcess::setupConfig(milliseconds subscribeTimeout) auto distr_cfg = config::ConfigGetter<StorDistributormanagerConfig>::getConfig( _configUri.getConfigId(), _configUri.getContext(), subscribeTimeout); - _num_distributor_stripes = distr_cfg->numDistributorStripes; + _num_distributor_stripes = adjusted_num_distributor_stripes(distr_cfg->numDistributorStripes); _distributorConfigHandler = _configSubscriber.subscribe<StorDistributormanagerConfig>(_configUri.getConfigId(), subscribeTimeout); _visitDispatcherConfigHandler = _configSubscriber.subscribe<StorVisitordispatcherConfig>(_configUri.getConfigId(), subscribeTimeout); Process::setupConfig(subscribeTimeout); diff --git a/vdslib/src/vespa/vdslib/distribution/redundancygroupdistribution.cpp b/vdslib/src/vespa/vdslib/distribution/redundancygroupdistribution.cpp index 6ac521a0f01..863ac45baa8 100644 --- a/vdslib/src/vespa/vdslib/distribution/redundancygroupdistribution.cpp +++ b/vdslib/src/vespa/vdslib/distribution/redundancygroupdistribution.cpp @@ -5,6 +5,7 @@ #include <vespa/vespalib/text/stringtokenizer.h> #include <boost/lexical_cast.hpp> #include <algorithm> +#include <cassert> namespace storage::lib { diff --git a/vdstestlib/src/vespa/vdstestlib/config/dirconfig.cpp b/vdstestlib/src/vespa/vdstestlib/config/dirconfig.cpp index 3a26ed9dec8..6cd9a132d5e 100644 --- a/vdstestlib/src/vespa/vdstestlib/config/dirconfig.cpp +++ b/vdstestlib/src/vespa/vdstestlib/config/dirconfig.cpp @@ -7,6 +7,7 @@ #include <vespa/vespalib/util/stringfmt.h> #include <fstream> #include <atomic> +#include <cassert> #include <vespa/log/log.h> LOG_SETUP(".dirconfig"); diff --git a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/zms/DefaultZmsClient.java b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/zms/DefaultZmsClient.java index f9fd1c5e7e9..f73ac9c3535 100644 --- a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/zms/DefaultZmsClient.java +++ b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/zms/DefaultZmsClient.java @@ -5,6 +5,7 @@ import com.yahoo.vespa.athenz.api.AthenzDomain; import com.yahoo.vespa.athenz.api.AthenzIdentity; import com.yahoo.vespa.athenz.api.AthenzResourceName; import com.yahoo.vespa.athenz.api.AthenzRole; +import com.yahoo.vespa.athenz.api.AthenzUser; import com.yahoo.vespa.athenz.api.OktaAccessToken; import com.yahoo.vespa.athenz.api.OktaIdentityToken; import com.yahoo.vespa.athenz.client.ErrorHandler; @@ -12,23 +13,29 @@ import com.yahoo.vespa.athenz.client.common.ClientBase; import com.yahoo.vespa.athenz.client.zms.bindings.AccessResponseEntity; import com.yahoo.vespa.athenz.client.zms.bindings.AssertionEntity; import com.yahoo.vespa.athenz.client.zms.bindings.DomainListResponseEntity; -import com.yahoo.vespa.athenz.client.zms.bindings.MembershipResponseEntity; +import com.yahoo.vespa.athenz.client.zms.bindings.MembershipEntity; import com.yahoo.vespa.athenz.client.zms.bindings.PolicyEntity; import com.yahoo.vespa.athenz.client.zms.bindings.ProviderResourceGroupRolesRequestEntity; +import com.yahoo.vespa.athenz.client.zms.bindings.RoleEntity; import com.yahoo.vespa.athenz.client.zms.bindings.TenancyRequestEntity; import com.yahoo.vespa.athenz.identity.ServiceIdentityProvider; +import com.yahoo.vespa.athenz.utils.AthenzIdentities; import org.apache.http.Header; import org.apache.http.client.methods.HttpUriRequest; import org.apache.http.client.methods.RequestBuilder; +import org.apache.http.entity.StringEntity; import org.apache.http.message.BasicHeader; import javax.net.ssl.SSLContext; import java.net.URI; +import java.time.Instant; import java.util.Collections; import java.util.List; import java.util.OptionalInt; import java.util.Set; +import java.util.function.Function; import java.util.function.Supplier; +import java.util.stream.Collectors; import static java.util.stream.Collectors.toList; @@ -105,7 +112,10 @@ public class DefaultZmsClient extends ClientBase implements ZmsClient { @Override public void addRoleMember(AthenzRole role, AthenzIdentity member) { URI uri = zmsUrl.resolve(String.format("domain/%s/role/%s/member/%s", role.domain().getName(), role.roleName(), member.getFullName())); - HttpUriRequest request = RequestBuilder.put(uri).build(); + MembershipEntity membership = new MembershipEntity(member.getFullName(), true, role.roleName(), null); + HttpUriRequest request = RequestBuilder.put(uri) + .setEntity(toJsonStringEntity(membership)) + .build(); execute(request, response -> readEntity(response, Void.class)); } @@ -123,7 +133,7 @@ public class DefaultZmsClient extends ClientBase implements ZmsClient { .setUri(uri) .build(); return execute(request, response -> { - MembershipResponseEntity membership = readEntity(response, MembershipResponseEntity.class); + MembershipEntity membership = readEntity(response, MembershipEntity.class); return membership.isMember; }); } @@ -194,6 +204,44 @@ public class DefaultZmsClient extends ClientBase implements ZmsClient { return true; } + @Override + public List<AthenzUser> listPendingRoleApprovals(AthenzRole athenzRole) { + URI uri = zmsUrl.resolve(String.format("domain/%s/role/%s?pending=true", athenzRole.domain().getName(), athenzRole.roleName())); + HttpUriRequest request = RequestBuilder.get() + .setUri(uri) + .build(); + RoleEntity roleEntity = execute(request, response -> readEntity(response, RoleEntity.class)); + return roleEntity.roleMembers().stream() + .filter(RoleEntity.Member::pendingApproval) + .map(RoleEntity.Member::memberName) + .map(AthenzIdentities::from) + .filter(identity -> AthenzIdentities.USER_PRINCIPAL_DOMAIN.equals(identity.getDomain())) + .map(AthenzUser.class::cast) + .collect(Collectors.toList()); + } + + @Override + public void approvePendingRoleMembership(AthenzRole athenzRole, AthenzUser athenzUser, Instant expiry) { + URI uri = zmsUrl.resolve(String.format("domain/%s/role/%s/member/%s/decision", athenzRole.domain().getName(), athenzRole.roleName(), athenzUser.getFullName())); + MembershipEntity membership = new MembershipEntity(athenzUser.getFullName(), true, athenzRole.roleName(), Long.toString(expiry.getEpochSecond())); + HttpUriRequest request = RequestBuilder.put() + .setUri(uri) + .setEntity(toJsonStringEntity(membership)) + .build(); + execute(request, response -> readEntity(response, Void.class)); + } + + @Override + public List<AthenzIdentity> listMembers(AthenzRole athenzRole) { + URI uri = zmsUrl.resolve(String.format("domain/%s/role/%s", athenzRole.domain().getName(), athenzRole.roleName())); + RoleEntity execute = execute(RequestBuilder.get(uri).build(), response -> readEntity(response, RoleEntity.class)); + return execute.roleMembers().stream() + .filter(member -> ! member.pendingApproval()) + .map(RoleEntity.Member::memberName) + .map(AthenzIdentities::from) + .collect(Collectors.toList()); + } + private static Header createCookieHeaderWithOktaTokens(OktaIdentityToken identityToken, OktaAccessToken accessToken) { return new BasicHeader("Cookie", String.format("okta_at=%s; okta_it=%s", accessToken.token(), identityToken.token())); } diff --git a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/zms/ZmsClient.java b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/zms/ZmsClient.java index c7f865a58bb..15e8ba77850 100644 --- a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/zms/ZmsClient.java +++ b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/zms/ZmsClient.java @@ -5,9 +5,11 @@ import com.yahoo.vespa.athenz.api.AthenzDomain; import com.yahoo.vespa.athenz.api.AthenzIdentity; import com.yahoo.vespa.athenz.api.AthenzResourceName; import com.yahoo.vespa.athenz.api.AthenzRole; +import com.yahoo.vespa.athenz.api.AthenzUser; import com.yahoo.vespa.athenz.api.OktaAccessToken; import com.yahoo.vespa.athenz.api.OktaIdentityToken; +import java.time.Instant; import java.util.List; import java.util.Set; @@ -42,5 +44,11 @@ public interface ZmsClient extends AutoCloseable { boolean deletePolicyRule(AthenzDomain athenzDomain, String athenzPolicy, String action, AthenzResourceName resourceName, AthenzRole athenzRole); + List<AthenzUser> listPendingRoleApprovals(AthenzRole athenzRole); + + void approvePendingRoleMembership(AthenzRole athenzRole, AthenzUser athenzUser, Instant expiry); + + List<AthenzIdentity> listMembers(AthenzRole athenzRole); + void close(); } diff --git a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/zms/bindings/MembershipEntity.java b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/zms/bindings/MembershipEntity.java new file mode 100644 index 00000000000..d0672473776 --- /dev/null +++ b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/zms/bindings/MembershipEntity.java @@ -0,0 +1,52 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.athenz.client.zms.bindings; + +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonGetter; +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.annotation.JsonProperty; + +/** + * @author bjorncs + * @author mortent + */ +@JsonInclude(JsonInclude.Include.NON_NULL) +@JsonIgnoreProperties(ignoreUnknown = true) +public class MembershipEntity { + public final String memberName; + public final boolean isMember; + public final String roleName; + public final String expiration; + + @JsonCreator + public MembershipEntity(@JsonProperty("memberName") String memberName, + @JsonProperty("isMember") boolean isMember, + @JsonProperty("roleName") String roleName, + @JsonProperty("expiration") String expiration) { + this.memberName = memberName; + this.isMember = isMember; + this.roleName = roleName; + this.expiration = expiration; + } + + @JsonGetter("memberName") + public String memberName() { + return memberName; + } + + @JsonGetter("isMember") + public boolean isMember() { + return isMember; + } + + @JsonGetter("roleName") + public String roleName() { + return roleName; + } + + @JsonGetter("expiration") + public String expiration() { + return expiration; + } +} diff --git a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/zms/bindings/MembershipResponseEntity.java b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/zms/bindings/MembershipResponseEntity.java deleted file mode 100644 index 499afb48f25..00000000000 --- a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/zms/bindings/MembershipResponseEntity.java +++ /dev/null @@ -1,28 +0,0 @@ -// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.athenz.client.zms.bindings; - -import com.fasterxml.jackson.annotation.JsonCreator; -import com.fasterxml.jackson.annotation.JsonIgnoreProperties; -import com.fasterxml.jackson.annotation.JsonProperty; - -/** - * @author bjorncs - */ -@JsonIgnoreProperties(ignoreUnknown = true) -public class MembershipResponseEntity { - public final String memberName; - public final boolean isMember; - public final String roleName; - public final String expiration; - - @JsonCreator - public MembershipResponseEntity(@JsonProperty("memberName") String memberName, - @JsonProperty("isMember") boolean isMember, - @JsonProperty("roleName") String roleName, - @JsonProperty("expiration") String expiration) { - this.memberName = memberName; - this.isMember = isMember; - this.roleName = roleName; - this.expiration = expiration; - } -} diff --git a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/zms/bindings/RoleEntity.java b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/zms/bindings/RoleEntity.java new file mode 100644 index 00000000000..5babe292138 --- /dev/null +++ b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/zms/bindings/RoleEntity.java @@ -0,0 +1,54 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +package com.yahoo.vespa.athenz.client.zms.bindings; + +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.fasterxml.jackson.annotation.JsonProperty; + +import java.util.List; + +/** + * @author mortent + */ +@JsonIgnoreProperties(ignoreUnknown = true) +public class RoleEntity { + private final String roleName; + private final List<Member> roleMembers; + + @JsonCreator + public RoleEntity(@JsonProperty("roleName") String roleName, @JsonProperty("roleMembers") List<Member> roleMembers) { + this.roleName = roleName; + this.roleMembers = roleMembers; + } + + public String roleName() { + return roleName; + } + + public List<Member> roleMembers() { + return roleMembers; + } + + @JsonIgnoreProperties(ignoreUnknown = true) + public static final class Member { + private final String memberName; + private final boolean active; + private final boolean approved; + + @JsonCreator + public Member(@JsonProperty("memberName") String memberName, @JsonProperty("active") boolean active, @JsonProperty("approved") boolean approved) { + this.memberName = memberName; + this.active = active; + this.approved = approved; + } + + public String memberName() { + return memberName; + } + + public boolean pendingApproval() { + return !approved; + } + } +} diff --git a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/zts/DefaultZtsClient.java b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/zts/DefaultZtsClient.java index cdbf0755059..24234757590 100644 --- a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/zts/DefaultZtsClient.java +++ b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/zts/DefaultZtsClient.java @@ -165,7 +165,7 @@ public class DefaultZtsClient extends ClientBase implements ZtsClient { @Override public X509Certificate getRoleCertificate(AthenzRole role, Pkcs10Csr csr, Duration expiry) { RoleCertificateRequestEntity requestEntity = new RoleCertificateRequestEntity(csr, expiry); - URI uri = ztsUrl.resolve(String.format("domain/%s/role/%s/token", role.domain().getName(), role.roleName())); + URI uri = ztsUrl.resolve("rolecert"); HttpUriRequest request = RequestBuilder.post(uri) .setEntity(toJsonStringEntity(requestEntity)) .build(); diff --git a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/zts/bindings/RoleCertificateRequestEntity.java b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/zts/bindings/RoleCertificateRequestEntity.java index 89bfce91154..16dd1d914ef 100644 --- a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/zts/bindings/RoleCertificateRequestEntity.java +++ b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/zts/bindings/RoleCertificateRequestEntity.java @@ -38,7 +38,7 @@ public class RoleCertificateRequestEntity { public void serialize(Duration duration, JsonGenerator jsonGenerator, SerializerProvider serializerProvider) throws IOException { - jsonGenerator.writeNumber(duration.getSeconds()); + jsonGenerator.writeNumber(duration.toMinutes()); } } } diff --git a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/zts/bindings/RoleCertificateResponseEntity.java b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/zts/bindings/RoleCertificateResponseEntity.java index 857bfad9143..cd9a12c0074 100644 --- a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/zts/bindings/RoleCertificateResponseEntity.java +++ b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/zts/bindings/RoleCertificateResponseEntity.java @@ -16,12 +16,9 @@ import java.time.Instant; @JsonIgnoreProperties(ignoreUnknown = true) public class RoleCertificateResponseEntity { public final X509Certificate certificate; - public final Instant expiry; @JsonCreator - public RoleCertificateResponseEntity(@JsonProperty("token") @JsonDeserialize(using = X509CertificateDeserializer.class) X509Certificate certificate, - @JsonProperty("expiryTime") Instant expiry) { + public RoleCertificateResponseEntity(@JsonProperty("x509Certificate") @JsonDeserialize(using = X509CertificateDeserializer.class) X509Certificate certificate) { this.certificate = certificate; - this.expiry = expiry; } } diff --git a/vespa-feed-client-cli/CMakeLists.txt b/vespa-feed-client-cli/CMakeLists.txt index a918981dcd3..3967c135d1c 100644 --- a/vespa-feed-client-cli/CMakeLists.txt +++ b/vespa-feed-client-cli/CMakeLists.txt @@ -2,3 +2,4 @@ install_java_artifact(vespa-feed-client-cli) vespa_install_script(src/main/sh/vespa-feed-client.sh vespa-feed-client bin) +install(FILES src/main/resources/logging.properties DESTINATION conf/vespa-feed-client) diff --git a/vespa-feed-client-cli/pom.xml b/vespa-feed-client-cli/pom.xml index 28e1ab01dae..9fd59f1cfa4 100644 --- a/vespa-feed-client-cli/pom.xml +++ b/vespa-feed-client-cli/pom.xml @@ -36,6 +36,12 @@ <artifactId>jackson-core</artifactId> <scope>compile</scope> </dependency> + <dependency> + <groupId>org.slf4j</groupId> + <artifactId>slf4j-jdk14</artifactId> + <scope>compile</scope> + </dependency> + <!-- test scope --> <dependency> diff --git a/vespa-feed-client-cli/src/main/java/ai/vespa/feed/client/CliArguments.java b/vespa-feed-client-cli/src/main/java/ai/vespa/feed/client/CliArguments.java index 8855b08b06b..a3e33699a4d 100644 --- a/vespa-feed-client-cli/src/main/java/ai/vespa/feed/client/CliArguments.java +++ b/vespa-feed-client-cli/src/main/java/ai/vespa/feed/client/CliArguments.java @@ -187,7 +187,6 @@ class CliArguments { } private static Options createOptions() { - // TODO Add description to each option return new Options() .addOption(Option.builder() .longOpt(HELP_OPTION) @@ -197,65 +196,79 @@ class CliArguments { .build()) .addOption(Option.builder() .longOpt(ENDPOINT_OPTION) + .desc("URI to feed endpoint") .hasArg() .type(URL.class) .build()) .addOption(Option.builder() .longOpt(HEADER_OPTION) + .desc("HTTP header on the form 'Name: value'") .hasArgs() .build()) .addOption(Option.builder() .longOpt(FILE_OPTION) .type(File.class) + .desc("Path to feed file in JSON format") .hasArg() .build()) .addOption(Option.builder() .longOpt(CONNECTIONS_OPTION) + .desc("Number of concurrent HTTP/2 connections") .hasArg() .type(Number.class) .build()) .addOption(Option.builder() .longOpt(MAX_STREAMS_PER_CONNECTION) + .desc("Number of concurrent streams per HTTP/2 connection") .hasArg() .type(Number.class) .build()) .addOption(Option.builder() .longOpt(CERTIFICATE_OPTION) + .desc("Path to PEM encoded X.509 certificate file") .type(File.class) .hasArg() .build()) .addOption(Option.builder() .longOpt(PRIVATE_KEY_OPTION) + .desc("Path to PEM/PKCS#8 encoded private key file") .type(File.class) .hasArg() .build()) .addOption(Option.builder() .longOpt(CA_CERTIFICATES_OPTION) + .desc("Path to file containing CA X.509 certificates encoded as PEM") .type(File.class) .hasArg() .build()) .addOption(Option.builder() .longOpt(DISABLE_SSL_HOSTNAME_VERIFICATION_OPTION) + .desc("Disable SSL hostname verification") .build()) .addOption(Option.builder() .longOpt(BENCHMARK_OPTION) + .desc("Enable benchmark mode") .build()) .addOption(Option.builder() .longOpt(ROUTE_OPTION) + .desc("Target Vespa route for feed operations") .hasArg() .build()) .addOption(Option.builder() .longOpt(TIMEOUT_OPTION) + .desc("Feed operation timeout (in seconds)") .hasArg() .type(Number.class) .build()) .addOption(Option.builder() .longOpt(TRACE_OPTION) + .desc("The trace level of network traffic. Disabled by default (=0)") .hasArg() .type(Number.class) .build()) .addOption(Option.builder() .longOpt(STDIN_OPTION) + .desc("Read JSON input from standard input") .build()) .addOption(Option.builder() .longOpt(VERBOSE_OPTION) diff --git a/vespa-feed-client-cli/src/main/java/ai/vespa/feed/client/CliClient.java b/vespa-feed-client-cli/src/main/java/ai/vespa/feed/client/CliClient.java index f30d44ba4f3..e3f726eaf11 100644 --- a/vespa-feed-client-cli/src/main/java/ai/vespa/feed/client/CliClient.java +++ b/vespa-feed-client-cli/src/main/java/ai/vespa/feed/client/CliClient.java @@ -6,11 +6,12 @@ import com.fasterxml.jackson.core.JsonGenerator; import javax.net.ssl.HostnameVerifier; import javax.net.ssl.SSLSession; -import java.io.BufferedInputStream; import java.io.IOException; import java.io.InputStream; import java.io.PrintStream; import java.nio.file.Files; +import java.util.Map; +import java.util.Optional; import java.util.Properties; /** @@ -24,16 +25,19 @@ public class CliClient { private final PrintStream systemError; private final InputStream systemIn; private final Properties systemProperties; + private final Map<String, String> environmentVariables; - private CliClient(PrintStream systemOut, PrintStream systemError, InputStream systemIn, Properties systemProperties) { + private CliClient(PrintStream systemOut, PrintStream systemError, InputStream systemIn, + Properties systemProperties, Map<String, String> environmentVariables) { this.systemOut = systemOut; this.systemError = systemError; this.systemIn = systemIn; this.systemProperties = systemProperties; + this.environmentVariables = environmentVariables; } public static void main(String[] args) { - CliClient client = new CliClient(System.out, System.err, System.in, System.getProperties()); + CliClient client = new CliClient(System.out, System.err, System.in, System.getProperties(), System.getenv()); int exitCode = client.run(args); System.exit(exitCode); } @@ -70,8 +74,8 @@ public class CliClient { private static FeedClient createFeedClient(CliArguments cliArgs) throws CliArguments.CliArgumentsException { FeedClientBuilder builder = FeedClientBuilder.create(cliArgs.endpoint()); - cliArgs.connections().ifPresent(builder::setMaxConnections); - cliArgs.maxStreamsPerConnection().ifPresent(builder::setMaxConnections); + cliArgs.connections().ifPresent(builder::setConnectionsPerEndpoint); + cliArgs.maxStreamsPerConnection().ifPresent(builder::setMaxStreamPerConnection); if (cliArgs.sslHostnameVerificationDisabled()) { builder.setHostnameVerifier(AcceptAllHostnameVerifier.INSTANCE); } @@ -117,7 +121,10 @@ public class CliClient { } private boolean debugMode() { - return Boolean.parseBoolean(systemProperties.getProperty("VESPA_DEBUG", Boolean.FALSE.toString())); + boolean enabledWithSystemProperty = Boolean.parseBoolean(systemProperties.getProperty("VESPA_DEBUG", Boolean.FALSE.toString())); + boolean enabledWithEnvironmentVariable = Optional.ofNullable(environmentVariables.get("VESPA_DEBUG")) + .map(Boolean::parseBoolean).orElse(false); + return enabledWithSystemProperty || enabledWithEnvironmentVariable; } private static class AcceptAllHostnameVerifier implements HostnameVerifier { diff --git a/vespa-feed-client-cli/src/main/resources/logging.properties b/vespa-feed-client-cli/src/main/resources/logging.properties new file mode 100644 index 00000000000..3f0e8b24e78 --- /dev/null +++ b/vespa-feed-client-cli/src/main/resources/logging.properties @@ -0,0 +1,2 @@ +# Disable verbose info logging from org.apache.hc.client5.http.impl.async.AsyncHttpRequestRetryExec +org.apache.hc.client5.http.impl.async.level = WARNING diff --git a/vespa-feed-client-cli/src/main/sh/vespa-feed-client.sh b/vespa-feed-client-cli/src/main/sh/vespa-feed-client.sh index 2a166dd40bb..ab43fca2f67 100755 --- a/vespa-feed-client-cli/src/main/sh/vespa-feed-client.sh +++ b/vespa-feed-client-cli/src/main/sh/vespa-feed-client.sh @@ -79,4 +79,6 @@ exec java \ -Djava.library.path=${VESPA_HOME}/libexec64/native:${VESPA_HOME}/lib64 \ -Djava.awt.headless=true \ -Xms128m -Xmx2048m $(getJavaOptionsIPV46) \ +--add-opens=java.base/sun.security.ssl=ALL-UNNAMED \ +-Djava.util.logging.config.file=${VESPA_HOME}/conf/vespa-feed-client/logging.properties \ -cp ${VESPA_HOME}/lib/jars/vespa-feed-client-cli.jar ai.vespa.feed.client.CliClient "$@" diff --git a/vespa-feed-client-cli/src/test/resources/help.txt b/vespa-feed-client-cli/src/test/resources/help.txt index 29f24e8c672..1c537defb01 100644 --- a/vespa-feed-client-cli/src/test/resources/help.txt +++ b/vespa-feed-client-cli/src/test/resources/help.txt @@ -1,19 +1,31 @@ usage: vespa-feed-client <options> Vespa feed client - --benchmark - --ca-certificates <arg> - --certificate <arg> - --connections <arg> - --disable-ssl-hostname-verification - --endpoint <arg> - --file <arg> - --header <arg> + --benchmark Enable benchmark mode + --ca-certificates <arg> Path to file containing CA X.509 + certificates encoded as PEM + --certificate <arg> Path to PEM encoded X.509 + certificate file + --connections <arg> Number of concurrent HTTP/2 + connections + --disable-ssl-hostname-verification Disable SSL hostname + verification + --endpoint <arg> URI to feed endpoint + --file <arg> Path to feed file in JSON format + --header <arg> HTTP header on the form 'Name: + value' --help - --max-streams-per-connection <arg> - --private-key <arg> - --route <arg> - --stdin - --timeout <arg> - --trace <arg> + --max-streams-per-connection <arg> Number of concurrent streams per + HTTP/2 connection + --private-key <arg> Path to PEM/PKCS#8 encoded + private key file + --route <arg> Target Vespa route for feed + operations + --stdin Read JSON input from standard + input + --timeout <arg> Feed operation timeout (in + seconds) + --trace <arg> The trace level of network + traffic. Disabled by default + (=0) --verbose --version diff --git a/vespa-feed-client/src/main/java/ai/vespa/feed/client/Cluster.java b/vespa-feed-client/src/main/java/ai/vespa/feed/client/Cluster.java new file mode 100644 index 00000000000..fde230d3ca4 --- /dev/null +++ b/vespa-feed-client/src/main/java/ai/vespa/feed/client/Cluster.java @@ -0,0 +1,21 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package ai.vespa.feed.client; + +import org.apache.hc.client5.http.async.methods.SimpleHttpRequest; +import org.apache.hc.client5.http.async.methods.SimpleHttpResponse; + +import java.io.Closeable; +import java.util.concurrent.CompletableFuture; + +/** + * Allows dispatch to a Vespa cluster. {@link #dispatch} should be called by a single thread, i.e., it is not thread-safe. + */ +interface Cluster extends Closeable { + + /** Dispatch the request to the cluster, causing the response vessel to complete at a later time. */ + void dispatch(SimpleHttpRequest request, CompletableFuture<SimpleHttpResponse> vessel); + + @Override + default void close() { } + +} diff --git a/vespa-feed-client/src/main/java/ai/vespa/feed/client/FeedClient.java b/vespa-feed-client/src/main/java/ai/vespa/feed/client/FeedClient.java index 1b616a70da9..2ac75a948d9 100644 --- a/vespa-feed-client/src/main/java/ai/vespa/feed/client/FeedClient.java +++ b/vespa-feed-client/src/main/java/ai/vespa/feed/client/FeedClient.java @@ -10,24 +10,70 @@ import java.util.concurrent.CompletableFuture; */ public interface FeedClient extends Closeable { + /** Send a document put with the given parameters, returning a future with the result of the operation. */ CompletableFuture<Result> put(DocumentId documentId, String documentJson, OperationParameters params); + + /** Send a document update with the given parameters, returning a future with the result of the operation. */ CompletableFuture<Result> update(DocumentId documentId, String updateJson, OperationParameters params); + + /** Send a document remove with the given parameters, returning a future with the result of the operation. */ CompletableFuture<Result> remove(DocumentId documentId, OperationParameters params); + /** Shut down, and reject new operations. Operations in flight are allowed to complete normally if graceful. */ + void close(boolean graceful); + + /** Initiates graceful shutdown. See {@link #close(boolean)}. */ + default void close() { close(true); } + + /** Controls what to retry, and how many times. */ interface RetryStrategy { /** Whether to retry operations of the given type. */ default boolean retry(OperationType type) { return true; } /** Number of retries per operation for non-backpressure problems. */ - default int retries() { return 5; } + default int retries() { return 32; } + + } + + /** Allows slowing down or halting completely operations against the configured endpoint on high failure rates. */ + interface CircuitBreaker { + + /** Called by the client whenever a successful response is obtained. */ + void success(); + + /** Called by the client whenever a transient or fatal error occurs. */ + void failure(); + + /** The current state of the circuit breaker. */ + State state(); + + enum State { + + /** Circuit is closed: business as usual. */ + CLOSED, + + /** Circuit is half-open: something is wrong, perhaps it recovers? */ + HALF_OPEN, + + /** Circuit is open: we have given up. */ + OPEN; + + } } enum OperationType { - put, - update, - remove; + + /** A document put operation. This is idempotent. */ + PUT, + + /** A document update operation. This is idempotent if all its contained updates are. */ + UPDATE, + + /** A document remove operation. This is idempotent. */ + REMOVE; + } } diff --git a/vespa-feed-client/src/main/java/ai/vespa/feed/client/FeedClientBuilder.java b/vespa-feed-client/src/main/java/ai/vespa/feed/client/FeedClientBuilder.java index 3cd3f3cb4ca..da575a7cf6d 100644 --- a/vespa-feed-client/src/main/java/ai/vespa/feed/client/FeedClientBuilder.java +++ b/vespa-feed-client/src/main/java/ai/vespa/feed/client/FeedClientBuilder.java @@ -7,7 +7,12 @@ import java.io.IOException; import java.io.UncheckedIOException; import java.net.URI; import java.nio.file.Path; +import java.time.Clock; +import java.time.Duration; +import java.util.ArrayList; +import java.util.Collections; import java.util.HashMap; +import java.util.List; import java.util.Map; import java.util.function.Supplier; @@ -21,37 +26,45 @@ import static java.util.Objects.requireNonNull; */ public class FeedClientBuilder { - FeedClient.RetryStrategy defaultRetryStrategy = new FeedClient.RetryStrategy() { }; + static final FeedClient.RetryStrategy defaultRetryStrategy = new FeedClient.RetryStrategy() { }; - final URI endpoint; + final List<URI> endpoints; final Map<String, Supplier<String>> requestHeaders = new HashMap<>(); SSLContext sslContext; HostnameVerifier hostnameVerifier; - int maxConnections = 4; - int maxStreamsPerConnection = 1024; + int connectionsPerEndpoint = 4; + int maxStreamsPerConnection = 128; FeedClient.RetryStrategy retryStrategy = defaultRetryStrategy; + FeedClient.CircuitBreaker circuitBreaker = new GracePeriodCircuitBreaker(Clock.systemUTC(), Duration.ofSeconds(1), Duration.ofMinutes(10)); Path certificate; Path privateKey; Path caCertificates; - public static FeedClientBuilder create(URI endpoint) { return new FeedClientBuilder(endpoint); } + public static FeedClientBuilder create(URI endpoint) { return new FeedClientBuilder(Collections.singletonList(endpoint)); } - private FeedClientBuilder(URI endpoint) { - requireNonNull(endpoint.getHost()); - this.endpoint = endpoint; + public static FeedClientBuilder create(List<URI> endpoints) { return new FeedClientBuilder(endpoints); } + + private FeedClientBuilder(List<URI> endpoints) { + if (endpoints.isEmpty()) + throw new IllegalArgumentException("At least one endpoint must be provided"); + + for (URI endpoint : endpoints) + requireNonNull(endpoint.getHost()); + + this.endpoints = new ArrayList<>(endpoints); } /** - * Sets the maximum number of connections this client will use. + * Sets the number of connections this client will use per endpoint. * * A reasonable value here is a small multiple of the numbers of containers in the * cluster to feed, so load can be balanced across these. * In general, this value should be kept as low as possible, but poor connectivity * between feeder and cluster may also warrant a higher number of connections. */ - public FeedClientBuilder setMaxConnections(int max) { + public FeedClientBuilder setConnectionsPerEndpoint(int max) { if (max < 1) throw new IllegalArgumentException("Max connections must be at least 1, but was " + max); - this.maxConnections = max; + this.connectionsPerEndpoint = max; return this; } @@ -95,6 +108,11 @@ public class FeedClientBuilder { return this; } + public FeedClientBuilder setCircuitBreaker(FeedClient.CircuitBreaker breaker) { + this.circuitBreaker = requireNonNull(breaker); + return this; + } + public FeedClientBuilder setCertificate(Path certificatePemFile, Path privateKeyPemFile) { if (sslContext != null) throw new IllegalArgumentException("Cannot set both SSLContext and certificate"); this.certificate = certificatePemFile; diff --git a/vespa-feed-client/src/main/java/ai/vespa/feed/client/GracePeriodCircuitBreaker.java b/vespa-feed-client/src/main/java/ai/vespa/feed/client/GracePeriodCircuitBreaker.java new file mode 100644 index 00000000000..974d18418ec --- /dev/null +++ b/vespa-feed-client/src/main/java/ai/vespa/feed/client/GracePeriodCircuitBreaker.java @@ -0,0 +1,62 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package ai.vespa.feed.client; + +import java.time.Clock; +import java.time.Duration; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicLong; +import java.util.logging.Logger; + +import static java.util.Objects.requireNonNull; +import static java.util.logging.Level.INFO; +import static java.util.logging.Level.WARNING; + +/** + * Breaks the circuit when no successes have been recorded for a specified time. + */ +public class GracePeriodCircuitBreaker implements FeedClient.CircuitBreaker { + + private static final Logger log = Logger.getLogger(GracePeriodCircuitBreaker.class.getName()); + + private final AtomicLong lastSuccessMillis = new AtomicLong(0); // Trigger if first response is a failure. + private final AtomicBoolean halfOpen = new AtomicBoolean(false); + private final AtomicBoolean open = new AtomicBoolean(false); + private final Clock clock; + private final long graceMillis; + private final long doomMillis; + + GracePeriodCircuitBreaker(Clock clock, Duration grace, Duration doom) { + if (grace.isNegative()) + throw new IllegalArgumentException("Grace delay must be non-negative"); + + if (doom.isNegative()) + throw new IllegalArgumentException("Doom delay must be non-negative"); + + this.clock = requireNonNull(clock); + this.graceMillis = grace.toMillis(); + this.doomMillis = doom.toMillis(); + } + + @Override + public void success() { + lastSuccessMillis.set(clock.millis()); + if (halfOpen.compareAndSet(true, false)) + log.log(INFO, "Circuit breaker is now closed"); + } + + @Override + public void failure() { + long nowMillis = clock.millis(); + if (lastSuccessMillis.get() < nowMillis - doomMillis && open.compareAndSet(false, true)) + log.log(WARNING, "Circuit breaker is now open"); + + if (lastSuccessMillis.get() < nowMillis - graceMillis && halfOpen.compareAndSet(false, true)) + log.log(INFO, "Circuit breaker is now half-open"); + } + + @Override + public State state() { + return open.get() ? State.OPEN : halfOpen.get() ? State.HALF_OPEN : State.CLOSED; + } + +} diff --git a/vespa-feed-client/src/main/java/ai/vespa/feed/client/HttpCluster.java b/vespa-feed-client/src/main/java/ai/vespa/feed/client/HttpCluster.java new file mode 100644 index 00000000000..34491d23c4b --- /dev/null +++ b/vespa-feed-client/src/main/java/ai/vespa/feed/client/HttpCluster.java @@ -0,0 +1,151 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package ai.vespa.feed.client; + +import org.apache.hc.client5.http.async.methods.SimpleHttpRequest; +import org.apache.hc.client5.http.async.methods.SimpleHttpResponse; +import org.apache.hc.client5.http.config.RequestConfig; +import org.apache.hc.client5.http.impl.async.CloseableHttpAsyncClient; +import org.apache.hc.client5.http.impl.async.H2AsyncClientBuilder; +import org.apache.hc.client5.http.ssl.ClientTlsStrategyBuilder; +import org.apache.hc.core5.concurrent.FutureCallback; +import org.apache.hc.core5.http.message.BasicHeader; +import org.apache.hc.core5.http2.config.H2Config; +import org.apache.hc.core5.net.URIAuthority; +import org.apache.hc.core5.reactor.IOReactorConfig; +import org.apache.hc.core5.util.Timeout; + +import javax.net.ssl.SSLContext; +import java.io.IOException; +import java.net.URI; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.atomic.AtomicInteger; + +import static org.apache.hc.core5.http.ssl.TlsCiphers.excludeH2Blacklisted; +import static org.apache.hc.core5.http.ssl.TlsCiphers.excludeWeak; + +/** + * @author jonmv + */ +class HttpCluster implements Cluster { + + private final List<Endpoint> endpoints = new ArrayList<>(); + + public HttpCluster(FeedClientBuilder builder) throws IOException { + for (URI endpoint : builder.endpoints) + for (int i = 0; i < builder.connectionsPerEndpoint; i++) + endpoints.add(new Endpoint(createHttpClient(builder), endpoint)); + } + + @Override + public void dispatch(SimpleHttpRequest request, CompletableFuture<SimpleHttpResponse> vessel) { + int index = 0; + int min = Integer.MAX_VALUE; + for (int i = 0; i < endpoints.size(); i++) + if (endpoints.get(i).inflight.get() < min) { + index = i; + min = endpoints.get(i).inflight.get(); + } + + Endpoint endpoint = endpoints.get(index); + endpoint.inflight.incrementAndGet(); + try { + request.setScheme(endpoint.url.getScheme()); + request.setAuthority(new URIAuthority(endpoint.url.getHost(), endpoint.url.getPort())); + endpoint.client.execute(request, + new FutureCallback<SimpleHttpResponse>() { + @Override public void completed(SimpleHttpResponse response) { vessel.complete(response); } + @Override public void failed(Exception ex) { vessel.completeExceptionally(ex); } + @Override public void cancelled() { vessel.cancel(false); } + }); + } + catch (Throwable thrown) { + vessel.completeExceptionally(thrown); + } + vessel.whenComplete((__, ___) -> endpoint.inflight.decrementAndGet()); + } + + @Override + public void close() { + Throwable thrown = null; + for (Endpoint endpoint : endpoints) + try { + endpoint.client.close(); + } + catch (Throwable t) { + if (thrown == null) thrown = t; + else thrown.addSuppressed(t); + } + if (thrown != null) throw new RuntimeException(thrown); + } + + + private static class Endpoint { + + private final CloseableHttpAsyncClient client; + private final AtomicInteger inflight = new AtomicInteger(0); + private final URI url; + + private Endpoint(CloseableHttpAsyncClient client, URI url) { + this.client = client; + this.url = url; + + this.client.start(); + } + + } + + private static CloseableHttpAsyncClient createHttpClient(FeedClientBuilder builder) throws IOException { + H2AsyncClientBuilder httpClientBuilder = H2AsyncClientBuilder.create() + .setUserAgent(String.format("vespa-feed-client/%s", Vespa.VERSION)) + .setDefaultHeaders(Collections.singletonList(new BasicHeader("Vespa-Client-Version", Vespa.VERSION))) + .disableCookieManagement() + .disableRedirectHandling() + .disableAutomaticRetries() + .setIOReactorConfig(IOReactorConfig.custom() + .setIoThreadCount(1) + .setTcpNoDelay(true) + .setSoTimeout(Timeout.ofSeconds(10)) + .build()) + .setDefaultRequestConfig( + RequestConfig.custom() + .setConnectTimeout(Timeout.ofSeconds(10)) + .setConnectionRequestTimeout(Timeout.DISABLED) + .setResponseTimeout(Timeout.ofMinutes(5)) + .build()) + .setH2Config(H2Config.initial() + .setMaxConcurrentStreams(builder.maxStreamsPerConnection) + .setCompressionEnabled(true) + .setPushEnabled(false) + .build()); + + SSLContext sslContext = constructSslContext(builder); + String[] allowedCiphers = excludeH2Blacklisted(excludeWeak(sslContext.getSupportedSSLParameters().getCipherSuites())); + if (allowedCiphers.length == 0) + throw new IllegalStateException("No adequate SSL cipher suites supported by the JVM"); + + ClientTlsStrategyBuilder tlsStrategyBuilder = ClientTlsStrategyBuilder.create() + .setCiphers(allowedCiphers) + .setSslContext(sslContext); + if (builder.hostnameVerifier != null) { + tlsStrategyBuilder.setHostnameVerifier(builder.hostnameVerifier); + } + return httpClientBuilder.setTlsStrategy(tlsStrategyBuilder.build()) + .build(); + } + + private static SSLContext constructSslContext(FeedClientBuilder builder) throws IOException { + if (builder.sslContext != null) return builder.sslContext; + SslContextBuilder sslContextBuilder = new SslContextBuilder(); + if (builder.certificate != null && builder.privateKey != null) { + sslContextBuilder.withCertificateAndKey(builder.certificate, builder.privateKey); + } + if (builder.caCertificates != null) { + sslContextBuilder.withCaCertificates(builder.caCertificates); + } + return sslContextBuilder.build(); + } + +} diff --git a/vespa-feed-client/src/main/java/ai/vespa/feed/client/HttpFeedClient.java b/vespa-feed-client/src/main/java/ai/vespa/feed/client/HttpFeedClient.java index e79f8dac67e..9b89595db25 100644 --- a/vespa-feed-client/src/main/java/ai/vespa/feed/client/HttpFeedClient.java +++ b/vespa-feed-client/src/main/java/ai/vespa/feed/client/HttpFeedClient.java @@ -1,34 +1,26 @@ // Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package ai.vespa.feed.client; +import com.fasterxml.jackson.core.JsonFactory; +import com.fasterxml.jackson.core.JsonParser; +import com.fasterxml.jackson.core.JsonToken; import org.apache.hc.client5.http.async.methods.SimpleHttpRequest; import org.apache.hc.client5.http.async.methods.SimpleHttpResponse; -import org.apache.hc.client5.http.config.RequestConfig; -import org.apache.hc.client5.http.impl.async.CloseableHttpAsyncClient; -import org.apache.hc.client5.http.impl.async.H2AsyncClientBuilder; -import org.apache.hc.client5.http.ssl.ClientTlsStrategyBuilder; -import org.apache.hc.core5.concurrent.FutureCallback; import org.apache.hc.core5.http.ContentType; -import org.apache.hc.core5.http.message.BasicHeader; -import org.apache.hc.core5.http2.config.H2Config; import org.apache.hc.core5.net.URIBuilder; -import org.apache.hc.core5.reactor.IOReactorConfig; -import org.apache.hc.core5.util.Timeout; -import javax.net.ssl.SSLContext; import java.io.ByteArrayOutputStream; import java.io.IOException; import java.io.PrintStream; +import java.io.UncheckedIOException; import java.net.URI; import java.net.URISyntaxException; import java.util.ArrayList; -import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.concurrent.CompletableFuture; import java.util.concurrent.atomic.AtomicBoolean; -import java.util.concurrent.atomic.AtomicInteger; import java.util.function.Supplier; import static java.util.Objects.requireNonNull; @@ -41,67 +33,19 @@ import static java.util.Objects.requireNonNull; */ class HttpFeedClient implements FeedClient { - private final URI endpoint; + private static final JsonFactory factory = new JsonFactory(); + private final Map<String, Supplier<String>> requestHeaders; - private final HttpRequestStrategy requestStrategy; - private final List<CloseableHttpAsyncClient> httpClients = new ArrayList<>(); - private final List<AtomicInteger> inflight = new ArrayList<>(); + private final RequestStrategy requestStrategy; private final AtomicBoolean closed = new AtomicBoolean(); HttpFeedClient(FeedClientBuilder builder) throws IOException { - this.endpoint = builder.endpoint; - this.requestHeaders = new HashMap<>(builder.requestHeaders); - this.requestStrategy = new HttpRequestStrategy(builder); - - for (int i = 0; i < builder.maxConnections; i++) { - CloseableHttpAsyncClient hc = createHttpClient(builder, requestStrategy); - hc.start(); - httpClients.add(hc); - inflight.add(new AtomicInteger()); - } - } - - private static CloseableHttpAsyncClient createHttpClient(FeedClientBuilder builder, HttpRequestStrategy retryStrategy) throws IOException { - H2AsyncClientBuilder httpClientBuilder = H2AsyncClientBuilder.create() - .setUserAgent(String.format("vespa-feed-client/%s", Vespa.VERSION)) - .setDefaultHeaders(Collections.singletonList(new BasicHeader("Vespa-Client-Version", Vespa.VERSION))) - .disableCookieManagement() - .disableRedirectHandling() - .setRetryStrategy(retryStrategy) - .setIOReactorConfig(IOReactorConfig.custom() - .setSoTimeout(Timeout.ofSeconds(10)) - .build()) - .setDefaultRequestConfig( - RequestConfig.custom() - .setConnectTimeout(Timeout.ofSeconds(10)) - .setConnectionRequestTimeout(Timeout.DISABLED) - .setResponseTimeout(Timeout.ofMinutes(5)) - .build()) - .setH2Config(H2Config.initial() - .setMaxConcurrentStreams(builder.maxStreamsPerConnection) - .setCompressionEnabled(true) - .setPushEnabled(false) - .build()); - - ClientTlsStrategyBuilder tlsStrategyBuilder = ClientTlsStrategyBuilder.create() - .setSslContext(constructSslContext(builder)); - if (builder.hostnameVerifier != null) { - tlsStrategyBuilder.setHostnameVerifier(builder.hostnameVerifier); - } - return httpClientBuilder.setTlsStrategy(tlsStrategyBuilder.build()) - .build(); + this(builder, new HttpRequestStrategy(builder)); } - private static SSLContext constructSslContext(FeedClientBuilder builder) throws IOException { - if (builder.sslContext != null) return builder.sslContext; - SslContextBuilder sslContextBuilder = new SslContextBuilder(); - if (builder.certificate != null && builder.privateKey != null) { - sslContextBuilder.withCertificateAndKey(builder.certificate, builder.privateKey); - } - if (builder.caCertificates != null) { - sslContextBuilder.withCaCertificates(builder.caCertificates); - } - return sslContextBuilder.build(); + HttpFeedClient(FeedClientBuilder builder, RequestStrategy requestStrategy) { + this.requestHeaders = new HashMap<>(builder.requestHeaders); + this.requestStrategy = requestStrategy; } @Override @@ -120,49 +64,41 @@ class HttpFeedClient implements FeedClient { } @Override - public void close() throws IOException { - if ( ! closed.getAndSet(true)) - for (CloseableHttpAsyncClient hc : httpClients) - hc.close(); + public void close(boolean graceful) { + closed.set(true); + if (graceful) + requestStrategy.await(); + + requestStrategy.destroy(); + } + + private void ensureOpen() { + if (requestStrategy.hasFailed()) + close(); + + if (closed.get()) + throw new IllegalStateException("Client is closed, no further operations may be sent"); } private CompletableFuture<Result> send(String method, DocumentId documentId, String operationJson, OperationParameters params) { - SimpleHttpRequest request = new SimpleHttpRequest(method, operationUrl(endpoint, documentId, params)); + ensureOpen(); + + String path = operationPath(documentId, params).toString(); + SimpleHttpRequest request = new SimpleHttpRequest(method, path); requestHeaders.forEach((name, value) -> request.setHeader(name, value.get())); if (operationJson != null) request.setBody(operationJson, ContentType.APPLICATION_JSON); - int index = 0; - int min = Integer.MAX_VALUE; - for (int i = 0; i < httpClients.size(); i++) - if (inflight.get(i).get() < min) { - min = inflight.get(i).get(); - index = i; - } - - CloseableHttpAsyncClient client = httpClients.get(index); - AtomicInteger counter = inflight.get(index); - counter.incrementAndGet(); - return requestStrategy.enqueue(documentId, future -> { - client.execute(request, - new FutureCallback<SimpleHttpResponse>() { - @Override public void completed(SimpleHttpResponse response) { future.complete(response); } - @Override public void failed(Exception ex) { future.completeExceptionally(ex); } - @Override public void cancelled() { future.cancel(false); } - }); - }).handle((response, thrown) -> { - counter.decrementAndGet(); - if (thrown != null) { - if (requestStrategy.hasFailed()) { - try { close(); } - catch (IOException exception) { thrown.addSuppressed(exception); } - } - ByteArrayOutputStream buffer = new ByteArrayOutputStream(); - thrown.printStackTrace(new PrintStream(buffer)); - return new Result(Result.Type.failure, documentId, buffer.toString(), null); - } - return toResult(response, documentId); - }); + return requestStrategy.enqueue(documentId, request) + .handle((response, thrown) -> { + if (thrown != null) { + // TODO: What to do with exceptions here? Ex on 400, 401, 403, etc, and wrap and throw? + ByteArrayOutputStream buffer = new ByteArrayOutputStream(); + thrown.printStackTrace(new PrintStream(buffer)); + return new Result(Result.Type.failure, documentId, buffer.toString(), null); + } + return toResult(response, documentId); + }); } static Result toResult(SimpleHttpResponse response, DocumentId documentId) { @@ -172,8 +108,31 @@ class HttpFeedClient implements FeedClient { case 412: type = Result.Type.conditionNotMet; break; default: type = Result.Type.failure; } - Map<String, String> responseJson = null; // TODO: parse JSON. - return new Result(type, documentId, response.getBodyText(), "trace"); + + String message = null; + String trace = null; + try { + JsonParser parser = factory.createParser(response.getBodyText()); + if (parser.nextToken() != JsonToken.START_OBJECT) + throw new IllegalArgumentException("Expected '" + JsonToken.START_OBJECT + "', but found '" + parser.currentToken() + "' in: " + response.getBodyText()); + + String name; + while ((name = parser.nextFieldName()) != null) { + switch (name) { + case "message": message = parser.nextTextValue(); break; + case "trace": trace = parser.nextTextValue(); break; + default: parser.nextToken(); + } + } + + if (parser.currentToken() != JsonToken.END_OBJECT) + throw new IllegalArgumentException("Expected '" + JsonToken.END_OBJECT + "', but found '" + parser.currentToken() + "' in: " + response.getBodyText()); + } + catch (IOException e) { + throw new UncheckedIOException(e); + } + + return new Result(type, documentId, message, trace); } static List<String> toPath(DocumentId documentId) { @@ -198,8 +157,8 @@ class HttpFeedClient implements FeedClient { return path; } - static URI operationUrl(URI endpoint, DocumentId documentId, OperationParameters params) { - URIBuilder url = new URIBuilder(endpoint); + static URI operationPath(DocumentId documentId, OperationParameters params) { + URIBuilder url = new URIBuilder(); url.setPathSegments(toPath(documentId)); if (params.createIfNonExistent()) url.addParameter("create", "true"); diff --git a/vespa-feed-client/src/main/java/ai/vespa/feed/client/HttpRequestStrategy.java b/vespa-feed-client/src/main/java/ai/vespa/feed/client/HttpRequestStrategy.java index 0512d6a64c9..7a6e2120be6 100644 --- a/vespa-feed-client/src/main/java/ai/vespa/feed/client/HttpRequestStrategy.java +++ b/vespa-feed-client/src/main/java/ai/vespa/feed/client/HttpRequestStrategy.java @@ -1,156 +1,248 @@ // Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package ai.vespa.feed.client; -import org.apache.hc.client5.http.HttpRequestRetryStrategy; +import ai.vespa.feed.client.FeedClient.CircuitBreaker; +import ai.vespa.feed.client.FeedClient.RetryStrategy; +import org.apache.hc.client5.http.async.methods.SimpleHttpRequest; import org.apache.hc.client5.http.async.methods.SimpleHttpResponse; -import org.apache.hc.core5.http.HttpRequest; -import org.apache.hc.core5.http.HttpResponse; -import org.apache.hc.core5.http.protocol.HttpContext; -import org.apache.hc.core5.util.TimeValue; import java.io.IOException; import java.util.Map; +import java.util.Queue; import java.util.concurrent.CompletableFuture; import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.atomic.AtomicReference; -import java.util.concurrent.locks.Condition; -import java.util.concurrent.locks.Lock; -import java.util.concurrent.locks.ReentrantLock; -import java.util.function.Consumer; - +import java.util.concurrent.ConcurrentLinkedQueue; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicLong; +import java.util.logging.Logger; + +import static ai.vespa.feed.client.FeedClient.CircuitBreaker.State.CLOSED; +import static ai.vespa.feed.client.FeedClient.CircuitBreaker.State.HALF_OPEN; +import static ai.vespa.feed.client.FeedClient.CircuitBreaker.State.OPEN; +import static java.lang.Math.max; +import static java.lang.Math.min; +import static java.util.logging.Level.FINE; +import static java.util.logging.Level.WARNING; + +// TODO: update doc /** * Controls request execution and retries: * <ul> - * <li>Retry all IO exceptions; however</li> - * <li>abort everything if more than 10% of requests result in an exception for some time.</li> - * <li>Whenever throttled, limit inflight to 99% of current; and</li> - * <li>on every successful response, increase inflight limit by 0.1.</li> + * <li>Whenever throttled (429, 503), set target inflight to 0.9 * current, and retry over a different connection;</li> + * <li>retry other transient errors (500, 502 and IOException) a specified number of times, for specified operation types;</li> + * <li>and on every successful response, increase target inflight by 0.1.</li> * </ul> * * @author jonmv */ -class HttpRequestStrategy implements RequestStrategy<SimpleHttpResponse>, HttpRequestRetryStrategy { +class HttpRequestStrategy implements RequestStrategy { + + private static final Logger log = Logger.getLogger(HttpRequestStrategy.class.getName()); - private final Map<DocumentId, CompletableFuture<SimpleHttpResponse>> byId = new ConcurrentHashMap<>(); - private final FeedClient.RetryStrategy wrapped; + private final Cluster cluster; + private final Map<DocumentId, CompletableFuture<?>> inflightById = new ConcurrentHashMap<>(); + private final RetryStrategy strategy; + private final CircuitBreaker breaker; + private final Queue<Runnable> queue = new ConcurrentLinkedQueue<>(); private final long maxInflight; - private double targetInflight; - private long inflight; - private final AtomicReference<Double> errorRate; - private final double errorThreshold; - private final Lock lock; - private final Condition available; + private final long minInflight; + private final AtomicLong targetInflightX10; // 10x target, so we can increment one every tenth success. + private final AtomicLong inflight = new AtomicLong(0); + private final AtomicBoolean destroyed = new AtomicBoolean(false); + private final AtomicLong delayedCount = new AtomicLong(0); + private final AtomicLong retries = new AtomicLong(0); + + HttpRequestStrategy(FeedClientBuilder builder) throws IOException { + this(builder, new HttpCluster(builder)); + } + + HttpRequestStrategy(FeedClientBuilder builder, Cluster cluster) { + this.cluster = cluster; + this.strategy = builder.retryStrategy; + this.breaker = builder.circuitBreaker; + this.maxInflight = builder.connectionsPerEndpoint * (long) builder.maxStreamsPerConnection; + this.minInflight = builder.connectionsPerEndpoint * (long) min(16, builder.maxStreamsPerConnection); + this.targetInflightX10 = new AtomicLong(10 * (long) (Math.sqrt(minInflight) * Math.sqrt(maxInflight))); + new Thread(this::dispatch, "feed-client-dispatcher").start(); + } + + private void dispatch() { + try { + while (breaker.state() != OPEN) { + while ( ! isInExcess() && poll() && breaker.state() == CLOSED); + // Sleep when circuit is half-open, nap when queue is empty, or we are throttled. + Thread.sleep(breaker.state() == HALF_OPEN ? 1000 : 10); + } + } + catch (InterruptedException e) { + Thread.currentThread().interrupt(); + log.log(WARNING, "Dispatch thread interrupted; shutting down"); + } + destroy(); + } - HttpRequestStrategy(FeedClientBuilder builder) { - this.wrapped = builder.retryStrategy; - this.maxInflight = builder.maxConnections * (long) builder.maxStreamsPerConnection; - this.targetInflight = maxInflight; - this.inflight = 0; - this.errorRate = new AtomicReference<>(0.0); - this.errorThreshold = 0.1; - this.lock = new ReentrantLock(true); - this.available = lock.newCondition(); + private void offer(Runnable task) { + delayedCount.incrementAndGet(); + queue.offer(task); } - private double cycle() { - return targetInflight; // TODO: tune this--could start way too high if limit is set too high. + private boolean poll() { + Runnable task = queue.poll(); + if (task == null) return false; + delayedCount.decrementAndGet(); + task.run(); + return true; } - @Override - public boolean retryRequest(HttpRequest request, IOException exception, int execCount, HttpContext context) { - if (errorRate.updateAndGet(rate -> rate + (1 - rate) / cycle()) > errorThreshold) - return false; + private boolean isInExcess() { + return inflight.get() - delayedCount.get() > targetInflight(); + } - if (execCount > wrapped.retries()) + private boolean retry(SimpleHttpRequest request, int attempt) { + if (attempt >= strategy.retries()) return false; switch (request.getMethod().toUpperCase()) { - case "POST": return wrapped.retry(FeedClient.OperationType.put); - case "PUT": return wrapped.retry(FeedClient.OperationType.update); - case "DELETE": return wrapped.retry(FeedClient.OperationType.remove); + case "POST": return strategy.retry(FeedClient.OperationType.PUT); + case "PUT": return strategy.retry(FeedClient.OperationType.UPDATE); + case "DELETE": return strategy.retry(FeedClient.OperationType.REMOVE); default: throw new IllegalStateException("Unexpected HTTP method: " + request.getMethod()); } } /** - * Called when a response is successfully obtained. + * Retries all IOExceptions, unless error rate has converged to a value higher than the threshold, + * or the user has turned off retries for this type of operation. */ - void success() { - errorRate.updateAndGet(rate -> rate - rate / cycle()); - lock.lock(); - targetInflight = Math.min(targetInflight + 0.1, maxInflight); - lock.unlock(); + private boolean retry(SimpleHttpRequest request, Throwable thrown, int attempt) { + breaker.failure(); + log.log(FINE, thrown, () -> "Failed attempt " + attempt + " at " + request); + + if ( ! (thrown instanceof IOException)) + return false; + + return retry(request, attempt); } - @Override - public boolean retryRequest(HttpResponse response, int execCount, HttpContext context) { - if (response.getCode() == 429 || response.getCode() == 503) { - lock.lock(); - targetInflight = Math.max(100, 99 * inflight / 100); - lock.unlock(); - return true; - } - return false; + private void incrementTargetInflight() { + targetInflightX10.incrementAndGet(); } - @Override - public TimeValue getRetryInterval(HttpResponse response, int execCount, HttpContext context) { - return TimeValue.ofMilliseconds(100); + private void decreaseTargetInflight() { + targetInflightX10.set(max((inflight.get() - delayedCount.get()) * 9, minInflight * 10)); } - void acquireSlot() { - lock.lock(); - try { - while (inflight >= targetInflight) - available.awaitUninterruptibly(); + private long targetInflight() { + return min(targetInflightX10.get() / 10, maxInflight); + } - ++inflight; + /** Retries throttled requests (429, 503), adjusting the target inflight count, and server errors (500, 502). */ + private boolean retry(SimpleHttpRequest request, SimpleHttpResponse response, int attempt) { + if (response.getCode() / 100 == 2) { + breaker.success(); + incrementTargetInflight(); + return false; } - finally { - lock.unlock(); + + log.log(FINE, () -> "Status code " + response.getCode() + " (" + response.getBodyText() + + ") on attempt " + attempt + " at " + request); + + if (response.getCode() == 429 || response.getCode() == 503) { // Throttling; reduce target inflight. + decreaseTargetInflight(); + return true; } + + breaker.failure(); + if (response.getCode() == 500 || response.getCode() == 502 || response.getCode() == 504) // Hopefully temporary errors. + return retry(request, attempt); + + return false; } - void releaseSlot() { - lock.lock(); + private void acquireSlot() { try { - --inflight; + while (inflight.get() >= targetInflight()) + Thread.sleep(1); - if (inflight < targetInflight) - available.signal(); + inflight.incrementAndGet(); } - finally { - lock.unlock(); + catch (InterruptedException e) { + throw new RuntimeException(e); } } + private void releaseSlot() { + inflight.decrementAndGet(); + } + @Override public boolean hasFailed() { - return errorRate.get() > errorThreshold; + return breaker.state() == OPEN; + } + + public void await() { + try { + while (inflight.get() > 0) + Thread.sleep(10); + } + catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } } @Override - public CompletableFuture<SimpleHttpResponse> enqueue(DocumentId documentId, Consumer<CompletableFuture<SimpleHttpResponse>> dispatch) { - acquireSlot(); - - Consumer<CompletableFuture<SimpleHttpResponse>> safeDispatch = vessel -> { - try { dispatch.accept(vessel); } - catch (Throwable t) { vessel.completeExceptionally(t); } - }; - CompletableFuture<SimpleHttpResponse> vessel = new CompletableFuture<>(); - byId.compute(documentId, (id, previous) -> { - if (previous == null) safeDispatch.accept(vessel); - else previous.whenComplete((__, ___) -> safeDispatch.accept(vessel)); - return vessel; + public CompletableFuture<SimpleHttpResponse> enqueue(DocumentId documentId, SimpleHttpRequest request) { + CompletableFuture<SimpleHttpResponse> result = new CompletableFuture<>(); // Carries the aggregate result of the operation, including retries. + CompletableFuture<SimpleHttpResponse> vessel = new CompletableFuture<>(); // Holds the computation of a single dispatch to the HTTP client. + CompletableFuture<?> previous = inflightById.put(documentId, result); + if (destroyed.get()) { + result.cancel(true); + return result; + } + + if (previous == null) { + acquireSlot(); + offer(() -> cluster.dispatch(request, vessel)); + } + else + previous.whenComplete((__, ___) -> offer(() -> cluster.dispatch(request, vessel))); + + handleAttempt(vessel, request, result, 1); + + result.whenComplete((__, ___) -> { + if (inflightById.compute(documentId, (____, current) -> current == result ? null : current) == null) + releaseSlot(); }); - return vessel.whenComplete((__, thrown) -> { - releaseSlot(); - if (thrown == null) - success(); + return result; + } - byId.compute(documentId, (id, current) -> current == vessel ? null : current); + /** Handles the result of one attempt at the given operation, retrying if necessary. */ + private void handleAttempt(CompletableFuture<SimpleHttpResponse> vessel, SimpleHttpRequest request, CompletableFuture<SimpleHttpResponse> result, int attempt) { + vessel.whenComplete((response, thrown) -> { + // Retry the operation if it failed with a transient error ... + if (thrown != null ? retry(request, thrown, attempt) + : retry(request, response, attempt)) { + retries.incrementAndGet(); + CircuitBreaker.State state = breaker.state(); + CompletableFuture<SimpleHttpResponse> retry = new CompletableFuture<>(); + offer(() -> cluster.dispatch(request, retry)); + handleAttempt(retry, request, result, attempt + (state == HALF_OPEN ? 0 : 1)); + } + // ... or accept the outcome and mark the operation as complete. + else { + if (thrown == null) result.complete(response); + else result.completeExceptionally(thrown); + } }); } + @Override + public void destroy() { + if ( ! destroyed.getAndSet(true)) + inflightById.values().forEach(result -> result.cancel(true)); + + cluster.close(); + } + } diff --git a/vespa-feed-client/src/main/java/ai/vespa/feed/client/JsonStreamFeeder.java b/vespa-feed-client/src/main/java/ai/vespa/feed/client/JsonStreamFeeder.java index 17162f19d3f..99d05a4bae8 100644 --- a/vespa-feed-client/src/main/java/ai/vespa/feed/client/JsonStreamFeeder.java +++ b/vespa-feed-client/src/main/java/ai/vespa/feed/client/JsonStreamFeeder.java @@ -17,9 +17,9 @@ import java.util.concurrent.CompletableFuture; import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicReference; -import static ai.vespa.feed.client.FeedClient.OperationType.put; -import static ai.vespa.feed.client.FeedClient.OperationType.remove; -import static ai.vespa.feed.client.FeedClient.OperationType.update; +import static ai.vespa.feed.client.FeedClient.OperationType.PUT; +import static ai.vespa.feed.client.FeedClient.OperationType.REMOVE; +import static ai.vespa.feed.client.FeedClient.OperationType.UPDATE; import static com.fasterxml.jackson.core.JsonToken.START_OBJECT; import static com.fasterxml.jackson.core.JsonToken.VALUE_FALSE; import static com.fasterxml.jackson.core.JsonToken.VALUE_STRING; @@ -187,9 +187,9 @@ public class JsonStreamFeeder implements Closeable { case FIELD_NAME: switch (parser.getText()) { case "id": - case "put": type = put; id = readId(); break; - case "update": type = update; id = readId(); break; - case "remove": type = remove; id = readId(); break; + case "put": type = PUT; id = readId(); break; + case "update": type = UPDATE; id = readId(); break; + case "remove": type = REMOVE; id = readId(); break; case "condition": parameters = parameters.testAndSetCondition(readString()); break; case "create": parameters = parameters.createIfNonExistent(readBoolean()); break; case "fields": { @@ -230,9 +230,9 @@ public class JsonStreamFeeder implements Closeable { } switch (type) { - case put: return client.put (id, payload, parameters); - case update: return client.update(id, payload, parameters); - case remove: return client.remove(id, parameters); + case PUT: return client.put (id, payload, parameters); + case UPDATE: return client.update(id, payload, parameters); + case REMOVE: return client.remove(id, parameters); default: throw new IllegalStateException("Unexpected operation type '" + type + "'"); } } diff --git a/vespa-feed-client/src/main/java/ai/vespa/feed/client/RequestStrategy.java b/vespa-feed-client/src/main/java/ai/vespa/feed/client/RequestStrategy.java index e5eb956114e..bda214405b5 100644 --- a/vespa-feed-client/src/main/java/ai/vespa/feed/client/RequestStrategy.java +++ b/vespa-feed-client/src/main/java/ai/vespa/feed/client/RequestStrategy.java @@ -1,20 +1,30 @@ // Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package ai.vespa.feed.client; +import org.apache.hc.client5.http.async.methods.SimpleHttpRequest; +import org.apache.hc.client5.http.async.methods.SimpleHttpResponse; + +import java.io.Closeable; import java.util.concurrent.CompletableFuture; -import java.util.function.Consumer; +import java.util.function.BiConsumer; /** * Controls execution of feed operations. * * @author jonmv */ -public interface RequestStrategy<T> { +interface RequestStrategy { - /** Whether this has failed, and we should stop. */ + /** Whether this has failed fatally, and we should cease sending further operations. */ boolean hasFailed(); - /** Enqueue the given operation, which is dispatched to a vessel future when ready. */ - CompletableFuture<T> enqueue(DocumentId documentId, Consumer<CompletableFuture<T>> dispatch); + /** Forcibly terminates this, causing all inflight operations to complete immediately. */ + void destroy(); + + /** Wait for all inflight requests to complete. */ + void await(); + + /** Enqueue the given operation, returning its future result. This may block if the client send queue is full. */ + CompletableFuture<SimpleHttpResponse> enqueue(DocumentId documentId, SimpleHttpRequest request); } diff --git a/vespa-feed-client/src/main/java/ai/vespa/feed/client/SslContextBuilder.java b/vespa-feed-client/src/main/java/ai/vespa/feed/client/SslContextBuilder.java index 1a5f27c5d66..7200d5fd943 100644 --- a/vespa-feed-client/src/main/java/ai/vespa/feed/client/SslContextBuilder.java +++ b/vespa-feed-client/src/main/java/ai/vespa/feed/client/SslContextBuilder.java @@ -34,7 +34,7 @@ import java.util.List; */ class SslContextBuilder { - private static final BouncyCastleProvider bcProvider = new BouncyCastleProvider(); + static final BouncyCastleProvider bcProvider = new BouncyCastleProvider(); private Path certificateFile; private Path privateKeyFile; @@ -54,6 +54,7 @@ class SslContextBuilder { SSLContext build() throws IOException { try { KeyStore keystore = KeyStore.getInstance("PKCS12"); + keystore.load(null); if (certificateFile != null && privateKeyFile != null) { keystore.setKeyEntry("cert", privateKey(privateKeyFile), new char[0], certificates(certificateFile)); } @@ -64,7 +65,7 @@ class SslContextBuilder { kmf.init(keystore, new char[0]); TrustManagerFactory tmf = TrustManagerFactory.getInstance(TrustManagerFactory.getDefaultAlgorithm()); tmf.init(keystore); - SSLContext sslContext = SSLContext.getDefault(); + SSLContext sslContext = SSLContext.getInstance("TLS"); sslContext.init(kmf.getKeyManagers(), tmf.getTrustManagers(), null); return sslContext; } catch (GeneralSecurityException e) { diff --git a/vespa-feed-client/src/test/java/ai/vespa/feed/client/JsonStreamFeederTest.java b/vespa-feed-client/src/test/java/ai/vespa/feed/client/JsonStreamFeederTest.java index 8ef8ae57f5e..28a50b88396 100644 --- a/vespa-feed-client/src/test/java/ai/vespa/feed/client/JsonStreamFeederTest.java +++ b/vespa-feed-client/src/test/java/ai/vespa/feed/client/JsonStreamFeederTest.java @@ -5,20 +5,20 @@ import org.junit.jupiter.api.Test; import java.io.ByteArrayInputStream; import java.io.IOException; +import java.util.HashSet; import java.util.Set; import java.util.concurrent.CompletableFuture; -import java.util.concurrent.ConcurrentSkipListSet; -import java.util.stream.Collectors; import java.util.stream.IntStream; import static java.nio.charset.StandardCharsets.UTF_8; +import static java.util.stream.Collectors.joining; import static org.junit.jupiter.api.Assertions.assertEquals; class JsonStreamFeederTest { @Test void test() throws IOException { - int docs = 1 << 10; + int docs = 1 << 14; String json = "[\n" + IntStream.range(0, docs).mapToObj(i -> @@ -28,7 +28,7 @@ class JsonStreamFeederTest { " \"lul\":\"lal\"\n" + " }\n" + " },\n" - ).collect(Collectors.joining()) + + ).collect(joining()) + " {\n" + " \"id\": \"id:ns:type::abc" + docs + "\",\n" + @@ -38,8 +38,10 @@ class JsonStreamFeederTest { " }\n" + "]"; ByteArrayInputStream in = new ByteArrayInputStream(json.getBytes(UTF_8)); - Set<String> ids = new ConcurrentSkipListSet<>(); + Set<String> ids = new HashSet<>(); + long startNanos = System.nanoTime(); JsonStreamFeeder.builder(new FeedClient() { + @Override public CompletableFuture<Result> put(DocumentId documentId, String documentJson, OperationParameters params) { ids.add(documentId.userSpecific()); @@ -57,10 +59,10 @@ class JsonStreamFeederTest { } @Override - public void close() throws IOException { + public void close(boolean graceful) { } - } - }).build().feed(in, 1 << 7, false); // TODO: hangs on 1 << 6. + }).build().feed(in, 1 << 7, false); // TODO: hangs when buffer is smaller than largest document + System.err.println((json.length() / 1048576.0) + " MB in " + (System.nanoTime() - startNanos) * 1e-9 + " seconds"); assertEquals(docs + 1, ids.size()); } diff --git a/vespa-feed-client/src/test/java/ai/vespa/feed/client/SslContextBuilderTest.java b/vespa-feed-client/src/test/java/ai/vespa/feed/client/SslContextBuilderTest.java new file mode 100644 index 00000000000..e930c602ec7 --- /dev/null +++ b/vespa-feed-client/src/test/java/ai/vespa/feed/client/SslContextBuilderTest.java @@ -0,0 +1,88 @@ +package ai.vespa.feed.client;// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +import org.bouncycastle.cert.jcajce.JcaX509CertificateConverter; +import org.bouncycastle.cert.jcajce.JcaX509v3CertificateBuilder; +import org.bouncycastle.openssl.jcajce.JcaPEMWriter; +import org.bouncycastle.operator.ContentSigner; +import org.bouncycastle.operator.OperatorCreationException; +import org.bouncycastle.operator.jcajce.JcaContentSignerBuilder; +import org.bouncycastle.util.io.pem.PemObject; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +import javax.net.ssl.SSLContext; +import javax.security.auth.x500.X500Principal; +import java.io.BufferedWriter; +import java.io.IOException; +import java.math.BigInteger; +import java.nio.file.Files; +import java.nio.file.Path; +import java.security.GeneralSecurityException; +import java.security.KeyPair; +import java.security.KeyPairGenerator; +import java.security.cert.CertificateException; +import java.security.cert.X509Certificate; +import java.security.spec.ECGenParameterSpec; +import java.time.Instant; +import java.time.temporal.ChronoUnit; +import java.util.Date; + +import static org.junit.jupiter.api.Assertions.assertDoesNotThrow; +import static org.junit.jupiter.api.Assertions.assertEquals; + +/** + * @author bjorncs + */ +class SslContextBuilderTest { + + private static Path certificateFile; + private static Path privateKeyFile; + + @BeforeAll + static void createPemFiles(@TempDir Path tempDirectory) throws GeneralSecurityException, OperatorCreationException, IOException { + KeyPair keypair = createKeypair(); + X509Certificate certificate = createCertificate(keypair); + certificateFile = tempDirectory.resolve("cert.pem"); + privateKeyFile = tempDirectory.resolve("key.pem"); + writePem(certificateFile, "CERTIFICATE", certificate.getEncoded()); + writePem(privateKeyFile, "PRIVATE KEY", keypair.getPrivate().getEncoded()); + } + + @Test + void successfully_constructs_sslcontext_from_pem_files() { + SSLContext sslContext = assertDoesNotThrow(() -> + new SslContextBuilder() + .withCaCertificates(certificateFile) + .withCertificateAndKey(certificateFile, privateKeyFile) + .build()); + assertEquals("TLS", sslContext.getProtocol()); + } + + private static void writePem(Path file, String type, byte[] asn1DerEncodedObject) throws IOException { + try (BufferedWriter fileWriter = Files.newBufferedWriter(file); + JcaPEMWriter pemWriter = new JcaPEMWriter(fileWriter)) { + pemWriter.writeObject(new PemObject(type, asn1DerEncodedObject)); + pemWriter.flush(); + } + } + + private static KeyPair createKeypair() throws GeneralSecurityException { + KeyPairGenerator generator = KeyPairGenerator.getInstance("EC", SslContextBuilder.bcProvider); + generator.initialize(new ECGenParameterSpec("prime256v1")); + return generator.generateKeyPair(); + } + + private static X509Certificate createCertificate(KeyPair keypair) throws OperatorCreationException, CertificateException { + JcaX509v3CertificateBuilder jcaCertBuilder = new JcaX509v3CertificateBuilder( + new X500Principal("CN=localhost"), BigInteger.ONE, Date.from(Instant.EPOCH), + Date.from(Instant.EPOCH.plus(100_000, ChronoUnit.DAYS)), new X500Principal("CN=localhost"), keypair.getPublic()); + ContentSigner contentSigner = new JcaContentSignerBuilder("SHA256withECDSA") + .setProvider(SslContextBuilder.bcProvider) + .build(keypair.getPrivate()); + return new JcaX509CertificateConverter() + .setProvider(SslContextBuilder.bcProvider) + .getCertificate(jcaCertBuilder.build(contentSigner)); + } + +}
\ No newline at end of file diff --git a/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/communication/ApacheGatewayConnection.java b/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/communication/ApacheGatewayConnection.java index 547ea524041..68cca286dac 100644 --- a/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/communication/ApacheGatewayConnection.java +++ b/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/communication/ApacheGatewayConnection.java @@ -8,11 +8,11 @@ import com.yahoo.security.SslContextBuilder; import com.yahoo.vespa.http.client.config.ConnectionParams; import com.yahoo.vespa.http.client.config.Endpoint; import com.yahoo.vespa.http.client.config.FeedParams; -import com.yahoo.vespa.http.client.core.Vtag; import com.yahoo.vespa.http.client.core.Document; import com.yahoo.vespa.http.client.core.Encoder; import com.yahoo.vespa.http.client.core.Headers; import com.yahoo.vespa.http.client.core.ServerResponseException; +import com.yahoo.vespa.http.client.core.Vtag; import org.apache.http.Header; import org.apache.http.HttpHost; import org.apache.http.HttpResponse; @@ -392,10 +392,12 @@ class ApacheGatewayConnection implements GatewayConnection { */ public static class HttpClientFactory { + private final FeedParams feedParams; final ConnectionParams connectionParams; final boolean useSsl; - public HttpClientFactory(ConnectionParams connectionParams, boolean useSsl) { + public HttpClientFactory(FeedParams feedParams, ConnectionParams connectionParams, boolean useSsl) { + this.feedParams = feedParams; this.connectionParams = connectionParams; this.useSsl = useSsl; } @@ -427,8 +429,10 @@ class ApacheGatewayConnection implements GatewayConnection { clientBuilder.setMaxConnTotal(1); clientBuilder.setUserAgent(String.format("vespa-http-client (%s)", Vtag.V_TAG_COMPONENT)); clientBuilder.setDefaultHeaders(Collections.singletonList(new BasicHeader(Headers.CLIENT_VERSION, Vtag.V_TAG_COMPONENT))); - RequestConfig.Builder requestConfigBuilder = RequestConfig.custom(); - requestConfigBuilder.setSocketTimeout(0); + int millisTotalTimeout = (int) (feedParams.getClientTimeout(TimeUnit.MILLISECONDS) + feedParams.getServerTimeout(TimeUnit.MILLISECONDS)); + RequestConfig.Builder requestConfigBuilder = RequestConfig.custom() + .setSocketTimeout(millisTotalTimeout) + .setConnectTimeout(millisTotalTimeout); if (connectionParams.getProxyHost() != null) { requestConfigBuilder.setProxy(new HttpHost(connectionParams.getProxyHost(), connectionParams.getProxyPort())); } diff --git a/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/communication/ClusterConnection.java b/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/communication/ClusterConnection.java index 16afd001c46..9dc214fb93d 100644 --- a/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/communication/ClusterConnection.java +++ b/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/communication/ClusterConnection.java @@ -78,7 +78,7 @@ public class ClusterConnection implements AutoCloseable { feedParams, cluster.getRoute(), connectionParams, - new ApacheGatewayConnection.HttpClientFactory(connectionParams, endpoint.isUseSsl()), + new ApacheGatewayConnection.HttpClientFactory(feedParams, connectionParams, endpoint.isUseSsl()), operationProcessor.getClientId(), clock ); diff --git a/vespaclient-container-plugin/src/main/java/com/yahoo/document/restapi/resource/DocumentV1ApiHandler.java b/vespaclient-container-plugin/src/main/java/com/yahoo/document/restapi/resource/DocumentV1ApiHandler.java index 8cb927b8989..58857d1d8e6 100644 --- a/vespaclient-container-plugin/src/main/java/com/yahoo/document/restapi/resource/DocumentV1ApiHandler.java +++ b/vespaclient-container-plugin/src/main/java/com/yahoo/document/restapi/resource/DocumentV1ApiHandler.java @@ -832,6 +832,7 @@ public class DocumentV1ApiHandler extends AbstractRequestHandler { private final ReadableContentChannel delegate = new ReadableContentChannel(); private final Consumer<InputStream> reader; + private volatile boolean errorReported = false; public ForwardingContentChannel(Consumer<InputStream> reader) { this.reader = reader; @@ -854,7 +855,9 @@ public class DocumentV1ApiHandler extends AbstractRequestHandler { public void close(CompletionHandler handler) { try { delegate.close(logException); - reader.accept(new UnsafeContentInputStream(delegate)); + if (!errorReported) { + reader.accept(new UnsafeContentInputStream(delegate)); + } handler.completed(); } catch (Exception e) { @@ -862,6 +865,12 @@ public class DocumentV1ApiHandler extends AbstractRequestHandler { } } + @Override + public void onError(Throwable error) { + // Jdisc will automatically generate an error response in this scenario + log.log(FINE, error, () -> "ContentChannel.onError(): " + error.getMessage()); + errorReported = true; + } } static class DocumentOperationParser { diff --git a/vespajlib/src/main/java/com/yahoo/concurrent/maintenance/JobMetrics.java b/vespajlib/src/main/java/com/yahoo/concurrent/maintenance/JobMetrics.java index fcc5b8e57a2..73a5dc77743 100644 --- a/vespajlib/src/main/java/com/yahoo/concurrent/maintenance/JobMetrics.java +++ b/vespajlib/src/main/java/com/yahoo/concurrent/maintenance/JobMetrics.java @@ -2,25 +2,18 @@ package com.yahoo.concurrent.maintenance; import java.util.concurrent.ConcurrentHashMap; -import java.util.function.BiConsumer; /** * Tracks and forwards maintenance job metrics. * * @author mpolden */ -public class JobMetrics { - - private final BiConsumer<String, Long> metricConsumer; +public abstract class JobMetrics { private final ConcurrentHashMap<String, Long> incompleteRuns = new ConcurrentHashMap<>(); - public JobMetrics(BiConsumer<String, Long> metricConsumer) { - this.metricConsumer = metricConsumer; - } - - /** Record a run for given job */ - public void recordRunOf(String job) { + /** Record starting of a run of a job */ + public void starting(String job) { incompleteRuns.merge(job, 1L, Long::sum); } @@ -29,12 +22,17 @@ public class JobMetrics { incompleteRuns.put(job, 0L); } - /** Forward metrics for given job to metric consumer */ - public void forward(String job) { + /** + * Records completion of a run of a job. + * This is guaranteed to always be called once whenever starting has been called. + */ + public void completed(String job, double successFactor) { Long incompleteRuns = this.incompleteRuns.get(job); if (incompleteRuns != null) { - metricConsumer.accept(job, incompleteRuns); + recordCompletion(job, incompleteRuns, successFactor); } } + protected abstract void recordCompletion(String job, Long incompleteRuns, double successFactor); + } diff --git a/vespajlib/src/main/java/com/yahoo/concurrent/maintenance/Maintainer.java b/vespajlib/src/main/java/com/yahoo/concurrent/maintenance/Maintainer.java index 734c46a2819..2a9e6dda6b6 100644 --- a/vespajlib/src/main/java/com/yahoo/concurrent/maintenance/Maintainer.java +++ b/vespajlib/src/main/java/com/yahoo/concurrent/maintenance/Maintainer.java @@ -83,8 +83,19 @@ public abstract class Maintainer implements Runnable { @Override public final String toString() { return name(); } - /** Called once each time this maintenance job should run. Returns whether the maintenance run was successful */ - protected abstract boolean maintain(); + /** + * Called once each time this maintenance job should run. + * + * @return the degree to which the run was successful - a number between 0 (no success), to 1 (complete success). + * Note that this indicates whether something is wrong, so e.g if the call did nothing because it should do + * nothing, 1.0 should be returned. + */ + protected abstract double maintain(); + + /** Convenience methods to convert attempts and failures into a success factor */ + protected final double asSuccessFactor(int attempts, int failures) { + return attempts == 0 ? 1.0 : 1 - (double)failures / attempts; + } /** Returns the interval at which this job is set to run */ protected Duration interval() { return interval; } @@ -93,9 +104,12 @@ public abstract class Maintainer implements Runnable { public final void lockAndMaintain(boolean force) { if (!force && !jobControl.isActive(name())) return; log.log(Level.FINE, () -> "Running " + this.getClass().getSimpleName()); - jobMetrics.recordRunOf(name()); + jobMetrics.starting(name()); + double successFactor = 0; try (var lock = jobControl.lockJob(name())) { - if (maintain()) jobMetrics.recordCompletionOf(name()); + successFactor = maintain(); + if (successFactor > 0.0) + jobMetrics.recordCompletionOf(name()); } catch (UncheckedTimeoutException e) { if (ignoreCollision) { jobMetrics.recordCompletionOf(name()); @@ -105,7 +119,7 @@ public abstract class Maintainer implements Runnable { } catch (Throwable e) { log.log(Level.WARNING, this + " failed. Will retry in " + interval, e); } finally { - jobMetrics.forward(name()); + jobMetrics.completed(name(), successFactor); } log.log(Level.FINE, () -> "Finished " + this.getClass().getSimpleName()); } diff --git a/vespajlib/src/main/java/com/yahoo/io/IOUtils.java b/vespajlib/src/main/java/com/yahoo/io/IOUtils.java index f2de0ace476..fa32ec1ff9c 100644 --- a/vespajlib/src/main/java/com/yahoo/io/IOUtils.java +++ b/vespajlib/src/main/java/com/yahoo/io/IOUtils.java @@ -12,10 +12,10 @@ import java.nio.ByteBuffer; /** - * <p>Some static io convenience methods.</p> + * Some static io convenience methods. * - * @author bratseth - * @author Bjorn Borud + * @author bratseth + * @author Bjorn Borud */ public abstract class IOUtils { diff --git a/vespajlib/src/test/java/com/yahoo/concurrent/maintenance/JobControlTest.java b/vespajlib/src/test/java/com/yahoo/concurrent/maintenance/JobControlTest.java index 139a2901cd3..01560c050ff 100644 --- a/vespajlib/src/test/java/com/yahoo/concurrent/maintenance/JobControlTest.java +++ b/vespajlib/src/test/java/com/yahoo/concurrent/maintenance/JobControlTest.java @@ -3,6 +3,8 @@ package com.yahoo.concurrent.maintenance; import org.junit.Test; +import java.util.concurrent.atomic.AtomicLong; + import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; @@ -19,9 +21,8 @@ public class JobControlTest { String job1 = "Job1"; String job2 = "Job2"; - JobMetrics metrics = new JobMetrics((job, instant) -> {}); - TestMaintainer maintainer1 = new TestMaintainer(job1, jobControl, metrics); - TestMaintainer maintainer2 = new TestMaintainer(job2, jobControl, metrics); + TestMaintainer maintainer1 = new TestMaintainer(job1, jobControl, new NoopJobMetrics()); + TestMaintainer maintainer2 = new TestMaintainer(job2, jobControl, new NoopJobMetrics()); assertEquals(2, jobControl.jobs().size()); assertTrue(jobControl.jobs().contains(job1)); assertTrue(jobControl.jobs().contains(job2)); @@ -62,7 +63,7 @@ public class JobControlTest { public void testJobControlMayDeactivateJobs() { JobControlStateMock state = new JobControlStateMock(); JobControl jobControl = new JobControl(state); - TestMaintainer mockMaintainer = new TestMaintainer(null, jobControl, new JobMetrics((job, instant) -> {})); + TestMaintainer mockMaintainer = new TestMaintainer(null, jobControl, new NoopJobMetrics()); assertTrue(jobControl.jobs().contains("TestMaintainer")); @@ -80,4 +81,11 @@ public class JobControlTest { assertEquals(2, mockMaintainer.totalRuns()); } + private static class NoopJobMetrics extends JobMetrics { + + @Override + protected void recordCompletion(String job, Long incompleteRuns, double successFactor) { } + + } + } diff --git a/vespajlib/src/test/java/com/yahoo/concurrent/maintenance/MaintainerTest.java b/vespajlib/src/test/java/com/yahoo/concurrent/maintenance/MaintainerTest.java index e881d4b3ff6..d2db380f4a1 100644 --- a/vespajlib/src/test/java/com/yahoo/concurrent/maintenance/MaintainerTest.java +++ b/vespajlib/src/test/java/com/yahoo/concurrent/maintenance/MaintainerTest.java @@ -42,35 +42,45 @@ public class MaintainerTest { @Test public void success_metric() { - AtomicLong consecutiveFailures = new AtomicLong(); - JobMetrics jobMetrics = new JobMetrics((job, count) -> consecutiveFailures.set(count)); + TestJobMetrics jobMetrics = new TestJobMetrics(); TestMaintainer maintainer = new TestMaintainer(null, jobControl, jobMetrics); // Maintainer fails twice in a row maintainer.successOnNextRun(false).run(); - assertEquals(1, consecutiveFailures.get()); + assertEquals(1, jobMetrics.consecutiveFailures.get()); maintainer.successOnNextRun(false).run(); - assertEquals(2, consecutiveFailures.get()); + assertEquals(2, jobMetrics.consecutiveFailures.get()); // Maintainer runs successfully maintainer.successOnNextRun(true).run(); - assertEquals(0, consecutiveFailures.get()); + assertEquals(0, jobMetrics.consecutiveFailures.get()); // Maintainer runs successfully again maintainer.run(); - assertEquals(0, consecutiveFailures.get()); + assertEquals(0, jobMetrics.consecutiveFailures.get()); // Maintainer throws maintainer.throwOnNextRun(new RuntimeException()).run(); - assertEquals(1, consecutiveFailures.get()); + assertEquals(1, jobMetrics.consecutiveFailures.get()); // Maintainer recovers maintainer.throwOnNextRun(null).run(); - assertEquals(0, consecutiveFailures.get()); + assertEquals(0, jobMetrics.consecutiveFailures.get()); // Lock exception is treated as a failure maintainer.throwOnNextRun(new UncheckedTimeoutException()).run(); - assertEquals(1, consecutiveFailures.get()); + assertEquals(1, jobMetrics.consecutiveFailures.get()); + } + + private static class TestJobMetrics extends JobMetrics { + + AtomicLong consecutiveFailures = new AtomicLong(); + + @Override + protected void recordCompletion(String job, Long incompleteRuns, double successFactor) { + consecutiveFailures.set(incompleteRuns); + } + } } diff --git a/vespajlib/src/test/java/com/yahoo/concurrent/maintenance/TestMaintainer.java b/vespajlib/src/test/java/com/yahoo/concurrent/maintenance/TestMaintainer.java index 44a00a37a83..7424b17cab2 100644 --- a/vespajlib/src/test/java/com/yahoo/concurrent/maintenance/TestMaintainer.java +++ b/vespajlib/src/test/java/com/yahoo/concurrent/maintenance/TestMaintainer.java @@ -33,10 +33,10 @@ class TestMaintainer extends Maintainer { } @Override - protected boolean maintain() { + protected double maintain() { if (exceptionToThrow != null) throw exceptionToThrow; totalRuns++; - return success; + return success ? 1.0 : 0.0; } } diff --git a/vespalib/src/tests/stllike/hashtable_test.cpp b/vespalib/src/tests/stllike/hashtable_test.cpp index cbd8b28d9a8..ac364fdf0df 100644 --- a/vespalib/src/tests/stllike/hashtable_test.cpp +++ b/vespalib/src/tests/stllike/hashtable_test.cpp @@ -5,9 +5,9 @@ #include <vespa/vespalib/stllike/hash_fun.h> #include <vespa/vespalib/stllike/identity.h> #include <vespa/vespalib/testkit/testapp.h> +#include <vespa/vespalib/stllike/hash_map.hpp> #include <memory> #include <vector> -#include <vespa/vespalib/stllike/hash_map.h> using vespalib::hashtable; using std::vector; @@ -134,6 +134,79 @@ TEST("require that hashtable<vector<int>> can be copied") { EXPECT_EQUAL(6, (*table.find(2))[2]); } +/** + * Test to profile destruction and recreation of hash map. + * It revealed some unexpected behaviour. Results with 10k iterations on 2018 macbook pro 2.6 Ghz i7 + * 1 - previous - 14.7s hash_node() : _node(), _next(invalid) {} + * 2 - test - 6.6s hash_node() : _next(invalid) { memset(_node, 0, sizeof(node)); } + * 3 - current - 2.3s hash_node() : _next(invalid) {} + */ +TEST("benchmark hash table reconstruction with POD objects") { + vespalib::hash_map<uint32_t, uint32_t> m(1000000); + constexpr size_t NUM_ITER = 10; // Set to 1k-10k to get measurable numbers 10k ~= 2.3s + for (size_t i(0); i < NUM_ITER; i++) { + m[46] = 17; + EXPECT_FALSE(m.empty()); + EXPECT_EQUAL(1u, m.size()); + EXPECT_EQUAL(1048576u, m.capacity()); + m.clear(); + EXPECT_TRUE(m.empty()); + EXPECT_EQUAL(1048576u, m.capacity()); + } +} + +class NonPOD { +public: + NonPOD() noexcept + : _v(rand()) + { + construction_count++; + } + NonPOD(NonPOD && rhs) noexcept { _v = rhs._v; rhs._v = -1; } + NonPOD & operator =(NonPOD && rhs) noexcept { _v = rhs._v; rhs._v = -1; return *this; } + NonPOD(const NonPOD &) = delete; + NonPOD & operator =(const NonPOD &) = delete; + ~NonPOD() { + if (_v != -1) { + destruction_count++; + } + } + int32_t _v; + static size_t construction_count; + static size_t destruction_count; +}; + +size_t NonPOD::construction_count = 0; +size_t NonPOD::destruction_count = 0; + +/** + * Performance is identical for NonPOD objects as with POD object. + * Object are are only constructed on insert, and destructed on erase/clear. + */ +TEST("benchmark hash table reconstruction with non POD objects") { + vespalib::hash_map<uint32_t, NonPOD> m(1000000); + constexpr size_t NUM_ITER = 10; // Set to 1k-10k to get measurable numbers 10k ~= 2.3s + NonPOD::construction_count = 0; + NonPOD::destruction_count = 0; + for (size_t i(0); i < NUM_ITER; i++) { + EXPECT_EQUAL(i, NonPOD::construction_count); + EXPECT_EQUAL(i, NonPOD::destruction_count); + m.insert(std::make_pair(46, NonPOD())); + EXPECT_EQUAL(i+1, NonPOD::construction_count); + EXPECT_EQUAL(i, NonPOD::destruction_count); + EXPECT_FALSE(m.empty()); + EXPECT_EQUAL(1u, m.size()); + EXPECT_EQUAL(1048576u, m.capacity()); + m.clear(); + EXPECT_EQUAL(i+1, NonPOD::construction_count); + EXPECT_EQUAL(i+1, NonPOD::destruction_count); + EXPECT_TRUE(m.empty()); + EXPECT_EQUAL(1048576u, m.capacity()); + } + EXPECT_EQUAL(NUM_ITER, NonPOD::construction_count); + EXPECT_EQUAL(NUM_ITER, NonPOD::destruction_count); +} + } // namespace TEST_MAIN() { TEST_RUN_ALL(); } diff --git a/vespalib/src/vespa/vespalib/data/smart_buffer.cpp b/vespalib/src/vespa/vespalib/data/smart_buffer.cpp index 401b6729601..4c3a3e6f379 100644 --- a/vespalib/src/vespa/vespalib/data/smart_buffer.cpp +++ b/vespalib/src/vespa/vespalib/data/smart_buffer.cpp @@ -23,6 +23,15 @@ SmartBuffer::ensure_free(size_t bytes) _read_pos = 0; } +void +SmartBuffer::drop() +{ + alloc::Alloc empty_buf; + _data.swap(empty_buf); + _read_pos = 0; + _write_pos = 0; +} + SmartBuffer::SmartBuffer(size_t initial_size) : _data(alloc::Alloc::alloc(initial_size)), _read_pos(0), diff --git a/vespalib/src/vespa/vespalib/data/smart_buffer.h b/vespalib/src/vespa/vespalib/data/smart_buffer.h index f7c4dd05c3e..c63c948b07d 100644 --- a/vespalib/src/vespa/vespalib/data/smart_buffer.h +++ b/vespalib/src/vespa/vespalib/data/smart_buffer.h @@ -28,10 +28,17 @@ private: size_t unused() const { return (_data.size() - read_len()); } void ensure_free(size_t bytes); + void drop(); + public: SmartBuffer(size_t initial_size); ~SmartBuffer(); size_t capacity() const { return _data.size(); } + void drop_if_empty() { + if ((read_len() == 0) && (_data.size() > 0)) { + drop(); + } + } Memory obtain() override; Input &evict(size_t bytes) override; WritableMemory reserve(size_t bytes) override; diff --git a/vespalib/src/vespa/vespalib/net/crypto_engine.cpp b/vespalib/src/vespa/vespalib/net/crypto_engine.cpp index 81ff28f17a1..a59bc6a9869 100644 --- a/vespalib/src/vespa/vespalib/net/crypto_engine.cpp +++ b/vespalib/src/vespa/vespalib/net/crypto_engine.cpp @@ -53,6 +53,7 @@ public: ssize_t write(const char *buf, size_t len) override { return _socket.write(buf, len); } ssize_t flush() override { return 0; } ssize_t half_close() override { return _socket.half_close(); } + void drop_empty_buffers() override {} }; class XorCryptoSocket : public CryptoSocket @@ -186,6 +187,10 @@ public: } return _socket.half_close(); } + void drop_empty_buffers() override { + _input.drop_if_empty(); + _output.drop_if_empty(); + } }; using net::tls::AuthorizationMode; diff --git a/vespalib/src/vespa/vespalib/net/crypto_socket.h b/vespalib/src/vespa/vespalib/net/crypto_socket.h index d51a97e2743..91af71296de 100644 --- a/vespalib/src/vespa/vespalib/net/crypto_socket.h +++ b/vespalib/src/vespa/vespalib/net/crypto_socket.h @@ -136,6 +136,13 @@ struct CryptoSocket { **/ virtual ssize_t half_close() = 0; + /** + * This function can be called at any time to drop any currently + * empty internal buffers. Typically called after drain or flush + * indicates that no further progress can be made. + **/ + virtual void drop_empty_buffers() = 0; + virtual ~CryptoSocket(); }; diff --git a/vespalib/src/vespa/vespalib/net/tls/crypto_codec_adapter.cpp b/vespalib/src/vespa/vespalib/net/tls/crypto_codec_adapter.cpp index 660eee95132..4c0533a409a 100644 --- a/vespalib/src/vespa/vespalib/net/tls/crypto_codec_adapter.cpp +++ b/vespalib/src/vespa/vespalib/net/tls/crypto_codec_adapter.cpp @@ -201,4 +201,11 @@ CryptoCodecAdapter::half_close() return _socket.half_close(); } +void +CryptoCodecAdapter::drop_empty_buffers() +{ + _input.drop_if_empty(); + _output.drop_if_empty(); +} + } // namespace vespalib::net::tls diff --git a/vespalib/src/vespa/vespalib/net/tls/crypto_codec_adapter.h b/vespalib/src/vespa/vespalib/net/tls/crypto_codec_adapter.h index a31e8c5dfed..372a2191a88 100644 --- a/vespalib/src/vespa/vespalib/net/tls/crypto_codec_adapter.h +++ b/vespalib/src/vespa/vespalib/net/tls/crypto_codec_adapter.h @@ -45,6 +45,7 @@ public: ssize_t write(const char *buf, size_t len) override; ssize_t flush() override; ssize_t half_close() override; + void drop_empty_buffers() override; }; } // namespace vespalib::net::tls diff --git a/vespalib/src/vespa/vespalib/net/tls/maybe_tls_crypto_socket.cpp b/vespalib/src/vespa/vespalib/net/tls/maybe_tls_crypto_socket.cpp index d5ab1113013..3035387e4f0 100644 --- a/vespalib/src/vespa/vespalib/net/tls/maybe_tls_crypto_socket.cpp +++ b/vespalib/src/vespa/vespalib/net/tls/maybe_tls_crypto_socket.cpp @@ -77,12 +77,14 @@ public: if (frame > 0) { memcpy(buf, src.data, frame); _buffer.evict(frame); + _buffer.drop_if_empty(); } return frame; } ssize_t write(const char *buf, size_t len) override { return _socket.write(buf, len); } ssize_t flush() override { return 0; } ssize_t half_close() override { return _socket.half_close(); } + void drop_empty_buffers() override {} }; } // namespace vespalib::<unnamed> diff --git a/vespalib/src/vespa/vespalib/net/tls/maybe_tls_crypto_socket.h b/vespalib/src/vespa/vespalib/net/tls/maybe_tls_crypto_socket.h index cd2d84bcc08..2768bc0fc52 100644 --- a/vespalib/src/vespa/vespalib/net/tls/maybe_tls_crypto_socket.h +++ b/vespalib/src/vespa/vespalib/net/tls/maybe_tls_crypto_socket.h @@ -32,6 +32,7 @@ public: ssize_t write(const char *buf, size_t len) override { return _socket->write(buf, len); } ssize_t flush() override { return _socket->flush(); } ssize_t half_close() override { return _socket->half_close(); } + void drop_empty_buffers() override { _socket->drop_empty_buffers(); } }; } // namespace vespalib diff --git a/vespalib/src/vespa/vespalib/stllike/hashtable.h b/vespalib/src/vespa/vespalib/stllike/hashtable.h index b94672aaa06..ede18f89dc2 100644 --- a/vespalib/src/vespa/vespalib/stllike/hashtable.h +++ b/vespalib/src/vespa/vespalib/stllike/hashtable.h @@ -102,7 +102,9 @@ class hash_node { public: using next_t=hashtable_base::next_t; enum {npos=-1u, invalid=-2u}; - hash_node() : _node(), _next(invalid) {} + hash_node() + : _next(invalid) + {} hash_node(const V & node, next_t next=npos) : _next(next) { diff --git a/vespalib/src/vespa/vespalib/stllike/hashtable.hpp b/vespalib/src/vespa/vespalib/stllike/hashtable.hpp index d80113a8f55..494dc223f5b 100644 --- a/vespalib/src/vespa/vespalib/stllike/hashtable.hpp +++ b/vespalib/src/vespa/vespalib/stllike/hashtable.hpp @@ -146,6 +146,8 @@ hashtable<Key, Value, Hash, Equal, KeyExtract, Modulator>::erase(const Key & key template< typename Key, typename Value, typename Hash, typename Equal, typename KeyExtract, typename Modulator > void hashtable<Key, Value, Hash, Equal, KeyExtract, Modulator>::clear() { + if (_count == 0) return; // Already empty and properly initialized + _nodes.clear(); _count = 0; _nodes.resize(getTableSize()); diff --git a/vespalib/src/vespa/vespalib/util/alloc.h b/vespalib/src/vespa/vespalib/util/alloc.h index f608a244035..a97e8c9f25e 100644 --- a/vespalib/src/vespa/vespalib/util/alloc.h +++ b/vespalib/src/vespa/vespalib/util/alloc.h @@ -102,13 +102,21 @@ private: namespace vespalib { /// Rounds up to the closest number that is a power of 2 -inline size_t roundUp2inN(size_t minimum) { +inline size_t +roundUp2inN(size_t minimum) { return 2ul << Optimized::msbIdx(minimum - 1); } /// Rounds minElems up to the closest number where minElems*elemSize is a power of 2 -inline size_t roundUp2inN(size_t minElems, size_t elemSize) { +inline size_t +roundUp2inN(size_t minElems, size_t elemSize) { return roundUp2inN(minElems * elemSize)/elemSize; } +template <typename T> +size_t +roundUp2inN(size_t elems) { + return roundUp2inN(elems, sizeof(T)); +} + } diff --git a/vespalib/src/vespa/vespalib/util/optimized.h b/vespalib/src/vespa/vespalib/util/optimized.h index 92cf1f0ca24..6c6d1b12a71 100644 --- a/vespalib/src/vespa/vespalib/util/optimized.h +++ b/vespalib/src/vespa/vespalib/util/optimized.h @@ -22,9 +22,9 @@ public: static int lsbIdx(unsigned int v); static int lsbIdx(unsigned long v); static int lsbIdx(unsigned long long v); - static int popCount(unsigned int v) { return __builtin_popcount(v); } - static int popCount(unsigned long v) { return __builtin_popcountl(v); } - static int popCount(unsigned long long v) { return __builtin_popcountll(v); } + static constexpr int popCount(unsigned int v) { return __builtin_popcount(v); } + static constexpr int popCount(unsigned long v) { return __builtin_popcountl(v); } + static constexpr int popCount(unsigned long long v) { return __builtin_popcountll(v); } }; /** |