diff options
Diffstat (limited to 'config-model/src')
22 files changed, 180 insertions, 313 deletions
diff --git a/config-model/src/main/java/com/yahoo/config/model/ConfigModelRepo.java b/config-model/src/main/java/com/yahoo/config/model/ConfigModelRepo.java index 0e0f992952a..8985dac91aa 100644 --- a/config-model/src/main/java/com/yahoo/config/model/ConfigModelRepo.java +++ b/config-model/src/main/java/com/yahoo/config/model/ConfigModelRepo.java @@ -4,27 +4,16 @@ package com.yahoo.config.model; import com.yahoo.config.application.api.ApplicationFile; import com.yahoo.config.application.api.ApplicationPackage; import com.yahoo.config.model.ConfigModelContext.ApplicationType; -import com.yahoo.config.model.deploy.DeployState; import com.yahoo.config.model.builder.xml.ConfigModelBuilder; import com.yahoo.config.model.builder.xml.ConfigModelId; import com.yahoo.config.model.builder.xml.XmlHelper; +import com.yahoo.config.model.deploy.DeployState; import com.yahoo.config.model.graph.ModelGraphBuilder; import com.yahoo.config.model.graph.ModelNode; +import com.yahoo.config.model.producer.AbstractConfigProducer; import com.yahoo.config.model.provision.HostsXmlProvisioner; - -import java.util.ArrayList; -import java.util.Collection; -import java.util.Collections; -import java.util.Iterator; -import java.util.LinkedHashMap; -import java.util.List; -import java.util.Map; -import java.util.Optional; -import java.util.TreeMap; -import java.util.logging.Level; import com.yahoo.path.Path; import com.yahoo.text.XML; -import com.yahoo.config.model.producer.AbstractConfigProducer; import com.yahoo.vespa.model.VespaModel; import com.yahoo.vespa.model.builder.VespaModelBuilder; import com.yahoo.vespa.model.clients.Clients; @@ -32,13 +21,22 @@ import com.yahoo.vespa.model.content.Content; import com.yahoo.vespa.model.routing.Routing; import org.w3c.dom.Document; import org.w3c.dom.Element; -import org.xml.sax.InputSource; import org.xml.sax.SAXException; import java.io.IOException; import java.io.Reader; import java.io.Serializable; import java.io.StringReader; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.Iterator; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.TreeMap; +import java.util.logging.Level; import java.util.logging.Logger; /** @@ -91,7 +89,7 @@ public class ConfigModelRepo implements ConfigModelRepoAdder, Serializable, Iter builder.postProc(deployState.getDeployLogger(), root, this); } - private Element getServicesFromApp(ApplicationPackage applicationPackage) throws IOException, SAXException { + private Element getServicesFromApp(ApplicationPackage applicationPackage) throws IOException { try (Reader servicesFile = applicationPackage.getServices()) { return getServicesFromReader(servicesFile); } @@ -188,8 +186,8 @@ public class ConfigModelRepo implements ConfigModelRepoAdder, Serializable, Iter return permanentServices; } - private Element getServicesFromReader(Reader reader) throws IOException, SAXException { - Document doc = XmlHelper.getDocumentBuilder().parse(new InputSource(reader)); + private Element getServicesFromReader(Reader reader) { + Document doc = XmlHelper.getDocument(reader); return doc.getDocumentElement(); } @@ -274,10 +272,10 @@ public class ConfigModelRepo implements ConfigModelRepoAdder, Serializable, Iter } // TODO: Doctoring on the XML is the wrong level for this. We should be able to mark a model as default instead -Jon - private static Element getImplicitAdmin(DeployState deployState) throws IOException, SAXException { + private static Element getImplicitAdmin(DeployState deployState) { String defaultAdminElement = deployState.isHosted() ? getImplicitAdminV4() : getImplicitAdminV2(); log.log(Level.FINE, () -> "No <admin> defined, using " + defaultAdminElement); - return XmlHelper.getDocumentBuilder().parse(new InputSource(new StringReader(defaultAdminElement))).getDocumentElement(); + return XmlHelper.getDocument(new StringReader(defaultAdminElement)).getDocumentElement(); } private static String getImplicitAdminV2() { diff --git a/config-model/src/main/java/com/yahoo/config/model/builder/xml/XmlHelper.java b/config-model/src/main/java/com/yahoo/config/model/builder/xml/XmlHelper.java index 5763be301a6..220d6bbb768 100644 --- a/config-model/src/main/java/com/yahoo/config/model/builder/xml/XmlHelper.java +++ b/config-model/src/main/java/com/yahoo/config/model/builder/xml/XmlHelper.java @@ -3,12 +3,14 @@ package com.yahoo.config.model.builder.xml; import com.yahoo.component.ComponentId; import com.yahoo.component.ComponentSpecification; -import java.util.logging.Level; import com.yahoo.text.XML; +import com.yahoo.yolean.Exceptions; import org.w3c.dom.Document; import org.w3c.dom.Element; +import org.xml.sax.ErrorHandler; import org.xml.sax.InputSource; import org.xml.sax.SAXException; +import org.xml.sax.SAXParseException; import javax.xml.parsers.DocumentBuilder; import javax.xml.parsers.DocumentBuilderFactory; @@ -19,6 +21,7 @@ import java.util.ArrayList; import java.util.Collection; import java.util.List; import java.util.Optional; +import java.util.logging.Level; import java.util.logging.Logger; @@ -74,9 +77,15 @@ public final class XmlHelper { } public static Document getDocument(Reader reader) { + return getDocument(reader, "unknown source"); + } + + public static Document getDocument(Reader reader, String source) { Document doc; try { - doc = getDocumentBuilder().parse(new InputSource(reader)); + InputSource inputSource = new InputSource(reader); + inputSource.setPublicId(source); + doc = getDocumentBuilder().parse(inputSource); } catch (SAXException | IOException e) { throw new IllegalArgumentException(e); } @@ -121,6 +130,7 @@ public final class XmlHelper { public static synchronized DocumentBuilder getDocumentBuilder() { try { DocumentBuilder docBuilder = factory.newDocumentBuilder(); + docBuilder.setErrorHandler(new CustomErrorHandler(log)); log.log(Level.FINE, "XML parser now operational!"); return docBuilder; } catch (ParserConfigurationException e) { @@ -144,4 +154,34 @@ public final class XmlHelper { if (child.getFirstChild() == null) return Optional.empty(); return Optional.ofNullable(child.getFirstChild().getNodeValue()); } + + /** Error handler which will output name of source for warnings and errors */ + private static class CustomErrorHandler implements ErrorHandler { + + private final Logger logger; + + CustomErrorHandler(Logger logger) { + super(); + this.logger = logger; + } + + public void warning(SAXParseException e) { + logger.log(Level.WARNING, message(e)); + } + + public void error(SAXParseException e) { + throw new IllegalArgumentException(message(e)); + } + + public void fatalError(SAXParseException e) { throw new IllegalArgumentException(message(e)); } + + private String message(SAXParseException e) { + String sourceId = e.getPublicId() == null ? "" : e.getPublicId(); + return "Invalid XML" + (sourceId.isEmpty() ? " (unknown source)" : " in " + sourceId) + + ": " + Exceptions.toMessageString(e) + + " [" + e.getLineNumber() + ":" + e.getColumnNumber() + "]"; + } + + } + } diff --git a/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java b/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java index 8366dde383b..86f1a9eb9f4 100644 --- a/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java +++ b/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java @@ -42,9 +42,6 @@ public class TestProperties implements ModelContext.Properties, ModelContext.Fea private double defaultTermwiseLimit = 1.0; private String jvmGCOptions = null; private String sequencerType = "THROUGHPUT"; - private int feedTaskLimit = 1000; - private int feedMasterTaskLimit = 1000; - private String sharedFieldWriterExecutor = "NONE"; private boolean firstTimeDeployment = false; private String responseSequencerType = "ADAPTIVE"; private int responseNumThreads = 2; @@ -70,14 +67,12 @@ public class TestProperties implements ModelContext.Properties, ModelContext.Fea private List<String> zoneDnsSuffixes = List.of(); private int maxCompactBuffers = 1; private boolean failDeploymentWithInvalidJvmOptions = false; - private String persistenceAsyncThrottling = "UNLIMITED"; private String mergeThrottlingPolicy = "STATIC"; private double persistenceThrottlingWsDecrementFactor = 1.2; private double persistenceThrottlingWsBackoff = 0.95; private int persistenceThrottlingWindowSize = -1; private double persistenceThrottlingWsResizeRate = 3.0; private boolean persistenceThrottlingOfMergeFeedOps = true; - private boolean inhibitDefaultMergesWhenGlobalMergesPending = false; private boolean useV8GeoPositions = false; private List<String> environmentVariables = List.of(); private boolean avoidRenamingSummaryFeatures = false; @@ -96,9 +91,6 @@ public class TestProperties implements ModelContext.Properties, ModelContext.Fea @Override public Set<ContainerEndpoint> endpoints() { return endpoints; } @Override public String jvmGCOptions(Optional<ClusterSpec.Type> clusterType) { return jvmGCOptions; } @Override public String feedSequencerType() { return sequencerType; } - @Override public int feedTaskLimit() { return feedTaskLimit; } - @Override public int feedMasterTaskLimit() { return feedMasterTaskLimit; } - @Override public String sharedFieldWriterExecutor() { return sharedFieldWriterExecutor; } @Override public boolean isBootstrap() { return false; } @Override public boolean isFirstTimeDeployment() { return firstTimeDeployment; } @Override public boolean useDedicatedNodeForLogserver() { return useDedicatedNodeForLogserver; } @@ -131,14 +123,12 @@ public class TestProperties implements ModelContext.Properties, ModelContext.Fea @Override public List<String> zoneDnsSuffixes() { return zoneDnsSuffixes; } @Override public int maxCompactBuffers() { return maxCompactBuffers; } @Override public boolean failDeploymentWithInvalidJvmOptions() { return failDeploymentWithInvalidJvmOptions; } - @Override public String persistenceAsyncThrottling() { return persistenceAsyncThrottling; } @Override public String mergeThrottlingPolicy() { return mergeThrottlingPolicy; } @Override public double persistenceThrottlingWsDecrementFactor() { return persistenceThrottlingWsDecrementFactor; } @Override public double persistenceThrottlingWsBackoff() { return persistenceThrottlingWsBackoff; } @Override public int persistenceThrottlingWindowSize() { return persistenceThrottlingWindowSize; } @Override public double persistenceThrottlingWsResizeRate() { return persistenceThrottlingWsResizeRate; } @Override public boolean persistenceThrottlingOfMergeFeedOps() { return persistenceThrottlingOfMergeFeedOps; } - @Override public boolean inhibitDefaultMergesWhenGlobalMergesPending() { return inhibitDefaultMergesWhenGlobalMergesPending; } @Override public boolean useV8GeoPositions() { return useV8GeoPositions; } @Override public List<String> environmentVariables() { return environmentVariables; } @Override public boolean avoidRenamingSummaryFeatures() { return this.avoidRenamingSummaryFeatures; } @@ -177,18 +167,6 @@ public class TestProperties implements ModelContext.Properties, ModelContext.Fea sequencerType = type; return this; } - public TestProperties setFeedTaskLimit(int value) { - feedTaskLimit = value; - return this; - } - public TestProperties setFeedMasterTaskLimit(int value) { - feedMasterTaskLimit = value; - return this; - } - public TestProperties setSharedFieldWriterExecutor(String value) { - sharedFieldWriterExecutor = value; - return this; - } public TestProperties setResponseSequencerType(String type) { responseSequencerType = type; return this; @@ -326,11 +304,6 @@ public class TestProperties implements ModelContext.Properties, ModelContext.Fea return this; } - public TestProperties setPersistenceAsyncThrottling(String type) { - this.persistenceAsyncThrottling = type; - return this; - } - public TestProperties setMergeThrottlingPolicy(String policy) { this.mergeThrottlingPolicy = policy; return this; @@ -361,11 +334,6 @@ public class TestProperties implements ModelContext.Properties, ModelContext.Fea return this; } - public TestProperties inhibitDefaultMergesWhenGlobalMergesPending(boolean value) { - this.inhibitDefaultMergesWhenGlobalMergesPending = value; - return this; - } - public TestProperties setUseV8GeoPositions(boolean value) { this.useV8GeoPositions = value; return this; diff --git a/config-model/src/main/java/com/yahoo/config/model/provision/Hosts.java b/config-model/src/main/java/com/yahoo/config/model/provision/Hosts.java index c9a03dad65e..2ef8cb4a0bf 100644 --- a/config-model/src/main/java/com/yahoo/config/model/provision/Hosts.java +++ b/config-model/src/main/java/com/yahoo/config/model/provision/Hosts.java @@ -8,12 +8,13 @@ import com.yahoo.text.XML; import com.yahoo.vespa.model.builder.xml.dom.VespaDomBuilder; import org.w3c.dom.Document; import org.w3c.dom.Element; -import org.xml.sax.InputSource; -import org.xml.sax.SAXException; -import java.io.IOException; import java.io.Reader; -import java.util.*; +import java.util.ArrayList; +import java.util.Collection; +import java.util.HashSet; +import java.util.List; +import java.util.Set; import java.util.logging.Logger; /** @@ -63,12 +64,7 @@ public class Hosts { */ public static Hosts readFrom(Reader hostsFile) { List<Host> hosts = new ArrayList<>(); - Document doc; - try { - doc = XmlHelper.getDocumentBuilder().parse(new InputSource(hostsFile)); - } catch (SAXException | IOException e) { - throw new IllegalArgumentException(e); - } + Document doc = XmlHelper.getDocument(hostsFile); for (Element hostE : XML.getChildren(doc.getDocumentElement(), "host")) { String name = hostE.getAttribute("name"); if (name.equals("")) { diff --git a/config-model/src/main/java/com/yahoo/config/model/test/MockRoot.java b/config-model/src/main/java/com/yahoo/config/model/test/MockRoot.java index b29ff1ee58b..7e2953e6606 100644 --- a/config-model/src/main/java/com/yahoo/config/model/test/MockRoot.java +++ b/config-model/src/main/java/com/yahoo/config/model/test/MockRoot.java @@ -19,10 +19,7 @@ import com.yahoo.vespa.model.builder.xml.dom.DomAdminV2Builder; import com.yahoo.vespa.model.filedistribution.FileDistributionConfigProducer; import com.yahoo.vespa.model.filedistribution.FileReferencesRepository; import org.w3c.dom.Document; -import org.xml.sax.InputSource; -import org.xml.sax.SAXException; -import java.io.IOException; import java.io.StringReader; import java.util.ArrayList; import java.util.Collections; @@ -143,13 +140,9 @@ public class MockRoot extends AbstractConfigProducerRoot { "<?xml version='1.0' encoding='utf-8' ?>" + "<services>" + xml + "</services>"; - try { - Document doc = XmlHelper.getDocumentBuilder().parse(new InputSource(new StringReader(servicesXml))); - setAdmin(new DomAdminV2Builder(ConfigModelContext.ApplicationType.DEFAULT, false, new ArrayList<>()). - build(deployState, this, XML.getChildren(doc.getDocumentElement(), "admin").get(0))); - } catch (SAXException | IOException e) { - throw new RuntimeException(e); - } + Document doc = XmlHelper.getDocument(new StringReader(servicesXml)); + setAdmin(new DomAdminV2Builder(ConfigModelContext.ApplicationType.DEFAULT, false, new ArrayList<>()) + .build(deployState, this, XML.getChildren(doc.getDocumentElement(), "admin").get(0))); } public final void setAdmin(Admin admin) { diff --git a/config-model/src/main/java/com/yahoo/config/model/test/TestUtil.java b/config-model/src/main/java/com/yahoo/config/model/test/TestUtil.java index df916907472..c05d7bf4942 100644 --- a/config-model/src/main/java/com/yahoo/config/model/test/TestUtil.java +++ b/config-model/src/main/java/com/yahoo/config/model/test/TestUtil.java @@ -25,9 +25,7 @@ public class TestUtil { lines.addAll(Arrays.asList(xmlLines)); try { - return XmlHelper.getDocumentBuilder().parse( - inputSource((CollectionUtil.mkString(lines, "\n").replace("'", "\"")))) - .getDocumentElement(); + return XmlHelper.getDocument(new StringReader(CollectionUtil.mkString(lines, "\n").replace("'", "\""))).getDocumentElement(); } catch (Exception e) { throw new RuntimeException(e); } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/VespaModelFactory.java b/config-model/src/main/java/com/yahoo/vespa/model/VespaModelFactory.java index f2d0ab03e27..25e2a7593b0 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/VespaModelFactory.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/VespaModelFactory.java @@ -119,14 +119,12 @@ public class VespaModelFactory implements ModelFactory { for (ConfigChangeAction action : changeActions) { if (action.getType().equals(ConfigChangeAction.Type.REINDEX)) { VespaModel currentModel = (VespaModel) currentActiveModel.get(); - var currentVersion = currentModel.version(); var currentMeta = currentModel.applicationPackage().getMetaData(); - var nextVersion = nextModel.version(); var nextMeta = nextModel.applicationPackage().getMetaData(); log.log(Level.INFO, String.format("Model [%s/%s] -> [%s/%s] triggers reindexing: %s", currentModel.version().toString(), currentMeta.toString(), nextModel.version().toString(), nextMeta.toString(), - action.toString())); + action)); } } } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/VespaMetricSet.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/VespaMetricSet.java index 34586df424f..3a8cf23a49e 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/VespaMetricSet.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/VespaMetricSet.java @@ -154,6 +154,17 @@ public class VespaMetricSet { addMetric(metrics, "jdisc.thread_pool.unhandled_exceptions", suffixes); addMetric(metrics, "jdisc.thread_pool.work_queue.capacity", suffixes); addMetric(metrics, "jdisc.thread_pool.work_queue.size", suffixes); + addMetric(metrics, "jdisc.thread_pool.rejected_tasks", suffixes); + addMetric(metrics, "jdisc.thread_pool.size", suffixes); + addMetric(metrics, "jdisc.thread_pool.max_allowed_size", suffixes); + addMetric(metrics, "jdisc.thread_pool.active_threads", suffixes); + + addMetric(metrics, "jdisc.http.jetty.threadpool.thread.max", suffixes); + addMetric(metrics, "jdisc.http.jetty.threadpool.thread.min", suffixes); + addMetric(metrics, "jdisc.http.jetty.threadpool.thread.reserved", suffixes); + addMetric(metrics, "jdisc.http.jetty.threadpool.thread.busy", suffixes); + addMetric(metrics, "jdisc.http.jetty.threadpool.thread.total", suffixes); + addMetric(metrics, "jdisc.http.jetty.threadpool.queue.size", suffixes); } metrics.add(new Metric("httpapi_latency.max")); @@ -223,12 +234,6 @@ public class VespaMetricSet { metrics.add(new Metric("jdisc.http.handler.unhandled_exceptions.rate")); - addMetric(metrics, "jdisc.http.jetty.threadpool.thread.max", List.of("last")); - addMetric(metrics, "jdisc.http.jetty.threadpool.thread.reserved", List.of("last")); - addMetric(metrics, "jdisc.http.jetty.threadpool.thread.busy", List.of("sum", "count", "min", "max")); - addMetric(metrics, "jdisc.http.jetty.threadpool.thread.total", List.of("sum", "count", "min", "max")); - addMetric(metrics, "jdisc.http.jetty.threadpool.queue.size", List.of("sum", "count", "min", "max")); - addMetric(metrics, "jdisc.http.filtering.request.handled", List.of("rate")); addMetric(metrics, "jdisc.http.filtering.request.unhandled", List.of("rate")); addMetric(metrics, "jdisc.http.filtering.response.handled", List.of("rate")); diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/RankSetupValidator.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/RankSetupValidator.java index 6a2ccec30c3..ed943317a1f 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/RankSetupValidator.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/RankSetupValidator.java @@ -22,11 +22,10 @@ import com.yahoo.vespa.config.search.core.RankingConstantsConfig; import com.yahoo.vespa.config.search.core.RankingExpressionsConfig; import com.yahoo.vespa.defaults.Defaults; import com.yahoo.vespa.model.VespaModel; -import com.yahoo.vespa.model.search.SearchCluster; import com.yahoo.vespa.model.search.DocumentDatabase; import com.yahoo.vespa.model.search.IndexedSearchCluster; +import com.yahoo.vespa.model.search.SearchCluster; import com.yahoo.yolean.Exceptions; - import java.io.File; import java.io.IOException; import java.nio.file.Files; @@ -64,7 +63,6 @@ public class RankSetupValidator extends Validator { deployState.getProperties().applicationId().toFullString() + ".") .toFile(); - for (SearchCluster cluster : model.getSearchClusters()) { // Skipping rank expression checking for streaming clusters, not implemented yet if (cluster.isStreaming()) continue; @@ -72,16 +70,15 @@ public class RankSetupValidator extends Validator { IndexedSearchCluster sc = (IndexedSearchCluster) cluster; String clusterDir = cfgDir.getAbsolutePath() + "/" + sc.getClusterName() + "/"; for (DocumentDatabase docDb : sc.getDocumentDbs()) { - final String name = docDb.getDerivedConfiguration().getSchema().getName(); - String searchDir = clusterDir + name + "/"; - writeConfigs(searchDir, docDb); - writeExtraVerifyRanksetupConfig(searchDir, docDb); - if (!validate("dir:" + searchDir, sc, name, deployState.getDeployLogger(), cfgDir)) { + String schemaName = docDb.getDerivedConfiguration().getSchema().getName(); + String schemaDir = clusterDir + schemaName + "/"; + writeConfigs(schemaDir, docDb); + writeExtraVerifyRankSetupConfig(schemaDir, docDb); + if (!validate("dir:" + schemaDir, sc, schemaName, deployState.getDeployLogger(), cfgDir)) { return; } } } - } catch (IOException e) { throw new RuntimeException(e); } finally { @@ -90,13 +87,13 @@ public class RankSetupValidator extends Validator { } } - private boolean validate(String configId, SearchCluster searchCluster, String sdName, DeployLogger deployLogger, File tempDir) { + private boolean validate(String configId, SearchCluster searchCluster, String schema, DeployLogger deployLogger, File tempDir) { Instant start = Instant.now(); try { - log.log(Level.FINE, () -> String.format("Validating schema '%s' for %s with config id %s", sdName, searchCluster, configId)); - boolean ret = execValidate(configId, searchCluster, sdName, deployLogger); + log.log(Level.FINE, () -> String.format("Validating schema '%s' for cluster %s with config id %s", schema, searchCluster, configId)); + boolean ret = execValidate(configId, searchCluster, schema, deployLogger); if (!ret) { - // Give up, don't say same error msg repeatedly + // Give up, don't log same error msg repeatedly deleteTempDir(tempDir); } log.log(Level.FINE, () -> String.format("Validation took %s ms", Duration.between(start, Instant.now()).toMillis())); @@ -141,7 +138,7 @@ public class RankSetupValidator extends Validator { writeConfig(dir, ImportedFieldsConfig.getDefName() + ".cfg", ifcb.build()); } - private void writeExtraVerifyRanksetupConfig(List<String> config, Collection<? extends DistributableResource> resources) { + private void writeExtraVerifyRankSetupConfig(List<String> config, Collection<? extends DistributableResource> resources) { for (DistributableResource model : resources) { String modelPath = getFileRepositoryPath(model.getFilePath().getName(), model.getFileReference()); int index = config.size() / 2; @@ -151,12 +148,12 @@ public class RankSetupValidator extends Validator { } } - private void writeExtraVerifyRanksetupConfig(String dir, DocumentDatabase db) throws IOException { + private void writeExtraVerifyRankSetupConfig(String dir, DocumentDatabase db) throws IOException { List<String> config = new ArrayList<>(); // Assist verify-ranksetup in finding the actual ONNX model files - writeExtraVerifyRanksetupConfig(config, db.getDerivedConfiguration().getSchema().onnxModels().asMap().values()); - writeExtraVerifyRanksetupConfig(config, db.getDerivedConfiguration().getSchema().rankExpressionFiles().asMap().values()); + writeExtraVerifyRankSetupConfig(config, db.getDerivedConfiguration().getSchema().onnxModels().asMap().values()); + writeExtraVerifyRankSetupConfig(config, db.getDerivedConfiguration().getSchema().rankExpressionFiles().asMap().values()); String configContent = config.isEmpty() ? "" : StringUtilities.implodeMultiline(config); IOUtils.writeFile(dir + "verify-ranksetup.cfg", configContent, false); @@ -173,12 +170,13 @@ public class RankSetupValidator extends Validator { } private boolean execValidate(String configId, SearchCluster sc, String sdName, DeployLogger deployLogger) { - String job = String.format("%s %s", binaryName, configId); - ProcessExecuter executer = new ProcessExecuter(true); + String command = String.format("%s %s", binaryName, configId); try { - Pair<Integer, String> ret = executer.exec(job); - if (ret.getFirst() != 0) { - validateFail(ret.getSecond(), sc, sdName, deployLogger); + Pair<Integer, String> ret = new ProcessExecuter(true).exec(command); + Integer exitCode = ret.getFirst(); + String output = ret.getSecond(); + if (exitCode != 0) { + validateFail(output, exitCode, sc, sdName, deployLogger); } } catch (IOException e) { validateWarn(e, deployLogger); @@ -194,23 +192,32 @@ public class RankSetupValidator extends Validator { deployLogger.logApplicationPackage(Level.WARNING, msg); } - private void validateFail(String output, SearchCluster sc, String sdName, DeployLogger deployLogger) { - StringBuilder errMsg = new StringBuilder("Error in rank setup in schema '").append(sdName) + private void validateFail(String output, int exitCode, SearchCluster sc, String sdName, DeployLogger deployLogger) { + StringBuilder message = new StringBuilder("Error in rank setup in schema '").append(sdName) .append("' for content cluster '").append(sc.getClusterName()).append("'.").append(" Details:\n"); - for (String line : output.split("\n")) { - // Remove debug lines from start script - if (line.startsWith("debug\t")) continue; - try { - LogMessage logMessage = LogMessage.parseNativeFormat(line); - errMsg.append(logMessage.getLevel()).append(": ").append(logMessage.getPayload()).append("\n"); - } catch (InvalidLogFormatException e) { - errMsg.append(line).append("\n"); + if (output.isEmpty()) { + message.append("Verifying rank setup failed and got no output from stderr and stdout from '") + .append(binaryName) + .append("' (exit code: ") + .append(exitCode) + .append("). This could be due to full disk, out of memory etc."); + } else { + for (String line : output.split("\n")) { + // Remove debug lines from start script + if (line.startsWith("debug\t")) continue; + try { + LogMessage logMessage = LogMessage.parseNativeFormat(line); + message.append(logMessage.getLevel()).append(": ").append(logMessage.getPayload()).append("\n"); + } catch (InvalidLogFormatException e) { + message.append(line).append("\n"); + } } } + if (ignoreValidationErrors) { - deployLogger.log(Level.WARNING, errMsg.append("(Continuing since ignoreValidationErrors flag is set.)").toString()); + deployLogger.log(Level.WARNING, message.append("(Continuing since ignoreValidationErrors flag is set.)").toString()); } else { - throw new IllegalArgumentException(errMsg.toString()); + throw new IllegalArgumentException(message.toString()); } } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/VespaDomBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/VespaDomBuilder.java index df77d83da83..b619210155f 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/VespaDomBuilder.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/VespaDomBuilder.java @@ -4,11 +4,10 @@ package com.yahoo.vespa.model.builder.xml.dom; import com.yahoo.config.application.api.DeployLogger; import com.yahoo.config.model.ApplicationConfigProducerRoot; import com.yahoo.config.model.ConfigModelRepo; -import com.yahoo.config.model.deploy.DeployState; import com.yahoo.config.model.builder.xml.XmlHelper; +import com.yahoo.config.model.deploy.DeployState; import com.yahoo.config.model.producer.AbstractConfigProducer; import com.yahoo.config.model.producer.UserConfigRepo; -import java.util.logging.Level; import com.yahoo.text.XML; import com.yahoo.vespa.model.AbstractService; import com.yahoo.vespa.model.Affinity; @@ -24,7 +23,6 @@ import com.yahoo.vespa.model.content.Content; import com.yahoo.vespa.model.generic.builder.DomServiceClusterBuilder; import com.yahoo.vespa.model.generic.service.ServiceCluster; import com.yahoo.vespa.model.search.SearchCluster; - import org.w3c.dom.Document; import org.w3c.dom.Element; import org.w3c.dom.Node; @@ -33,6 +31,7 @@ import org.w3c.dom.NodeList; import java.util.ArrayList; import java.util.LinkedList; import java.util.List; +import java.util.logging.Level; import java.util.logging.Logger; /** @@ -94,7 +93,8 @@ public class VespaDomBuilder extends VespaModelBuilder { public ApplicationConfigProducerRoot getRoot(String name, DeployState deployState, AbstractConfigProducer parent) { try { return new DomRootBuilder(name). - build(deployState, parent, XmlHelper.getDocument(deployState.getApplicationPackage().getServices()).getDocumentElement()); + build(deployState, parent, XmlHelper.getDocument(deployState.getApplicationPackage().getServices(), "services.xml") + .getDocumentElement()); } catch (Exception e) { throw new IllegalArgumentException(e); } @@ -303,7 +303,7 @@ public class VespaDomBuilder extends VespaModelBuilder { @Override public List<ServiceCluster> getClusters(DeployState deployState, AbstractConfigProducer parent) { List<ServiceCluster> clusters = new ArrayList<>(); - Document services = XmlHelper.getDocument(deployState.getApplicationPackage().getServices()); + Document services = XmlHelper.getDocument(deployState.getApplicationPackage().getServices(), "services.xml"); for (Element clusterSpec : XML.getChildren(services.getDocumentElement(), "cluster")) { DomServiceClusterBuilder clusterBuilder = new DomServiceClusterBuilder(clusterSpec.getAttribute("name")); clusters.add(clusterBuilder.build(deployState, parent.getRoot(), clusterSpec)); diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerThreadpool.java b/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerThreadpool.java index 7111a88fc01..489e4cc135a 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerThreadpool.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerThreadpool.java @@ -4,7 +4,7 @@ package com.yahoo.vespa.model.container; import com.yahoo.container.bundle.BundleInstantiationSpecification; import com.yahoo.container.handler.threadpool.ContainerThreadPool; import com.yahoo.container.handler.threadpool.ContainerThreadpoolConfig; -import com.yahoo.container.handler.threadpool.DefaultContainerThreadpool; +import com.yahoo.container.handler.threadpool.ContainerThreadpoolImpl; import com.yahoo.osgi.provider.model.ComponentModel; import com.yahoo.text.XML; import com.yahoo.vespa.model.container.component.SimpleComponent; @@ -26,7 +26,7 @@ public class ContainerThreadpool extends SimpleComponent implements ContainerThr super(new ComponentModel( BundleInstantiationSpecification.getFromStrings( "threadpool@" + name, - DefaultContainerThreadpool.class.getName(), + ContainerThreadpoolImpl.class.getName(), null))); this.name = name; this.userOptions = userOptions; diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/ContentSearchCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/content/ContentSearchCluster.java index 9bd3b455ef9..b137c04b5c4 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/content/ContentSearchCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/content/ContentSearchCluster.java @@ -66,13 +66,9 @@ public class ContentSearchCluster extends AbstractConfigProducer<SearchCluster> private final Map<StorageGroup, NodeSpec> groupToSpecMap = new LinkedHashMap<>(); private Optional<ResourceLimits> resourceLimits = Optional.empty(); private final ProtonConfig.Indexing.Optimize.Enum feedSequencerType; - private final int feedTaskLimit; - private final int feedMasterTaskLimit; - private final ProtonConfig.Feeding.Shared_field_writer_executor.Enum sharedFieldWriterExecutor; private final double defaultFeedConcurrency; private final boolean forwardIssuesToQrs; private final int defaultMaxCompactBuffers; - private final ProtonConfig.Replay_throttling_policy.Type.Enum persistenceAsyncThrottling; /** Whether the nodes of this cluster also hosts a container cluster in a hosted system */ private final double fractionOfMemoryReserved; @@ -198,22 +194,6 @@ public class ContentSearchCluster extends AbstractConfigProducer<SearchCluster> } } - private static ProtonConfig.Feeding.Shared_field_writer_executor.Enum convertSharedFieldWriterExecutor(String value) { - try { - return ProtonConfig.Feeding.Shared_field_writer_executor.Enum.valueOf(value); - } catch (Throwable t) { - return ProtonConfig.Feeding.Shared_field_writer_executor.Enum.NONE; - } - } - - private static ProtonConfig.Replay_throttling_policy.Type.Enum convertPersistenceAsyncThrottling(String value) { - try { - return ProtonConfig.Replay_throttling_policy.Type.Enum.valueOf(value); - } catch (Throwable t) { - return ProtonConfig.Replay_throttling_policy.Type.Enum.UNLIMITED; - } - } - private ContentSearchCluster(AbstractConfigProducer<?> parent, String clusterName, ModelContext.FeatureFlags featureFlags, @@ -232,13 +212,9 @@ public class ContentSearchCluster extends AbstractConfigProducer<SearchCluster> this.fractionOfMemoryReserved = fractionOfMemoryReserved; this.feedSequencerType = convertFeedSequencerType(featureFlags.feedSequencerType()); - this.feedTaskLimit = featureFlags.feedTaskLimit(); - this.feedMasterTaskLimit = featureFlags.feedMasterTaskLimit(); - this.sharedFieldWriterExecutor = convertSharedFieldWriterExecutor(featureFlags.sharedFieldWriterExecutor()); this.defaultFeedConcurrency = featureFlags.feedConcurrency(); this.forwardIssuesToQrs = featureFlags.forwardIssuesAsErrors(); this.defaultMaxCompactBuffers = featureFlags.maxCompactBuffers(); - this.persistenceAsyncThrottling = convertPersistenceAsyncThrottling(featureFlags.persistenceAsyncThrottling()); } public void setVisibilityDelay(double delay) { @@ -452,10 +428,6 @@ public class ContentSearchCluster extends AbstractConfigProducer<SearchCluster> } builder.indexing.optimize(feedSequencerType); - builder.indexing.tasklimit(feedTaskLimit); - builder.feeding.master_task_limit(feedMasterTaskLimit); - builder.feeding.shared_field_writer_executor(sharedFieldWriterExecutor); - builder.replay_throttling_policy.type(persistenceAsyncThrottling); } private boolean isGloballyDistributed(NewDocumentType docType) { diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/DistributorCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/content/DistributorCluster.java index 3f01f5610f1..b8d2a4f91fe 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/content/DistributorCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/content/DistributorCluster.java @@ -43,7 +43,6 @@ public class DistributorCluster extends AbstractConfigProducer<Distributor> impl private final boolean useThreePhaseUpdates; private final int maxActivationInhibitedOutOfSyncGroups; private final boolean unorderedMergeChaining; - private final boolean inhibitDefaultMergesWhenGlobalMergesPending; public static class Builder extends VespaDomBuilder.DomConfigProducerBuilder<DistributorCluster> { @@ -107,12 +106,11 @@ public class DistributorCluster extends AbstractConfigProducer<Distributor> impl boolean useThreePhaseUpdates = deployState.getProperties().featureFlags().useThreePhaseUpdates(); int maxInhibitedGroups = deployState.getProperties().featureFlags().maxActivationInhibitedOutOfSyncGroups(); boolean unorderedMergeChaining = deployState.getProperties().featureFlags().unorderedMergeChaining(); - boolean inhibitDefaultMerges = deployState.getProperties().featureFlags().inhibitDefaultMergesWhenGlobalMergesPending(); return new DistributorCluster(parent, new BucketSplitting.Builder().build(new ModelElement(producerSpec)), gc, hasIndexedDocumentType, useThreePhaseUpdates, - maxInhibitedGroups, unorderedMergeChaining, inhibitDefaultMerges); + maxInhibitedGroups, unorderedMergeChaining); } } @@ -120,8 +118,7 @@ public class DistributorCluster extends AbstractConfigProducer<Distributor> impl GcOptions gc, boolean hasIndexedDocumentType, boolean useThreePhaseUpdates, int maxActivationInhibitedOutOfSyncGroups, - boolean unorderedMergeChaining, - boolean inhibitDefaultMergesWhenGlobalMergesPending) + boolean unorderedMergeChaining) { super(parent, "distributor"); this.parent = parent; @@ -131,7 +128,6 @@ public class DistributorCluster extends AbstractConfigProducer<Distributor> impl this.useThreePhaseUpdates = useThreePhaseUpdates; this.maxActivationInhibitedOutOfSyncGroups = maxActivationInhibitedOutOfSyncGroups; this.unorderedMergeChaining = unorderedMergeChaining; - this.inhibitDefaultMergesWhenGlobalMergesPending = inhibitDefaultMergesWhenGlobalMergesPending; } @Override @@ -146,7 +142,6 @@ public class DistributorCluster extends AbstractConfigProducer<Distributor> impl builder.enable_metadata_only_fetch_phase_for_inconsistent_updates(useThreePhaseUpdates); builder.max_activation_inhibited_out_of_sync_groups(maxActivationInhibitedOutOfSyncGroups); builder.use_unordered_merge_chaining(unorderedMergeChaining); - builder.inhibit_default_merges_when_global_merges_pending(inhibitDefaultMergesWhenGlobalMergesPending); bucketSplitting.getConfig(builder); } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/storagecluster/FileStorProducer.java b/config-model/src/main/java/com/yahoo/vespa/model/content/storagecluster/FileStorProducer.java index a1c4fc41d55..fb4016f4cf4 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/content/storagecluster/FileStorProducer.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/content/storagecluster/FileStorProducer.java @@ -46,7 +46,6 @@ public class FileStorProducer implements StorFilestorConfig.Producer { private final ContentCluster cluster; private final int reponseNumThreads; private final StorFilestorConfig.Response_sequencer_type.Enum responseSequencerType; - private final StorFilestorConfig.Async_operation_throttler.Type.Enum asyncOperationThrottlerType; private final double persistenceThrottlingWsDecrementFactor; private final double persistenceThrottlingWsBackoff; private final int persistenceThrottingWindowSize; @@ -62,20 +61,11 @@ public class FileStorProducer implements StorFilestorConfig.Producer { } } - private static StorFilestorConfig.Async_operation_throttler.Type.Enum toAsyncOperationThrottlerType(String throttlerType) { - try { - return StorFilestorConfig.Async_operation_throttler.Type.Enum.valueOf(throttlerType); - } catch (Throwable t) { - return StorFilestorConfig.Async_operation_throttler.Type.UNLIMITED; - } - } - public FileStorProducer(ModelContext.FeatureFlags featureFlags, ContentCluster parent, Integer numThreads) { this.numThreads = numThreads; this.cluster = parent; this.reponseNumThreads = featureFlags.defaultNumResponseThreads(); this.responseSequencerType = convertResponseSequencerType(featureFlags.responseSequencerType()); - this.asyncOperationThrottlerType = toAsyncOperationThrottlerType(featureFlags.persistenceAsyncThrottling()); this.persistenceThrottlingWsDecrementFactor = featureFlags.persistenceThrottlingWsDecrementFactor(); this.persistenceThrottlingWsBackoff = featureFlags.persistenceThrottlingWsBackoff(); this.persistenceThrottingWindowSize = featureFlags.persistenceThrottlingWindowSize(); @@ -93,13 +83,7 @@ public class FileStorProducer implements StorFilestorConfig.Producer { builder.num_response_threads(reponseNumThreads); builder.response_sequencer_type(responseSequencerType); builder.use_async_message_handling_on_schedule(useAsyncMessageHandlingOnSchedule); - // TODO remove deprecated throttler type config - builder.async_operation_throttler_type((asyncOperationThrottlerType == StorFilestorConfig.Async_operation_throttler.Type.DYNAMIC) - ? StorFilestorConfig.Async_operation_throttler_type.Enum.DYNAMIC - : StorFilestorConfig.Async_operation_throttler_type.Enum.UNLIMITED); - var throttleBuilder = new StorFilestorConfig.Async_operation_throttler.Builder(); - throttleBuilder.type(asyncOperationThrottlerType); throttleBuilder.window_size_decrement_factor(persistenceThrottlingWsDecrementFactor); throttleBuilder.window_size_backoff(persistenceThrottlingWsBackoff); if (persistenceThrottingWindowSize > 0) { diff --git a/config-model/src/main/java/com/yahoo/vespa/model/ml/OnnxModelProbe.java b/config-model/src/main/java/com/yahoo/vespa/model/ml/OnnxModelProbe.java index ba5ebdad56a..d2205fb64b3 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/ml/OnnxModelProbe.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/ml/OnnxModelProbe.java @@ -50,7 +50,7 @@ public class OnnxModelProbe { } } catch (IllegalArgumentException | IOException | InterruptedException e) { - e.printStackTrace(System.err); + System.err.println(e.getMessage()); } return outputType; @@ -148,7 +148,8 @@ public class OnnxModelProbe { int returnCode = process.waitFor(); if (returnCode != 0) { - throw new IllegalArgumentException("Error from '" + binary + "'. Return code: " + returnCode + ". Output:\n" + output); + throw new IllegalArgumentException("Error from '" + binary + "'. Return code: " + returnCode + ". " + + "Output: '" + output + "'"); } return output.toString(); } diff --git a/config-model/src/test/java/com/yahoo/config/model/builder/xml/XmlErrorHandlingTest.java b/config-model/src/test/java/com/yahoo/config/model/builder/xml/XmlErrorHandlingTest.java new file mode 100644 index 00000000000..ee616f59d04 --- /dev/null +++ b/config-model/src/test/java/com/yahoo/config/model/builder/xml/XmlErrorHandlingTest.java @@ -0,0 +1,37 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.config.model.builder.xml; + +import org.junit.Test; +import org.xml.sax.InputSource; +import java.io.FileReader; + +import static org.junit.Assert.assertEquals; + +/** + * @author hmusum + */ +public class XmlErrorHandlingTest { + + @Test + public void requireExceptionWithSourceAndFilenameAndLineNumber() { + try { + XmlHelper.getDocument(new FileReader("src/test/cfg/application/invalid-services-syntax/services.xml"), "services.xml"); + } catch (Exception e) { + assertEquals("Invalid XML in services.xml: The element type \"config\" must be terminated by the matching end-tag \"</config>\". [7:5]", + e.getMessage()); + } + } + + + @Test + public void requireExceptionWithLineNumber() { + try { + XmlHelper.getDocumentBuilder().parse( + new InputSource(new FileReader("src/test/cfg/application/invalid-services-syntax/services.xml"))); + } catch (Exception e) { + assertEquals("Invalid XML (unknown source): The element type \"config\" must be terminated by the matching end-tag \"</config>\". [7:5]", + e.getMessage()); + } + } + +} diff --git a/config-model/src/test/java/com/yahoo/vespa/model/builder/UserConfigBuilderTest.java b/config-model/src/test/java/com/yahoo/vespa/model/builder/UserConfigBuilderTest.java index bb3a0f26ee9..114038b884e 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/builder/UserConfigBuilderTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/builder/UserConfigBuilderTest.java @@ -15,7 +15,6 @@ import com.yahoo.vespa.configdefinition.SpecialtokensConfig; import org.junit.Test; import org.w3c.dom.Document; import org.w3c.dom.Element; -import org.xml.sax.InputSource; import java.io.Reader; import java.io.StringReader; @@ -112,7 +111,7 @@ public class UserConfigBuilderTest { Reader xmlReader = new StringReader("<model>" + xml + "</model>"); Document doc; try { - doc = XmlHelper.getDocumentBuilder().parse(new InputSource(xmlReader)); + doc = XmlHelper.getDocument(xmlReader); } catch (Exception e) { throw new RuntimeException(); } diff --git a/config-model/src/test/java/com/yahoo/vespa/model/builder/xml/dom/ContentBuilderTest.java b/config-model/src/test/java/com/yahoo/vespa/model/builder/xml/dom/ContentBuilderTest.java index 7ef3594c844..90d1dba4e4a 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/builder/xml/dom/ContentBuilderTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/builder/xml/dom/ContentBuilderTest.java @@ -847,59 +847,6 @@ public class ContentBuilderTest extends DomBuilderTest { } - @Test - public void feed_task_limit_is_controlled_by_feature_flag() { - assertEquals(1000, resolveFeedTaskLimitConfigWithFeatureFlag(null)); - assertEquals(2000, resolveFeedTaskLimitConfigWithFeatureFlag(2000)); - } - - private int resolveFeedTaskLimitConfigWithFeatureFlag(Integer value) { - var props = new TestProperties(); - if (value != null) { - props.setFeedTaskLimit(value); - } - return resolveProtonConfig(props, singleNodeContentXml()).indexing().tasklimit(); - } - - @Test - public void feed_master_task_limit_is_controlled_by_feature_flag() { - assertEquals(1000, resolveFeedMasterTaskLimitConfigWithFeatureFlag(null)); - assertEquals(2000, resolveFeedMasterTaskLimitConfigWithFeatureFlag(2000)); - } - - private int resolveFeedMasterTaskLimitConfigWithFeatureFlag(Integer value) { - var props = new TestProperties(); - if (value != null) { - props.setFeedMasterTaskLimit(value); - } - return resolveProtonConfig(props, singleNodeContentXml()).feeding().master_task_limit(); - } - - @Test - public void shared_field_writer_executor_is_controlled_by_feature_flag() { - - assertEquals(Shared_field_writer_executor.Enum.NONE, - resolveSharedFieldWriterExecutorConfigWithFeatureFlag(null)); - assertEquals(Shared_field_writer_executor.Enum.NONE, - resolveSharedFieldWriterExecutorConfigWithFeatureFlag("NONE")); - assertEquals(Shared_field_writer_executor.Enum.INDEX, - resolveSharedFieldWriterExecutorConfigWithFeatureFlag("INDEX")); - assertEquals(Shared_field_writer_executor.Enum.INDEX_AND_ATTRIBUTE, - resolveSharedFieldWriterExecutorConfigWithFeatureFlag("INDEX_AND_ATTRIBUTE")); - assertEquals(Shared_field_writer_executor.Enum.DOCUMENT_DB, - resolveSharedFieldWriterExecutorConfigWithFeatureFlag("DOCUMENT_DB")); - assertEquals(Shared_field_writer_executor.Enum.NONE, - resolveSharedFieldWriterExecutorConfigWithFeatureFlag("invalid")); - } - - private ProtonConfig.Feeding.Shared_field_writer_executor.Enum resolveSharedFieldWriterExecutorConfigWithFeatureFlag(String value) { - var props = new TestProperties(); - if (value != null) { - props.setSharedFieldWriterExecutor(value); - } - return resolveProtonConfig(props, singleNodeContentXml()).feeding().shared_field_writer_executor(); - } - private void verifyThatFeatureFlagControlsVisibilityDelayDefault(Double xmlOverride, double expected) { String hostedXml = xmlWithVisibilityDelay(xmlOverride); var config = resolveProtonConfig(new TestProperties(), hostedXml); diff --git a/config-model/src/test/java/com/yahoo/vespa/model/builder/xml/dom/DomConfigPayloadBuilderTest.java b/config-model/src/test/java/com/yahoo/vespa/model/builder/xml/dom/DomConfigPayloadBuilderTest.java index 264d102161a..a20ce425ac0 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/builder/xml/dom/DomConfigPayloadBuilderTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/builder/xml/dom/DomConfigPayloadBuilderTest.java @@ -11,11 +11,9 @@ import com.yahoo.vespa.config.ConfigDefinitionBuilder; import com.yahoo.vespa.config.ConfigDefinitionKey; import com.yahoo.vespa.config.ConfigPayload; import com.yahoo.vespa.config.ConfigPayloadBuilder; - import org.junit.Test; import org.w3c.dom.Document; import org.w3c.dom.Element; -import org.xml.sax.InputSource; import java.io.ByteArrayOutputStream; import java.io.FileNotFoundException; @@ -273,7 +271,7 @@ public class DomConfigPayloadBuilderTest { private Element getDocument(Reader xmlReader) { Document doc; try { - doc = XmlHelper.getDocumentBuilder().parse(new InputSource(xmlReader)); + doc = XmlHelper.getDocument(xmlReader); } catch (Exception e) { throw new RuntimeException(); } diff --git a/config-model/src/test/java/com/yahoo/vespa/model/container/xml/BundleInstantiationSpecificationBuilderTest.java b/config-model/src/test/java/com/yahoo/vespa/model/container/xml/BundleInstantiationSpecificationBuilderTest.java index 909ac9edef2..686f7bbd1f1 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/container/xml/BundleInstantiationSpecificationBuilderTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/container/xml/BundleInstantiationSpecificationBuilderTest.java @@ -8,12 +8,8 @@ import com.yahoo.search.grouping.GroupingValidator; import com.yahoo.vespa.model.container.PlatformBundles; import org.junit.Test; import org.w3c.dom.Element; -import org.xml.sax.SAXException; -import java.io.ByteArrayInputStream; -import java.io.IOException; -import java.io.InputStream; -import java.nio.charset.StandardCharsets; +import java.io.StringReader; import static org.junit.Assert.assertEquals; @@ -25,34 +21,31 @@ import static org.junit.Assert.assertEquals; public class BundleInstantiationSpecificationBuilderTest { @Test - public void bundle_is_not_replaced_for_user_defined_class() throws IOException, SAXException { + public void bundle_is_not_replaced_for_user_defined_class() { final String userDefinedClass = "my own class that will also be set as bundle"; verifyExpectedBundle(userDefinedClass, null, userDefinedClass); } @Test - public void bundle_is_replaced_for_internal_class() throws IOException, SAXException { + public void bundle_is_replaced_for_internal_class() { String internalClass = GroupingValidator.class.getName(); verifyExpectedBundle(internalClass, null, PlatformBundles.searchAndDocprocBundle); } @Test - public void bundle_is_not_replaced_for_internal_class_with_explicitly_set_bundle() - throws IOException, SAXException { + public void bundle_is_not_replaced_for_internal_class_with_explicitly_set_bundle() { String internalClass = GroupingValidator.class.getName(); String explicitBundle = "my-own-implementation"; verifyExpectedBundle(internalClass, explicitBundle, explicitBundle); } - private static void verifyExpectedBundle(String className, String explicitBundle, String expectedBundle) - throws IOException, SAXException { + private static void verifyExpectedBundle(String className, String explicitBundle, String expectedBundle) { String xml = "<component id=\"_\" class=\"" + className + "\""; if (explicitBundle != null) { xml += " bundle=\"" + explicitBundle + "\""; } xml += " />"; - InputStream xmlStream = new ByteArrayInputStream(xml.getBytes(StandardCharsets.UTF_8)); - Element component = XmlHelper.getDocumentBuilder().parse(xmlStream).getDocumentElement(); + Element component = XmlHelper.getDocument(new StringReader(xml)).getDocumentElement(); BundleInstantiationSpecification spec = BundleInstantiationSpecificationBuilder.build(component); assertEquals(ComponentSpecification.fromString(expectedBundle), spec.bundle); diff --git a/config-model/src/test/java/com/yahoo/vespa/model/content/ContentClusterTest.java b/config-model/src/test/java/com/yahoo/vespa/model/content/ContentClusterTest.java index 9f571167d8c..10a2feaba5b 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/content/ContentClusterTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/content/ContentClusterTest.java @@ -1045,28 +1045,6 @@ public class ContentClusterTest extends ContentBaseTest { assertEquals(7, resolveMaxCompactBuffers(OptionalInt.of(7))); } - private ProtonConfig.Replay_throttling_policy.Type.Enum resolveReplayThrottlePolicyType(Optional<String> throttlerType) { - TestProperties testProperties = new TestProperties(); - if (throttlerType.isPresent()) { - testProperties.setPersistenceAsyncThrottling(throttlerType.get()); - } - VespaModel model = createEnd2EndOneNode(testProperties); - ContentCluster cc = model.getContentClusters().get("storage"); - ProtonConfig.Builder protonBuilder = new ProtonConfig.Builder(); - cc.getSearch().getConfig(protonBuilder); - ProtonConfig protonConfig = new ProtonConfig(protonBuilder); - assertEquals(1, protonConfig.documentdb().size()); - return protonConfig.replay_throttling_policy().type(); - } - - @Test - public void replay_throttling_policy_type_controlled_by_properties() { - assertEquals(ProtonConfig.Replay_throttling_policy.Type.Enum.UNLIMITED, resolveReplayThrottlePolicyType(Optional.empty())); - assertEquals(ProtonConfig.Replay_throttling_policy.Type.Enum.UNLIMITED, resolveReplayThrottlePolicyType(Optional.of("UNLIMITED"))); - assertEquals(ProtonConfig.Replay_throttling_policy.Type.Enum.UNLIMITED, resolveReplayThrottlePolicyType(Optional.of("INVALID"))); - assertEquals(ProtonConfig.Replay_throttling_policy.Type.Enum.DYNAMIC, resolveReplayThrottlePolicyType(Optional.of("DYNAMIC"))); - } - private long resolveMaxTLSSize(Optional<Flavor> flavor) throws Exception { TestProperties testProperties = new TestProperties(); @@ -1159,24 +1137,6 @@ public class ContentClusterTest extends ContentBaseTest { } @Test - public void inhibit_default_merges_when_global_merges_pending_controlled_by_properties() throws Exception { - assertFalse(resolveInhibitDefaultMergesConfig(Optional.empty())); - assertFalse(resolveInhibitDefaultMergesConfig(Optional.of(false))); - assertTrue(resolveInhibitDefaultMergesConfig(Optional.of(true))); - } - - private boolean resolveInhibitDefaultMergesConfig(Optional<Boolean> inhibitDefaultMerges) throws Exception { - var props = new TestProperties(); - if (inhibitDefaultMerges.isPresent()) { - props.inhibitDefaultMergesWhenGlobalMergesPending(inhibitDefaultMerges.get()); - } - var cluster = createOneNodeCluster(props); - var builder = new StorDistributormanagerConfig.Builder(); - cluster.getDistributorNodes().getConfig(builder); - return (new StorDistributormanagerConfig(builder)).inhibit_default_merges_when_global_merges_pending(); - } - - @Test public void testDedicatedClusterControllers() { VespaModel noContentModel = createEnd2EndOneNode(new TestProperties().setHostedVespa(true) .setMultitenant(true), diff --git a/config-model/src/test/java/com/yahoo/vespa/model/content/StorageClusterTest.java b/config-model/src/test/java/com/yahoo/vespa/model/content/StorageClusterTest.java index ebc94e1f481..8908ab9f5b9 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/content/StorageClusterTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/content/StorageClusterTest.java @@ -307,31 +307,9 @@ public class StorageClusterTest { } @Test - public void persistence_async_throttle_config_defaults_to_unlimited() { - var config = filestorConfigFromProducer(simpleCluster(new TestProperties())); - assertEquals(StorFilestorConfig.Async_operation_throttler_type.UNLIMITED, config.async_operation_throttler_type()); // TODO remove - assertEquals(StorFilestorConfig.Async_operation_throttler.Type.UNLIMITED, config.async_operation_throttler().type()); - } - - @Test - public void persistence_async_throttle_config_is_derived_from_flag() { - var config = filestorConfigFromProducer(simpleCluster(new TestProperties().setPersistenceAsyncThrottling("UNLIMITED"))); - assertEquals(StorFilestorConfig.Async_operation_throttler_type.UNLIMITED, config.async_operation_throttler_type()); // TODO remove - assertEquals(StorFilestorConfig.Async_operation_throttler.Type.UNLIMITED, config.async_operation_throttler().type()); - - config = filestorConfigFromProducer(simpleCluster(new TestProperties().setPersistenceAsyncThrottling("DYNAMIC"))); - assertEquals(StorFilestorConfig.Async_operation_throttler_type.DYNAMIC, config.async_operation_throttler_type()); // TODO remove - assertEquals(StorFilestorConfig.Async_operation_throttler.Type.DYNAMIC, config.async_operation_throttler().type()); - - // Invalid enum values fall back to the default - config = filestorConfigFromProducer(simpleCluster(new TestProperties().setPersistenceAsyncThrottling("BANANAS"))); - assertEquals(StorFilestorConfig.Async_operation_throttler_type.UNLIMITED, config.async_operation_throttler_type()); // TODO remove - assertEquals(StorFilestorConfig.Async_operation_throttler.Type.UNLIMITED, config.async_operation_throttler().type()); - } - - @Test public void persistence_dynamic_throttling_parameters_have_sane_defaults() { var config = filestorConfigFromProducer(simpleCluster(new TestProperties())); + assertEquals(StorFilestorConfig.Async_operation_throttler.Type.DYNAMIC, config.async_operation_throttler().type()); assertEquals(1.2, config.async_operation_throttler().window_size_decrement_factor(), 0.0001); assertEquals(0.95, config.async_operation_throttler().window_size_backoff(), 0.0001); assertEquals(20, config.async_operation_throttler().min_window_size()); |