diff options
257 files changed, 3256 insertions, 3751 deletions
diff --git a/bundle-plugin/pom.xml b/bundle-plugin/pom.xml index d53c2c94d5c..8f52187357f 100644 --- a/bundle-plugin/pom.xml +++ b/bundle-plugin/pom.xml @@ -19,6 +19,10 @@ </prerequisites> <dependencies> <dependency> + <groupId>com.google.guava</groupId> + <artifactId>guava</artifactId> + </dependency> + <dependency> <groupId>org.apache.maven</groupId> <artifactId>maven-plugin-api</artifactId> <version>3.8.5</version> diff --git a/client/go/cmd/config.go b/client/go/cmd/config.go index 726676ce476..b8a7cb9c24c 100644 --- a/client/go/cmd/config.go +++ b/client/go/cmd/config.go @@ -20,7 +20,6 @@ import ( "github.com/spf13/pflag" "github.com/vespa-engine/vespa/client/go/auth/auth0" "github.com/vespa-engine/vespa/client/go/config" - "github.com/vespa-engine/vespa/client/go/util" "github.com/vespa-engine/vespa/client/go/vespa" ) @@ -433,30 +432,24 @@ func (c *Config) authConfigPath() string { return filepath.Join(c.homeDir, "auth.json") } -func (c *Config) readAPIKey(tenantName string) ([]byte, error) { +func (c *Config) readAPIKey(cli *CLI, system vespa.System, tenantName string) ([]byte, error) { if override, ok := c.apiKeyFromEnv(); ok { return override, nil } - return os.ReadFile(c.apiKeyPath(tenantName)) -} - -// useAPIKey returns true if an API key should be used when authenticating with system. -func (c *Config) useAPIKey(cli *CLI, system vespa.System, tenantName string) bool { - if _, ok := c.apiKeyFromEnv(); ok { - return true + if path, ok := c.apiKeyFileFromEnv(); ok { + return os.ReadFile(path) } - if _, ok := c.apiKeyFileFromEnv(); ok { - return true + if cli.isCloudCI() { + return nil, nil // Vespa Cloud CI only talks to data plane and does not have an API key } if !cli.isCI() { - // Fall back to API key, if present and Auth0 has not been configured client, err := auth0.New(c.authConfigPath(), system.Name, system.URL) - if err != nil || !client.HasCredentials() { - cli.printWarning("Regular authentication is preferred over API key in a non-CI context", "Authenticate with 'vespa auth login'") - return util.PathExists(c.apiKeyPath(tenantName)) + if err == nil && client.HasCredentials() { + return nil, nil // use Auth0 } + cli.printWarning("Authenticating with API key. This is discouraged in non-CI environments", "Authenticate with 'vespa auth login'") } - return false + return os.ReadFile(c.apiKeyPath(tenantName)) } func (c *Config) readSessionID(app vespa.ApplicationID) (int64, error) { diff --git a/client/go/cmd/config_test.go b/client/go/cmd/config_test.go index f89e752f82d..86c7e2695fa 100644 --- a/client/go/cmd/config_test.go +++ b/client/go/cmd/config_test.go @@ -145,17 +145,39 @@ func assertConfigCommandErr(t *testing.T, configHome, expected string, args ...s assert.NotNil(t, assertConfigCommandStdErr(t, configHome, expected, args...)) } -func TestUseAPIKey(t *testing.T) { +func TestReadAPIKey(t *testing.T) { cli, _, _ := newTestCLI(t) - assert.False(t, cli.config.useAPIKey(cli, vespa.PublicSystem, "t1")) + key, err := cli.config.readAPIKey(cli, vespa.PublicSystem, "t1") + assert.Nil(t, key) + require.NotNil(t, err) - cli, _, _ = newTestCLI(t, "VESPA_CLI_API_KEY_FILE=/tmp/foo") - assert.True(t, cli.config.useAPIKey(cli, vespa.PublicSystem, "t1")) + // From default path when it exists + require.Nil(t, os.WriteFile(filepath.Join(cli.config.homeDir, "t1.api-key.pem"), []byte("foo"), 0600)) + key, err = cli.config.readAPIKey(cli, vespa.PublicSystem, "t1") + require.Nil(t, err) + assert.Equal(t, []byte("foo"), key) + + // Cloud CI does not read key from disk as it's not expected to have any + cli, _, _ = newTestCLI(t, "VESPA_CLI_CLOUD_CI=true") + key, err = cli.config.readAPIKey(cli, vespa.PublicSystem, "t1") + require.Nil(t, err) + assert.Nil(t, key) + + // From file specified in environment + keyFile := filepath.Join(t.TempDir(), "key") + require.Nil(t, os.WriteFile(keyFile, []byte("bar"), 0600)) + cli, _, _ = newTestCLI(t, "VESPA_CLI_API_KEY_FILE="+keyFile) + key, err = cli.config.readAPIKey(cli, vespa.PublicSystem, "t1") + require.Nil(t, err) + assert.Equal(t, []byte("bar"), key) - cli, _, _ = newTestCLI(t, "VESPA_CLI_API_KEY=foo") - assert.True(t, cli.config.useAPIKey(cli, vespa.PublicSystem, "t1")) + // From key specified in environment + cli, _, _ = newTestCLI(t, "VESPA_CLI_API_KEY=baz") + key, err = cli.config.readAPIKey(cli, vespa.PublicSystem, "t1") + require.Nil(t, err) + assert.Equal(t, []byte("baz"), key) - // Prefer Auth0, if configured + // Auth0 is preferred when configured authContent := ` { "version": 1, @@ -172,10 +194,9 @@ func TestUseAPIKey(t *testing.T) { } } }` - cli, _, _ = newTestCLI(t, "VESPA_CLI_CLOUD_SYSTEM=public") - _, err := os.Create(filepath.Join(cli.config.homeDir, "t2.api-key.pem")) - require.Nil(t, err) - assert.True(t, cli.config.useAPIKey(cli, vespa.PublicSystem, "t2")) + cli, _, _ = newTestCLI(t) require.Nil(t, os.WriteFile(filepath.Join(cli.config.homeDir, "auth.json"), []byte(authContent), 0600)) - assert.False(t, cli.config.useAPIKey(cli, vespa.PublicSystem, "t2")) + key, err = cli.config.readAPIKey(cli, vespa.PublicSystem, "t1") + require.Nil(t, err) + assert.Nil(t, key) } diff --git a/client/go/cmd/root.go b/client/go/cmd/root.go index e88398a7fde..0557248cedf 100644 --- a/client/go/cmd/root.go +++ b/client/go/cmd/root.go @@ -324,11 +324,9 @@ func (c *CLI) createCloudTarget(targetType string, opts targetOptions) (vespa.Ta ) switch targetType { case vespa.TargetCloud: - if c.config.useAPIKey(c, system, deployment.Application.Tenant) { - apiKey, err = c.config.readAPIKey(deployment.Application.Tenant) - if err != nil { - return nil, err - } + apiKey, err = c.config.readAPIKey(c, system, deployment.Application.Tenant) + if err != nil { + return nil, err } authConfigPath = c.config.authConfigPath() deploymentTLSOptions = vespa.TLSOptions{} diff --git a/config-application-package/src/main/java/com/yahoo/config/model/application/provider/FilesApplicationPackage.java b/config-application-package/src/main/java/com/yahoo/config/model/application/provider/FilesApplicationPackage.java index 36ebb621475..0579aebe771 100644 --- a/config-application-package/src/main/java/com/yahoo/config/model/application/provider/FilesApplicationPackage.java +++ b/config-application-package/src/main/java/com/yahoo/config/model/application/provider/FilesApplicationPackage.java @@ -481,12 +481,13 @@ public class FilesApplicationPackage extends AbstractApplicationPackage { } private static ApplicationMetaData readMetaData(File appDir) { + String originalAppDir = preprocessed.equals(appDir.getName()) ? appDir.getParentFile().getName() : appDir.getName(); ApplicationMetaData defaultMetaData = new ApplicationMetaData("n/a", "n/a", 0L, false, ApplicationId.from(TenantName.defaultName(), - ApplicationName.from(appDir.getName()), + ApplicationName.from(originalAppDir), InstanceName.defaultName()), "", 0L, diff --git a/config-model/src/main/java/com/yahoo/config/model/ConfigModelRepo.java b/config-model/src/main/java/com/yahoo/config/model/ConfigModelRepo.java index 0e0f992952a..8985dac91aa 100644 --- a/config-model/src/main/java/com/yahoo/config/model/ConfigModelRepo.java +++ b/config-model/src/main/java/com/yahoo/config/model/ConfigModelRepo.java @@ -4,27 +4,16 @@ package com.yahoo.config.model; import com.yahoo.config.application.api.ApplicationFile; import com.yahoo.config.application.api.ApplicationPackage; import com.yahoo.config.model.ConfigModelContext.ApplicationType; -import com.yahoo.config.model.deploy.DeployState; import com.yahoo.config.model.builder.xml.ConfigModelBuilder; import com.yahoo.config.model.builder.xml.ConfigModelId; import com.yahoo.config.model.builder.xml.XmlHelper; +import com.yahoo.config.model.deploy.DeployState; import com.yahoo.config.model.graph.ModelGraphBuilder; import com.yahoo.config.model.graph.ModelNode; +import com.yahoo.config.model.producer.AbstractConfigProducer; import com.yahoo.config.model.provision.HostsXmlProvisioner; - -import java.util.ArrayList; -import java.util.Collection; -import java.util.Collections; -import java.util.Iterator; -import java.util.LinkedHashMap; -import java.util.List; -import java.util.Map; -import java.util.Optional; -import java.util.TreeMap; -import java.util.logging.Level; import com.yahoo.path.Path; import com.yahoo.text.XML; -import com.yahoo.config.model.producer.AbstractConfigProducer; import com.yahoo.vespa.model.VespaModel; import com.yahoo.vespa.model.builder.VespaModelBuilder; import com.yahoo.vespa.model.clients.Clients; @@ -32,13 +21,22 @@ import com.yahoo.vespa.model.content.Content; import com.yahoo.vespa.model.routing.Routing; import org.w3c.dom.Document; import org.w3c.dom.Element; -import org.xml.sax.InputSource; import org.xml.sax.SAXException; import java.io.IOException; import java.io.Reader; import java.io.Serializable; import java.io.StringReader; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.Iterator; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.TreeMap; +import java.util.logging.Level; import java.util.logging.Logger; /** @@ -91,7 +89,7 @@ public class ConfigModelRepo implements ConfigModelRepoAdder, Serializable, Iter builder.postProc(deployState.getDeployLogger(), root, this); } - private Element getServicesFromApp(ApplicationPackage applicationPackage) throws IOException, SAXException { + private Element getServicesFromApp(ApplicationPackage applicationPackage) throws IOException { try (Reader servicesFile = applicationPackage.getServices()) { return getServicesFromReader(servicesFile); } @@ -188,8 +186,8 @@ public class ConfigModelRepo implements ConfigModelRepoAdder, Serializable, Iter return permanentServices; } - private Element getServicesFromReader(Reader reader) throws IOException, SAXException { - Document doc = XmlHelper.getDocumentBuilder().parse(new InputSource(reader)); + private Element getServicesFromReader(Reader reader) { + Document doc = XmlHelper.getDocument(reader); return doc.getDocumentElement(); } @@ -274,10 +272,10 @@ public class ConfigModelRepo implements ConfigModelRepoAdder, Serializable, Iter } // TODO: Doctoring on the XML is the wrong level for this. We should be able to mark a model as default instead -Jon - private static Element getImplicitAdmin(DeployState deployState) throws IOException, SAXException { + private static Element getImplicitAdmin(DeployState deployState) { String defaultAdminElement = deployState.isHosted() ? getImplicitAdminV4() : getImplicitAdminV2(); log.log(Level.FINE, () -> "No <admin> defined, using " + defaultAdminElement); - return XmlHelper.getDocumentBuilder().parse(new InputSource(new StringReader(defaultAdminElement))).getDocumentElement(); + return XmlHelper.getDocument(new StringReader(defaultAdminElement)).getDocumentElement(); } private static String getImplicitAdminV2() { diff --git a/config-model/src/main/java/com/yahoo/config/model/builder/xml/XmlHelper.java b/config-model/src/main/java/com/yahoo/config/model/builder/xml/XmlHelper.java index 5763be301a6..220d6bbb768 100644 --- a/config-model/src/main/java/com/yahoo/config/model/builder/xml/XmlHelper.java +++ b/config-model/src/main/java/com/yahoo/config/model/builder/xml/XmlHelper.java @@ -3,12 +3,14 @@ package com.yahoo.config.model.builder.xml; import com.yahoo.component.ComponentId; import com.yahoo.component.ComponentSpecification; -import java.util.logging.Level; import com.yahoo.text.XML; +import com.yahoo.yolean.Exceptions; import org.w3c.dom.Document; import org.w3c.dom.Element; +import org.xml.sax.ErrorHandler; import org.xml.sax.InputSource; import org.xml.sax.SAXException; +import org.xml.sax.SAXParseException; import javax.xml.parsers.DocumentBuilder; import javax.xml.parsers.DocumentBuilderFactory; @@ -19,6 +21,7 @@ import java.util.ArrayList; import java.util.Collection; import java.util.List; import java.util.Optional; +import java.util.logging.Level; import java.util.logging.Logger; @@ -74,9 +77,15 @@ public final class XmlHelper { } public static Document getDocument(Reader reader) { + return getDocument(reader, "unknown source"); + } + + public static Document getDocument(Reader reader, String source) { Document doc; try { - doc = getDocumentBuilder().parse(new InputSource(reader)); + InputSource inputSource = new InputSource(reader); + inputSource.setPublicId(source); + doc = getDocumentBuilder().parse(inputSource); } catch (SAXException | IOException e) { throw new IllegalArgumentException(e); } @@ -121,6 +130,7 @@ public final class XmlHelper { public static synchronized DocumentBuilder getDocumentBuilder() { try { DocumentBuilder docBuilder = factory.newDocumentBuilder(); + docBuilder.setErrorHandler(new CustomErrorHandler(log)); log.log(Level.FINE, "XML parser now operational!"); return docBuilder; } catch (ParserConfigurationException e) { @@ -144,4 +154,34 @@ public final class XmlHelper { if (child.getFirstChild() == null) return Optional.empty(); return Optional.ofNullable(child.getFirstChild().getNodeValue()); } + + /** Error handler which will output name of source for warnings and errors */ + private static class CustomErrorHandler implements ErrorHandler { + + private final Logger logger; + + CustomErrorHandler(Logger logger) { + super(); + this.logger = logger; + } + + public void warning(SAXParseException e) { + logger.log(Level.WARNING, message(e)); + } + + public void error(SAXParseException e) { + throw new IllegalArgumentException(message(e)); + } + + public void fatalError(SAXParseException e) { throw new IllegalArgumentException(message(e)); } + + private String message(SAXParseException e) { + String sourceId = e.getPublicId() == null ? "" : e.getPublicId(); + return "Invalid XML" + (sourceId.isEmpty() ? " (unknown source)" : " in " + sourceId) + + ": " + Exceptions.toMessageString(e) + + " [" + e.getLineNumber() + ":" + e.getColumnNumber() + "]"; + } + + } + } diff --git a/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java b/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java index 8366dde383b..86f1a9eb9f4 100644 --- a/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java +++ b/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java @@ -42,9 +42,6 @@ public class TestProperties implements ModelContext.Properties, ModelContext.Fea private double defaultTermwiseLimit = 1.0; private String jvmGCOptions = null; private String sequencerType = "THROUGHPUT"; - private int feedTaskLimit = 1000; - private int feedMasterTaskLimit = 1000; - private String sharedFieldWriterExecutor = "NONE"; private boolean firstTimeDeployment = false; private String responseSequencerType = "ADAPTIVE"; private int responseNumThreads = 2; @@ -70,14 +67,12 @@ public class TestProperties implements ModelContext.Properties, ModelContext.Fea private List<String> zoneDnsSuffixes = List.of(); private int maxCompactBuffers = 1; private boolean failDeploymentWithInvalidJvmOptions = false; - private String persistenceAsyncThrottling = "UNLIMITED"; private String mergeThrottlingPolicy = "STATIC"; private double persistenceThrottlingWsDecrementFactor = 1.2; private double persistenceThrottlingWsBackoff = 0.95; private int persistenceThrottlingWindowSize = -1; private double persistenceThrottlingWsResizeRate = 3.0; private boolean persistenceThrottlingOfMergeFeedOps = true; - private boolean inhibitDefaultMergesWhenGlobalMergesPending = false; private boolean useV8GeoPositions = false; private List<String> environmentVariables = List.of(); private boolean avoidRenamingSummaryFeatures = false; @@ -96,9 +91,6 @@ public class TestProperties implements ModelContext.Properties, ModelContext.Fea @Override public Set<ContainerEndpoint> endpoints() { return endpoints; } @Override public String jvmGCOptions(Optional<ClusterSpec.Type> clusterType) { return jvmGCOptions; } @Override public String feedSequencerType() { return sequencerType; } - @Override public int feedTaskLimit() { return feedTaskLimit; } - @Override public int feedMasterTaskLimit() { return feedMasterTaskLimit; } - @Override public String sharedFieldWriterExecutor() { return sharedFieldWriterExecutor; } @Override public boolean isBootstrap() { return false; } @Override public boolean isFirstTimeDeployment() { return firstTimeDeployment; } @Override public boolean useDedicatedNodeForLogserver() { return useDedicatedNodeForLogserver; } @@ -131,14 +123,12 @@ public class TestProperties implements ModelContext.Properties, ModelContext.Fea @Override public List<String> zoneDnsSuffixes() { return zoneDnsSuffixes; } @Override public int maxCompactBuffers() { return maxCompactBuffers; } @Override public boolean failDeploymentWithInvalidJvmOptions() { return failDeploymentWithInvalidJvmOptions; } - @Override public String persistenceAsyncThrottling() { return persistenceAsyncThrottling; } @Override public String mergeThrottlingPolicy() { return mergeThrottlingPolicy; } @Override public double persistenceThrottlingWsDecrementFactor() { return persistenceThrottlingWsDecrementFactor; } @Override public double persistenceThrottlingWsBackoff() { return persistenceThrottlingWsBackoff; } @Override public int persistenceThrottlingWindowSize() { return persistenceThrottlingWindowSize; } @Override public double persistenceThrottlingWsResizeRate() { return persistenceThrottlingWsResizeRate; } @Override public boolean persistenceThrottlingOfMergeFeedOps() { return persistenceThrottlingOfMergeFeedOps; } - @Override public boolean inhibitDefaultMergesWhenGlobalMergesPending() { return inhibitDefaultMergesWhenGlobalMergesPending; } @Override public boolean useV8GeoPositions() { return useV8GeoPositions; } @Override public List<String> environmentVariables() { return environmentVariables; } @Override public boolean avoidRenamingSummaryFeatures() { return this.avoidRenamingSummaryFeatures; } @@ -177,18 +167,6 @@ public class TestProperties implements ModelContext.Properties, ModelContext.Fea sequencerType = type; return this; } - public TestProperties setFeedTaskLimit(int value) { - feedTaskLimit = value; - return this; - } - public TestProperties setFeedMasterTaskLimit(int value) { - feedMasterTaskLimit = value; - return this; - } - public TestProperties setSharedFieldWriterExecutor(String value) { - sharedFieldWriterExecutor = value; - return this; - } public TestProperties setResponseSequencerType(String type) { responseSequencerType = type; return this; @@ -326,11 +304,6 @@ public class TestProperties implements ModelContext.Properties, ModelContext.Fea return this; } - public TestProperties setPersistenceAsyncThrottling(String type) { - this.persistenceAsyncThrottling = type; - return this; - } - public TestProperties setMergeThrottlingPolicy(String policy) { this.mergeThrottlingPolicy = policy; return this; @@ -361,11 +334,6 @@ public class TestProperties implements ModelContext.Properties, ModelContext.Fea return this; } - public TestProperties inhibitDefaultMergesWhenGlobalMergesPending(boolean value) { - this.inhibitDefaultMergesWhenGlobalMergesPending = value; - return this; - } - public TestProperties setUseV8GeoPositions(boolean value) { this.useV8GeoPositions = value; return this; diff --git a/config-model/src/main/java/com/yahoo/config/model/provision/Hosts.java b/config-model/src/main/java/com/yahoo/config/model/provision/Hosts.java index c9a03dad65e..2ef8cb4a0bf 100644 --- a/config-model/src/main/java/com/yahoo/config/model/provision/Hosts.java +++ b/config-model/src/main/java/com/yahoo/config/model/provision/Hosts.java @@ -8,12 +8,13 @@ import com.yahoo.text.XML; import com.yahoo.vespa.model.builder.xml.dom.VespaDomBuilder; import org.w3c.dom.Document; import org.w3c.dom.Element; -import org.xml.sax.InputSource; -import org.xml.sax.SAXException; -import java.io.IOException; import java.io.Reader; -import java.util.*; +import java.util.ArrayList; +import java.util.Collection; +import java.util.HashSet; +import java.util.List; +import java.util.Set; import java.util.logging.Logger; /** @@ -63,12 +64,7 @@ public class Hosts { */ public static Hosts readFrom(Reader hostsFile) { List<Host> hosts = new ArrayList<>(); - Document doc; - try { - doc = XmlHelper.getDocumentBuilder().parse(new InputSource(hostsFile)); - } catch (SAXException | IOException e) { - throw new IllegalArgumentException(e); - } + Document doc = XmlHelper.getDocument(hostsFile); for (Element hostE : XML.getChildren(doc.getDocumentElement(), "host")) { String name = hostE.getAttribute("name"); if (name.equals("")) { diff --git a/config-model/src/main/java/com/yahoo/config/model/test/MockRoot.java b/config-model/src/main/java/com/yahoo/config/model/test/MockRoot.java index b29ff1ee58b..7e2953e6606 100644 --- a/config-model/src/main/java/com/yahoo/config/model/test/MockRoot.java +++ b/config-model/src/main/java/com/yahoo/config/model/test/MockRoot.java @@ -19,10 +19,7 @@ import com.yahoo.vespa.model.builder.xml.dom.DomAdminV2Builder; import com.yahoo.vespa.model.filedistribution.FileDistributionConfigProducer; import com.yahoo.vespa.model.filedistribution.FileReferencesRepository; import org.w3c.dom.Document; -import org.xml.sax.InputSource; -import org.xml.sax.SAXException; -import java.io.IOException; import java.io.StringReader; import java.util.ArrayList; import java.util.Collections; @@ -143,13 +140,9 @@ public class MockRoot extends AbstractConfigProducerRoot { "<?xml version='1.0' encoding='utf-8' ?>" + "<services>" + xml + "</services>"; - try { - Document doc = XmlHelper.getDocumentBuilder().parse(new InputSource(new StringReader(servicesXml))); - setAdmin(new DomAdminV2Builder(ConfigModelContext.ApplicationType.DEFAULT, false, new ArrayList<>()). - build(deployState, this, XML.getChildren(doc.getDocumentElement(), "admin").get(0))); - } catch (SAXException | IOException e) { - throw new RuntimeException(e); - } + Document doc = XmlHelper.getDocument(new StringReader(servicesXml)); + setAdmin(new DomAdminV2Builder(ConfigModelContext.ApplicationType.DEFAULT, false, new ArrayList<>()) + .build(deployState, this, XML.getChildren(doc.getDocumentElement(), "admin").get(0))); } public final void setAdmin(Admin admin) { diff --git a/config-model/src/main/java/com/yahoo/config/model/test/TestUtil.java b/config-model/src/main/java/com/yahoo/config/model/test/TestUtil.java index df916907472..c05d7bf4942 100644 --- a/config-model/src/main/java/com/yahoo/config/model/test/TestUtil.java +++ b/config-model/src/main/java/com/yahoo/config/model/test/TestUtil.java @@ -25,9 +25,7 @@ public class TestUtil { lines.addAll(Arrays.asList(xmlLines)); try { - return XmlHelper.getDocumentBuilder().parse( - inputSource((CollectionUtil.mkString(lines, "\n").replace("'", "\"")))) - .getDocumentElement(); + return XmlHelper.getDocument(new StringReader(CollectionUtil.mkString(lines, "\n").replace("'", "\""))).getDocumentElement(); } catch (Exception e) { throw new RuntimeException(e); } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/VespaModelFactory.java b/config-model/src/main/java/com/yahoo/vespa/model/VespaModelFactory.java index f2d0ab03e27..25e2a7593b0 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/VespaModelFactory.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/VespaModelFactory.java @@ -119,14 +119,12 @@ public class VespaModelFactory implements ModelFactory { for (ConfigChangeAction action : changeActions) { if (action.getType().equals(ConfigChangeAction.Type.REINDEX)) { VespaModel currentModel = (VespaModel) currentActiveModel.get(); - var currentVersion = currentModel.version(); var currentMeta = currentModel.applicationPackage().getMetaData(); - var nextVersion = nextModel.version(); var nextMeta = nextModel.applicationPackage().getMetaData(); log.log(Level.INFO, String.format("Model [%s/%s] -> [%s/%s] triggers reindexing: %s", currentModel.version().toString(), currentMeta.toString(), nextModel.version().toString(), nextMeta.toString(), - action.toString())); + action)); } } } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/VespaMetricSet.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/VespaMetricSet.java index 34586df424f..3a8cf23a49e 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/VespaMetricSet.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/VespaMetricSet.java @@ -154,6 +154,17 @@ public class VespaMetricSet { addMetric(metrics, "jdisc.thread_pool.unhandled_exceptions", suffixes); addMetric(metrics, "jdisc.thread_pool.work_queue.capacity", suffixes); addMetric(metrics, "jdisc.thread_pool.work_queue.size", suffixes); + addMetric(metrics, "jdisc.thread_pool.rejected_tasks", suffixes); + addMetric(metrics, "jdisc.thread_pool.size", suffixes); + addMetric(metrics, "jdisc.thread_pool.max_allowed_size", suffixes); + addMetric(metrics, "jdisc.thread_pool.active_threads", suffixes); + + addMetric(metrics, "jdisc.http.jetty.threadpool.thread.max", suffixes); + addMetric(metrics, "jdisc.http.jetty.threadpool.thread.min", suffixes); + addMetric(metrics, "jdisc.http.jetty.threadpool.thread.reserved", suffixes); + addMetric(metrics, "jdisc.http.jetty.threadpool.thread.busy", suffixes); + addMetric(metrics, "jdisc.http.jetty.threadpool.thread.total", suffixes); + addMetric(metrics, "jdisc.http.jetty.threadpool.queue.size", suffixes); } metrics.add(new Metric("httpapi_latency.max")); @@ -223,12 +234,6 @@ public class VespaMetricSet { metrics.add(new Metric("jdisc.http.handler.unhandled_exceptions.rate")); - addMetric(metrics, "jdisc.http.jetty.threadpool.thread.max", List.of("last")); - addMetric(metrics, "jdisc.http.jetty.threadpool.thread.reserved", List.of("last")); - addMetric(metrics, "jdisc.http.jetty.threadpool.thread.busy", List.of("sum", "count", "min", "max")); - addMetric(metrics, "jdisc.http.jetty.threadpool.thread.total", List.of("sum", "count", "min", "max")); - addMetric(metrics, "jdisc.http.jetty.threadpool.queue.size", List.of("sum", "count", "min", "max")); - addMetric(metrics, "jdisc.http.filtering.request.handled", List.of("rate")); addMetric(metrics, "jdisc.http.filtering.request.unhandled", List.of("rate")); addMetric(metrics, "jdisc.http.filtering.response.handled", List.of("rate")); diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/RankSetupValidator.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/RankSetupValidator.java index 6a2ccec30c3..ed943317a1f 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/RankSetupValidator.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/RankSetupValidator.java @@ -22,11 +22,10 @@ import com.yahoo.vespa.config.search.core.RankingConstantsConfig; import com.yahoo.vespa.config.search.core.RankingExpressionsConfig; import com.yahoo.vespa.defaults.Defaults; import com.yahoo.vespa.model.VespaModel; -import com.yahoo.vespa.model.search.SearchCluster; import com.yahoo.vespa.model.search.DocumentDatabase; import com.yahoo.vespa.model.search.IndexedSearchCluster; +import com.yahoo.vespa.model.search.SearchCluster; import com.yahoo.yolean.Exceptions; - import java.io.File; import java.io.IOException; import java.nio.file.Files; @@ -64,7 +63,6 @@ public class RankSetupValidator extends Validator { deployState.getProperties().applicationId().toFullString() + ".") .toFile(); - for (SearchCluster cluster : model.getSearchClusters()) { // Skipping rank expression checking for streaming clusters, not implemented yet if (cluster.isStreaming()) continue; @@ -72,16 +70,15 @@ public class RankSetupValidator extends Validator { IndexedSearchCluster sc = (IndexedSearchCluster) cluster; String clusterDir = cfgDir.getAbsolutePath() + "/" + sc.getClusterName() + "/"; for (DocumentDatabase docDb : sc.getDocumentDbs()) { - final String name = docDb.getDerivedConfiguration().getSchema().getName(); - String searchDir = clusterDir + name + "/"; - writeConfigs(searchDir, docDb); - writeExtraVerifyRanksetupConfig(searchDir, docDb); - if (!validate("dir:" + searchDir, sc, name, deployState.getDeployLogger(), cfgDir)) { + String schemaName = docDb.getDerivedConfiguration().getSchema().getName(); + String schemaDir = clusterDir + schemaName + "/"; + writeConfigs(schemaDir, docDb); + writeExtraVerifyRankSetupConfig(schemaDir, docDb); + if (!validate("dir:" + schemaDir, sc, schemaName, deployState.getDeployLogger(), cfgDir)) { return; } } } - } catch (IOException e) { throw new RuntimeException(e); } finally { @@ -90,13 +87,13 @@ public class RankSetupValidator extends Validator { } } - private boolean validate(String configId, SearchCluster searchCluster, String sdName, DeployLogger deployLogger, File tempDir) { + private boolean validate(String configId, SearchCluster searchCluster, String schema, DeployLogger deployLogger, File tempDir) { Instant start = Instant.now(); try { - log.log(Level.FINE, () -> String.format("Validating schema '%s' for %s with config id %s", sdName, searchCluster, configId)); - boolean ret = execValidate(configId, searchCluster, sdName, deployLogger); + log.log(Level.FINE, () -> String.format("Validating schema '%s' for cluster %s with config id %s", schema, searchCluster, configId)); + boolean ret = execValidate(configId, searchCluster, schema, deployLogger); if (!ret) { - // Give up, don't say same error msg repeatedly + // Give up, don't log same error msg repeatedly deleteTempDir(tempDir); } log.log(Level.FINE, () -> String.format("Validation took %s ms", Duration.between(start, Instant.now()).toMillis())); @@ -141,7 +138,7 @@ public class RankSetupValidator extends Validator { writeConfig(dir, ImportedFieldsConfig.getDefName() + ".cfg", ifcb.build()); } - private void writeExtraVerifyRanksetupConfig(List<String> config, Collection<? extends DistributableResource> resources) { + private void writeExtraVerifyRankSetupConfig(List<String> config, Collection<? extends DistributableResource> resources) { for (DistributableResource model : resources) { String modelPath = getFileRepositoryPath(model.getFilePath().getName(), model.getFileReference()); int index = config.size() / 2; @@ -151,12 +148,12 @@ public class RankSetupValidator extends Validator { } } - private void writeExtraVerifyRanksetupConfig(String dir, DocumentDatabase db) throws IOException { + private void writeExtraVerifyRankSetupConfig(String dir, DocumentDatabase db) throws IOException { List<String> config = new ArrayList<>(); // Assist verify-ranksetup in finding the actual ONNX model files - writeExtraVerifyRanksetupConfig(config, db.getDerivedConfiguration().getSchema().onnxModels().asMap().values()); - writeExtraVerifyRanksetupConfig(config, db.getDerivedConfiguration().getSchema().rankExpressionFiles().asMap().values()); + writeExtraVerifyRankSetupConfig(config, db.getDerivedConfiguration().getSchema().onnxModels().asMap().values()); + writeExtraVerifyRankSetupConfig(config, db.getDerivedConfiguration().getSchema().rankExpressionFiles().asMap().values()); String configContent = config.isEmpty() ? "" : StringUtilities.implodeMultiline(config); IOUtils.writeFile(dir + "verify-ranksetup.cfg", configContent, false); @@ -173,12 +170,13 @@ public class RankSetupValidator extends Validator { } private boolean execValidate(String configId, SearchCluster sc, String sdName, DeployLogger deployLogger) { - String job = String.format("%s %s", binaryName, configId); - ProcessExecuter executer = new ProcessExecuter(true); + String command = String.format("%s %s", binaryName, configId); try { - Pair<Integer, String> ret = executer.exec(job); - if (ret.getFirst() != 0) { - validateFail(ret.getSecond(), sc, sdName, deployLogger); + Pair<Integer, String> ret = new ProcessExecuter(true).exec(command); + Integer exitCode = ret.getFirst(); + String output = ret.getSecond(); + if (exitCode != 0) { + validateFail(output, exitCode, sc, sdName, deployLogger); } } catch (IOException e) { validateWarn(e, deployLogger); @@ -194,23 +192,32 @@ public class RankSetupValidator extends Validator { deployLogger.logApplicationPackage(Level.WARNING, msg); } - private void validateFail(String output, SearchCluster sc, String sdName, DeployLogger deployLogger) { - StringBuilder errMsg = new StringBuilder("Error in rank setup in schema '").append(sdName) + private void validateFail(String output, int exitCode, SearchCluster sc, String sdName, DeployLogger deployLogger) { + StringBuilder message = new StringBuilder("Error in rank setup in schema '").append(sdName) .append("' for content cluster '").append(sc.getClusterName()).append("'.").append(" Details:\n"); - for (String line : output.split("\n")) { - // Remove debug lines from start script - if (line.startsWith("debug\t")) continue; - try { - LogMessage logMessage = LogMessage.parseNativeFormat(line); - errMsg.append(logMessage.getLevel()).append(": ").append(logMessage.getPayload()).append("\n"); - } catch (InvalidLogFormatException e) { - errMsg.append(line).append("\n"); + if (output.isEmpty()) { + message.append("Verifying rank setup failed and got no output from stderr and stdout from '") + .append(binaryName) + .append("' (exit code: ") + .append(exitCode) + .append("). This could be due to full disk, out of memory etc."); + } else { + for (String line : output.split("\n")) { + // Remove debug lines from start script + if (line.startsWith("debug\t")) continue; + try { + LogMessage logMessage = LogMessage.parseNativeFormat(line); + message.append(logMessage.getLevel()).append(": ").append(logMessage.getPayload()).append("\n"); + } catch (InvalidLogFormatException e) { + message.append(line).append("\n"); + } } } + if (ignoreValidationErrors) { - deployLogger.log(Level.WARNING, errMsg.append("(Continuing since ignoreValidationErrors flag is set.)").toString()); + deployLogger.log(Level.WARNING, message.append("(Continuing since ignoreValidationErrors flag is set.)").toString()); } else { - throw new IllegalArgumentException(errMsg.toString()); + throw new IllegalArgumentException(message.toString()); } } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/VespaDomBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/VespaDomBuilder.java index df77d83da83..b619210155f 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/VespaDomBuilder.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/VespaDomBuilder.java @@ -4,11 +4,10 @@ package com.yahoo.vespa.model.builder.xml.dom; import com.yahoo.config.application.api.DeployLogger; import com.yahoo.config.model.ApplicationConfigProducerRoot; import com.yahoo.config.model.ConfigModelRepo; -import com.yahoo.config.model.deploy.DeployState; import com.yahoo.config.model.builder.xml.XmlHelper; +import com.yahoo.config.model.deploy.DeployState; import com.yahoo.config.model.producer.AbstractConfigProducer; import com.yahoo.config.model.producer.UserConfigRepo; -import java.util.logging.Level; import com.yahoo.text.XML; import com.yahoo.vespa.model.AbstractService; import com.yahoo.vespa.model.Affinity; @@ -24,7 +23,6 @@ import com.yahoo.vespa.model.content.Content; import com.yahoo.vespa.model.generic.builder.DomServiceClusterBuilder; import com.yahoo.vespa.model.generic.service.ServiceCluster; import com.yahoo.vespa.model.search.SearchCluster; - import org.w3c.dom.Document; import org.w3c.dom.Element; import org.w3c.dom.Node; @@ -33,6 +31,7 @@ import org.w3c.dom.NodeList; import java.util.ArrayList; import java.util.LinkedList; import java.util.List; +import java.util.logging.Level; import java.util.logging.Logger; /** @@ -94,7 +93,8 @@ public class VespaDomBuilder extends VespaModelBuilder { public ApplicationConfigProducerRoot getRoot(String name, DeployState deployState, AbstractConfigProducer parent) { try { return new DomRootBuilder(name). - build(deployState, parent, XmlHelper.getDocument(deployState.getApplicationPackage().getServices()).getDocumentElement()); + build(deployState, parent, XmlHelper.getDocument(deployState.getApplicationPackage().getServices(), "services.xml") + .getDocumentElement()); } catch (Exception e) { throw new IllegalArgumentException(e); } @@ -303,7 +303,7 @@ public class VespaDomBuilder extends VespaModelBuilder { @Override public List<ServiceCluster> getClusters(DeployState deployState, AbstractConfigProducer parent) { List<ServiceCluster> clusters = new ArrayList<>(); - Document services = XmlHelper.getDocument(deployState.getApplicationPackage().getServices()); + Document services = XmlHelper.getDocument(deployState.getApplicationPackage().getServices(), "services.xml"); for (Element clusterSpec : XML.getChildren(services.getDocumentElement(), "cluster")) { DomServiceClusterBuilder clusterBuilder = new DomServiceClusterBuilder(clusterSpec.getAttribute("name")); clusters.add(clusterBuilder.build(deployState, parent.getRoot(), clusterSpec)); diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerThreadpool.java b/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerThreadpool.java index 7111a88fc01..489e4cc135a 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerThreadpool.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerThreadpool.java @@ -4,7 +4,7 @@ package com.yahoo.vespa.model.container; import com.yahoo.container.bundle.BundleInstantiationSpecification; import com.yahoo.container.handler.threadpool.ContainerThreadPool; import com.yahoo.container.handler.threadpool.ContainerThreadpoolConfig; -import com.yahoo.container.handler.threadpool.DefaultContainerThreadpool; +import com.yahoo.container.handler.threadpool.ContainerThreadpoolImpl; import com.yahoo.osgi.provider.model.ComponentModel; import com.yahoo.text.XML; import com.yahoo.vespa.model.container.component.SimpleComponent; @@ -26,7 +26,7 @@ public class ContainerThreadpool extends SimpleComponent implements ContainerThr super(new ComponentModel( BundleInstantiationSpecification.getFromStrings( "threadpool@" + name, - DefaultContainerThreadpool.class.getName(), + ContainerThreadpoolImpl.class.getName(), null))); this.name = name; this.userOptions = userOptions; diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/ContentSearchCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/content/ContentSearchCluster.java index 9bd3b455ef9..b137c04b5c4 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/content/ContentSearchCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/content/ContentSearchCluster.java @@ -66,13 +66,9 @@ public class ContentSearchCluster extends AbstractConfigProducer<SearchCluster> private final Map<StorageGroup, NodeSpec> groupToSpecMap = new LinkedHashMap<>(); private Optional<ResourceLimits> resourceLimits = Optional.empty(); private final ProtonConfig.Indexing.Optimize.Enum feedSequencerType; - private final int feedTaskLimit; - private final int feedMasterTaskLimit; - private final ProtonConfig.Feeding.Shared_field_writer_executor.Enum sharedFieldWriterExecutor; private final double defaultFeedConcurrency; private final boolean forwardIssuesToQrs; private final int defaultMaxCompactBuffers; - private final ProtonConfig.Replay_throttling_policy.Type.Enum persistenceAsyncThrottling; /** Whether the nodes of this cluster also hosts a container cluster in a hosted system */ private final double fractionOfMemoryReserved; @@ -198,22 +194,6 @@ public class ContentSearchCluster extends AbstractConfigProducer<SearchCluster> } } - private static ProtonConfig.Feeding.Shared_field_writer_executor.Enum convertSharedFieldWriterExecutor(String value) { - try { - return ProtonConfig.Feeding.Shared_field_writer_executor.Enum.valueOf(value); - } catch (Throwable t) { - return ProtonConfig.Feeding.Shared_field_writer_executor.Enum.NONE; - } - } - - private static ProtonConfig.Replay_throttling_policy.Type.Enum convertPersistenceAsyncThrottling(String value) { - try { - return ProtonConfig.Replay_throttling_policy.Type.Enum.valueOf(value); - } catch (Throwable t) { - return ProtonConfig.Replay_throttling_policy.Type.Enum.UNLIMITED; - } - } - private ContentSearchCluster(AbstractConfigProducer<?> parent, String clusterName, ModelContext.FeatureFlags featureFlags, @@ -232,13 +212,9 @@ public class ContentSearchCluster extends AbstractConfigProducer<SearchCluster> this.fractionOfMemoryReserved = fractionOfMemoryReserved; this.feedSequencerType = convertFeedSequencerType(featureFlags.feedSequencerType()); - this.feedTaskLimit = featureFlags.feedTaskLimit(); - this.feedMasterTaskLimit = featureFlags.feedMasterTaskLimit(); - this.sharedFieldWriterExecutor = convertSharedFieldWriterExecutor(featureFlags.sharedFieldWriterExecutor()); this.defaultFeedConcurrency = featureFlags.feedConcurrency(); this.forwardIssuesToQrs = featureFlags.forwardIssuesAsErrors(); this.defaultMaxCompactBuffers = featureFlags.maxCompactBuffers(); - this.persistenceAsyncThrottling = convertPersistenceAsyncThrottling(featureFlags.persistenceAsyncThrottling()); } public void setVisibilityDelay(double delay) { @@ -452,10 +428,6 @@ public class ContentSearchCluster extends AbstractConfigProducer<SearchCluster> } builder.indexing.optimize(feedSequencerType); - builder.indexing.tasklimit(feedTaskLimit); - builder.feeding.master_task_limit(feedMasterTaskLimit); - builder.feeding.shared_field_writer_executor(sharedFieldWriterExecutor); - builder.replay_throttling_policy.type(persistenceAsyncThrottling); } private boolean isGloballyDistributed(NewDocumentType docType) { diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/DistributorCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/content/DistributorCluster.java index 3f01f5610f1..b8d2a4f91fe 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/content/DistributorCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/content/DistributorCluster.java @@ -43,7 +43,6 @@ public class DistributorCluster extends AbstractConfigProducer<Distributor> impl private final boolean useThreePhaseUpdates; private final int maxActivationInhibitedOutOfSyncGroups; private final boolean unorderedMergeChaining; - private final boolean inhibitDefaultMergesWhenGlobalMergesPending; public static class Builder extends VespaDomBuilder.DomConfigProducerBuilder<DistributorCluster> { @@ -107,12 +106,11 @@ public class DistributorCluster extends AbstractConfigProducer<Distributor> impl boolean useThreePhaseUpdates = deployState.getProperties().featureFlags().useThreePhaseUpdates(); int maxInhibitedGroups = deployState.getProperties().featureFlags().maxActivationInhibitedOutOfSyncGroups(); boolean unorderedMergeChaining = deployState.getProperties().featureFlags().unorderedMergeChaining(); - boolean inhibitDefaultMerges = deployState.getProperties().featureFlags().inhibitDefaultMergesWhenGlobalMergesPending(); return new DistributorCluster(parent, new BucketSplitting.Builder().build(new ModelElement(producerSpec)), gc, hasIndexedDocumentType, useThreePhaseUpdates, - maxInhibitedGroups, unorderedMergeChaining, inhibitDefaultMerges); + maxInhibitedGroups, unorderedMergeChaining); } } @@ -120,8 +118,7 @@ public class DistributorCluster extends AbstractConfigProducer<Distributor> impl GcOptions gc, boolean hasIndexedDocumentType, boolean useThreePhaseUpdates, int maxActivationInhibitedOutOfSyncGroups, - boolean unorderedMergeChaining, - boolean inhibitDefaultMergesWhenGlobalMergesPending) + boolean unorderedMergeChaining) { super(parent, "distributor"); this.parent = parent; @@ -131,7 +128,6 @@ public class DistributorCluster extends AbstractConfigProducer<Distributor> impl this.useThreePhaseUpdates = useThreePhaseUpdates; this.maxActivationInhibitedOutOfSyncGroups = maxActivationInhibitedOutOfSyncGroups; this.unorderedMergeChaining = unorderedMergeChaining; - this.inhibitDefaultMergesWhenGlobalMergesPending = inhibitDefaultMergesWhenGlobalMergesPending; } @Override @@ -146,7 +142,6 @@ public class DistributorCluster extends AbstractConfigProducer<Distributor> impl builder.enable_metadata_only_fetch_phase_for_inconsistent_updates(useThreePhaseUpdates); builder.max_activation_inhibited_out_of_sync_groups(maxActivationInhibitedOutOfSyncGroups); builder.use_unordered_merge_chaining(unorderedMergeChaining); - builder.inhibit_default_merges_when_global_merges_pending(inhibitDefaultMergesWhenGlobalMergesPending); bucketSplitting.getConfig(builder); } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/storagecluster/FileStorProducer.java b/config-model/src/main/java/com/yahoo/vespa/model/content/storagecluster/FileStorProducer.java index a1c4fc41d55..fb4016f4cf4 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/content/storagecluster/FileStorProducer.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/content/storagecluster/FileStorProducer.java @@ -46,7 +46,6 @@ public class FileStorProducer implements StorFilestorConfig.Producer { private final ContentCluster cluster; private final int reponseNumThreads; private final StorFilestorConfig.Response_sequencer_type.Enum responseSequencerType; - private final StorFilestorConfig.Async_operation_throttler.Type.Enum asyncOperationThrottlerType; private final double persistenceThrottlingWsDecrementFactor; private final double persistenceThrottlingWsBackoff; private final int persistenceThrottingWindowSize; @@ -62,20 +61,11 @@ public class FileStorProducer implements StorFilestorConfig.Producer { } } - private static StorFilestorConfig.Async_operation_throttler.Type.Enum toAsyncOperationThrottlerType(String throttlerType) { - try { - return StorFilestorConfig.Async_operation_throttler.Type.Enum.valueOf(throttlerType); - } catch (Throwable t) { - return StorFilestorConfig.Async_operation_throttler.Type.UNLIMITED; - } - } - public FileStorProducer(ModelContext.FeatureFlags featureFlags, ContentCluster parent, Integer numThreads) { this.numThreads = numThreads; this.cluster = parent; this.reponseNumThreads = featureFlags.defaultNumResponseThreads(); this.responseSequencerType = convertResponseSequencerType(featureFlags.responseSequencerType()); - this.asyncOperationThrottlerType = toAsyncOperationThrottlerType(featureFlags.persistenceAsyncThrottling()); this.persistenceThrottlingWsDecrementFactor = featureFlags.persistenceThrottlingWsDecrementFactor(); this.persistenceThrottlingWsBackoff = featureFlags.persistenceThrottlingWsBackoff(); this.persistenceThrottingWindowSize = featureFlags.persistenceThrottlingWindowSize(); @@ -93,13 +83,7 @@ public class FileStorProducer implements StorFilestorConfig.Producer { builder.num_response_threads(reponseNumThreads); builder.response_sequencer_type(responseSequencerType); builder.use_async_message_handling_on_schedule(useAsyncMessageHandlingOnSchedule); - // TODO remove deprecated throttler type config - builder.async_operation_throttler_type((asyncOperationThrottlerType == StorFilestorConfig.Async_operation_throttler.Type.DYNAMIC) - ? StorFilestorConfig.Async_operation_throttler_type.Enum.DYNAMIC - : StorFilestorConfig.Async_operation_throttler_type.Enum.UNLIMITED); - var throttleBuilder = new StorFilestorConfig.Async_operation_throttler.Builder(); - throttleBuilder.type(asyncOperationThrottlerType); throttleBuilder.window_size_decrement_factor(persistenceThrottlingWsDecrementFactor); throttleBuilder.window_size_backoff(persistenceThrottlingWsBackoff); if (persistenceThrottingWindowSize > 0) { diff --git a/config-model/src/main/java/com/yahoo/vespa/model/ml/OnnxModelProbe.java b/config-model/src/main/java/com/yahoo/vespa/model/ml/OnnxModelProbe.java index ba5ebdad56a..d2205fb64b3 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/ml/OnnxModelProbe.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/ml/OnnxModelProbe.java @@ -50,7 +50,7 @@ public class OnnxModelProbe { } } catch (IllegalArgumentException | IOException | InterruptedException e) { - e.printStackTrace(System.err); + System.err.println(e.getMessage()); } return outputType; @@ -148,7 +148,8 @@ public class OnnxModelProbe { int returnCode = process.waitFor(); if (returnCode != 0) { - throw new IllegalArgumentException("Error from '" + binary + "'. Return code: " + returnCode + ". Output:\n" + output); + throw new IllegalArgumentException("Error from '" + binary + "'. Return code: " + returnCode + ". " + + "Output: '" + output + "'"); } return output.toString(); } diff --git a/config-model/src/test/java/com/yahoo/config/model/builder/xml/XmlErrorHandlingTest.java b/config-model/src/test/java/com/yahoo/config/model/builder/xml/XmlErrorHandlingTest.java new file mode 100644 index 00000000000..ee616f59d04 --- /dev/null +++ b/config-model/src/test/java/com/yahoo/config/model/builder/xml/XmlErrorHandlingTest.java @@ -0,0 +1,37 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.config.model.builder.xml; + +import org.junit.Test; +import org.xml.sax.InputSource; +import java.io.FileReader; + +import static org.junit.Assert.assertEquals; + +/** + * @author hmusum + */ +public class XmlErrorHandlingTest { + + @Test + public void requireExceptionWithSourceAndFilenameAndLineNumber() { + try { + XmlHelper.getDocument(new FileReader("src/test/cfg/application/invalid-services-syntax/services.xml"), "services.xml"); + } catch (Exception e) { + assertEquals("Invalid XML in services.xml: The element type \"config\" must be terminated by the matching end-tag \"</config>\". [7:5]", + e.getMessage()); + } + } + + + @Test + public void requireExceptionWithLineNumber() { + try { + XmlHelper.getDocumentBuilder().parse( + new InputSource(new FileReader("src/test/cfg/application/invalid-services-syntax/services.xml"))); + } catch (Exception e) { + assertEquals("Invalid XML (unknown source): The element type \"config\" must be terminated by the matching end-tag \"</config>\". [7:5]", + e.getMessage()); + } + } + +} diff --git a/config-model/src/test/java/com/yahoo/vespa/model/builder/UserConfigBuilderTest.java b/config-model/src/test/java/com/yahoo/vespa/model/builder/UserConfigBuilderTest.java index bb3a0f26ee9..114038b884e 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/builder/UserConfigBuilderTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/builder/UserConfigBuilderTest.java @@ -15,7 +15,6 @@ import com.yahoo.vespa.configdefinition.SpecialtokensConfig; import org.junit.Test; import org.w3c.dom.Document; import org.w3c.dom.Element; -import org.xml.sax.InputSource; import java.io.Reader; import java.io.StringReader; @@ -112,7 +111,7 @@ public class UserConfigBuilderTest { Reader xmlReader = new StringReader("<model>" + xml + "</model>"); Document doc; try { - doc = XmlHelper.getDocumentBuilder().parse(new InputSource(xmlReader)); + doc = XmlHelper.getDocument(xmlReader); } catch (Exception e) { throw new RuntimeException(); } diff --git a/config-model/src/test/java/com/yahoo/vespa/model/builder/xml/dom/ContentBuilderTest.java b/config-model/src/test/java/com/yahoo/vespa/model/builder/xml/dom/ContentBuilderTest.java index 7ef3594c844..90d1dba4e4a 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/builder/xml/dom/ContentBuilderTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/builder/xml/dom/ContentBuilderTest.java @@ -847,59 +847,6 @@ public class ContentBuilderTest extends DomBuilderTest { } - @Test - public void feed_task_limit_is_controlled_by_feature_flag() { - assertEquals(1000, resolveFeedTaskLimitConfigWithFeatureFlag(null)); - assertEquals(2000, resolveFeedTaskLimitConfigWithFeatureFlag(2000)); - } - - private int resolveFeedTaskLimitConfigWithFeatureFlag(Integer value) { - var props = new TestProperties(); - if (value != null) { - props.setFeedTaskLimit(value); - } - return resolveProtonConfig(props, singleNodeContentXml()).indexing().tasklimit(); - } - - @Test - public void feed_master_task_limit_is_controlled_by_feature_flag() { - assertEquals(1000, resolveFeedMasterTaskLimitConfigWithFeatureFlag(null)); - assertEquals(2000, resolveFeedMasterTaskLimitConfigWithFeatureFlag(2000)); - } - - private int resolveFeedMasterTaskLimitConfigWithFeatureFlag(Integer value) { - var props = new TestProperties(); - if (value != null) { - props.setFeedMasterTaskLimit(value); - } - return resolveProtonConfig(props, singleNodeContentXml()).feeding().master_task_limit(); - } - - @Test - public void shared_field_writer_executor_is_controlled_by_feature_flag() { - - assertEquals(Shared_field_writer_executor.Enum.NONE, - resolveSharedFieldWriterExecutorConfigWithFeatureFlag(null)); - assertEquals(Shared_field_writer_executor.Enum.NONE, - resolveSharedFieldWriterExecutorConfigWithFeatureFlag("NONE")); - assertEquals(Shared_field_writer_executor.Enum.INDEX, - resolveSharedFieldWriterExecutorConfigWithFeatureFlag("INDEX")); - assertEquals(Shared_field_writer_executor.Enum.INDEX_AND_ATTRIBUTE, - resolveSharedFieldWriterExecutorConfigWithFeatureFlag("INDEX_AND_ATTRIBUTE")); - assertEquals(Shared_field_writer_executor.Enum.DOCUMENT_DB, - resolveSharedFieldWriterExecutorConfigWithFeatureFlag("DOCUMENT_DB")); - assertEquals(Shared_field_writer_executor.Enum.NONE, - resolveSharedFieldWriterExecutorConfigWithFeatureFlag("invalid")); - } - - private ProtonConfig.Feeding.Shared_field_writer_executor.Enum resolveSharedFieldWriterExecutorConfigWithFeatureFlag(String value) { - var props = new TestProperties(); - if (value != null) { - props.setSharedFieldWriterExecutor(value); - } - return resolveProtonConfig(props, singleNodeContentXml()).feeding().shared_field_writer_executor(); - } - private void verifyThatFeatureFlagControlsVisibilityDelayDefault(Double xmlOverride, double expected) { String hostedXml = xmlWithVisibilityDelay(xmlOverride); var config = resolveProtonConfig(new TestProperties(), hostedXml); diff --git a/config-model/src/test/java/com/yahoo/vespa/model/builder/xml/dom/DomConfigPayloadBuilderTest.java b/config-model/src/test/java/com/yahoo/vespa/model/builder/xml/dom/DomConfigPayloadBuilderTest.java index 264d102161a..a20ce425ac0 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/builder/xml/dom/DomConfigPayloadBuilderTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/builder/xml/dom/DomConfigPayloadBuilderTest.java @@ -11,11 +11,9 @@ import com.yahoo.vespa.config.ConfigDefinitionBuilder; import com.yahoo.vespa.config.ConfigDefinitionKey; import com.yahoo.vespa.config.ConfigPayload; import com.yahoo.vespa.config.ConfigPayloadBuilder; - import org.junit.Test; import org.w3c.dom.Document; import org.w3c.dom.Element; -import org.xml.sax.InputSource; import java.io.ByteArrayOutputStream; import java.io.FileNotFoundException; @@ -273,7 +271,7 @@ public class DomConfigPayloadBuilderTest { private Element getDocument(Reader xmlReader) { Document doc; try { - doc = XmlHelper.getDocumentBuilder().parse(new InputSource(xmlReader)); + doc = XmlHelper.getDocument(xmlReader); } catch (Exception e) { throw new RuntimeException(); } diff --git a/config-model/src/test/java/com/yahoo/vespa/model/container/xml/BundleInstantiationSpecificationBuilderTest.java b/config-model/src/test/java/com/yahoo/vespa/model/container/xml/BundleInstantiationSpecificationBuilderTest.java index 909ac9edef2..686f7bbd1f1 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/container/xml/BundleInstantiationSpecificationBuilderTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/container/xml/BundleInstantiationSpecificationBuilderTest.java @@ -8,12 +8,8 @@ import com.yahoo.search.grouping.GroupingValidator; import com.yahoo.vespa.model.container.PlatformBundles; import org.junit.Test; import org.w3c.dom.Element; -import org.xml.sax.SAXException; -import java.io.ByteArrayInputStream; -import java.io.IOException; -import java.io.InputStream; -import java.nio.charset.StandardCharsets; +import java.io.StringReader; import static org.junit.Assert.assertEquals; @@ -25,34 +21,31 @@ import static org.junit.Assert.assertEquals; public class BundleInstantiationSpecificationBuilderTest { @Test - public void bundle_is_not_replaced_for_user_defined_class() throws IOException, SAXException { + public void bundle_is_not_replaced_for_user_defined_class() { final String userDefinedClass = "my own class that will also be set as bundle"; verifyExpectedBundle(userDefinedClass, null, userDefinedClass); } @Test - public void bundle_is_replaced_for_internal_class() throws IOException, SAXException { + public void bundle_is_replaced_for_internal_class() { String internalClass = GroupingValidator.class.getName(); verifyExpectedBundle(internalClass, null, PlatformBundles.searchAndDocprocBundle); } @Test - public void bundle_is_not_replaced_for_internal_class_with_explicitly_set_bundle() - throws IOException, SAXException { + public void bundle_is_not_replaced_for_internal_class_with_explicitly_set_bundle() { String internalClass = GroupingValidator.class.getName(); String explicitBundle = "my-own-implementation"; verifyExpectedBundle(internalClass, explicitBundle, explicitBundle); } - private static void verifyExpectedBundle(String className, String explicitBundle, String expectedBundle) - throws IOException, SAXException { + private static void verifyExpectedBundle(String className, String explicitBundle, String expectedBundle) { String xml = "<component id=\"_\" class=\"" + className + "\""; if (explicitBundle != null) { xml += " bundle=\"" + explicitBundle + "\""; } xml += " />"; - InputStream xmlStream = new ByteArrayInputStream(xml.getBytes(StandardCharsets.UTF_8)); - Element component = XmlHelper.getDocumentBuilder().parse(xmlStream).getDocumentElement(); + Element component = XmlHelper.getDocument(new StringReader(xml)).getDocumentElement(); BundleInstantiationSpecification spec = BundleInstantiationSpecificationBuilder.build(component); assertEquals(ComponentSpecification.fromString(expectedBundle), spec.bundle); diff --git a/config-model/src/test/java/com/yahoo/vespa/model/content/ContentClusterTest.java b/config-model/src/test/java/com/yahoo/vespa/model/content/ContentClusterTest.java index 9f571167d8c..10a2feaba5b 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/content/ContentClusterTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/content/ContentClusterTest.java @@ -1045,28 +1045,6 @@ public class ContentClusterTest extends ContentBaseTest { assertEquals(7, resolveMaxCompactBuffers(OptionalInt.of(7))); } - private ProtonConfig.Replay_throttling_policy.Type.Enum resolveReplayThrottlePolicyType(Optional<String> throttlerType) { - TestProperties testProperties = new TestProperties(); - if (throttlerType.isPresent()) { - testProperties.setPersistenceAsyncThrottling(throttlerType.get()); - } - VespaModel model = createEnd2EndOneNode(testProperties); - ContentCluster cc = model.getContentClusters().get("storage"); - ProtonConfig.Builder protonBuilder = new ProtonConfig.Builder(); - cc.getSearch().getConfig(protonBuilder); - ProtonConfig protonConfig = new ProtonConfig(protonBuilder); - assertEquals(1, protonConfig.documentdb().size()); - return protonConfig.replay_throttling_policy().type(); - } - - @Test - public void replay_throttling_policy_type_controlled_by_properties() { - assertEquals(ProtonConfig.Replay_throttling_policy.Type.Enum.UNLIMITED, resolveReplayThrottlePolicyType(Optional.empty())); - assertEquals(ProtonConfig.Replay_throttling_policy.Type.Enum.UNLIMITED, resolveReplayThrottlePolicyType(Optional.of("UNLIMITED"))); - assertEquals(ProtonConfig.Replay_throttling_policy.Type.Enum.UNLIMITED, resolveReplayThrottlePolicyType(Optional.of("INVALID"))); - assertEquals(ProtonConfig.Replay_throttling_policy.Type.Enum.DYNAMIC, resolveReplayThrottlePolicyType(Optional.of("DYNAMIC"))); - } - private long resolveMaxTLSSize(Optional<Flavor> flavor) throws Exception { TestProperties testProperties = new TestProperties(); @@ -1159,24 +1137,6 @@ public class ContentClusterTest extends ContentBaseTest { } @Test - public void inhibit_default_merges_when_global_merges_pending_controlled_by_properties() throws Exception { - assertFalse(resolveInhibitDefaultMergesConfig(Optional.empty())); - assertFalse(resolveInhibitDefaultMergesConfig(Optional.of(false))); - assertTrue(resolveInhibitDefaultMergesConfig(Optional.of(true))); - } - - private boolean resolveInhibitDefaultMergesConfig(Optional<Boolean> inhibitDefaultMerges) throws Exception { - var props = new TestProperties(); - if (inhibitDefaultMerges.isPresent()) { - props.inhibitDefaultMergesWhenGlobalMergesPending(inhibitDefaultMerges.get()); - } - var cluster = createOneNodeCluster(props); - var builder = new StorDistributormanagerConfig.Builder(); - cluster.getDistributorNodes().getConfig(builder); - return (new StorDistributormanagerConfig(builder)).inhibit_default_merges_when_global_merges_pending(); - } - - @Test public void testDedicatedClusterControllers() { VespaModel noContentModel = createEnd2EndOneNode(new TestProperties().setHostedVespa(true) .setMultitenant(true), diff --git a/config-model/src/test/java/com/yahoo/vespa/model/content/StorageClusterTest.java b/config-model/src/test/java/com/yahoo/vespa/model/content/StorageClusterTest.java index ebc94e1f481..8908ab9f5b9 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/content/StorageClusterTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/content/StorageClusterTest.java @@ -307,31 +307,9 @@ public class StorageClusterTest { } @Test - public void persistence_async_throttle_config_defaults_to_unlimited() { - var config = filestorConfigFromProducer(simpleCluster(new TestProperties())); - assertEquals(StorFilestorConfig.Async_operation_throttler_type.UNLIMITED, config.async_operation_throttler_type()); // TODO remove - assertEquals(StorFilestorConfig.Async_operation_throttler.Type.UNLIMITED, config.async_operation_throttler().type()); - } - - @Test - public void persistence_async_throttle_config_is_derived_from_flag() { - var config = filestorConfigFromProducer(simpleCluster(new TestProperties().setPersistenceAsyncThrottling("UNLIMITED"))); - assertEquals(StorFilestorConfig.Async_operation_throttler_type.UNLIMITED, config.async_operation_throttler_type()); // TODO remove - assertEquals(StorFilestorConfig.Async_operation_throttler.Type.UNLIMITED, config.async_operation_throttler().type()); - - config = filestorConfigFromProducer(simpleCluster(new TestProperties().setPersistenceAsyncThrottling("DYNAMIC"))); - assertEquals(StorFilestorConfig.Async_operation_throttler_type.DYNAMIC, config.async_operation_throttler_type()); // TODO remove - assertEquals(StorFilestorConfig.Async_operation_throttler.Type.DYNAMIC, config.async_operation_throttler().type()); - - // Invalid enum values fall back to the default - config = filestorConfigFromProducer(simpleCluster(new TestProperties().setPersistenceAsyncThrottling("BANANAS"))); - assertEquals(StorFilestorConfig.Async_operation_throttler_type.UNLIMITED, config.async_operation_throttler_type()); // TODO remove - assertEquals(StorFilestorConfig.Async_operation_throttler.Type.UNLIMITED, config.async_operation_throttler().type()); - } - - @Test public void persistence_dynamic_throttling_parameters_have_sane_defaults() { var config = filestorConfigFromProducer(simpleCluster(new TestProperties())); + assertEquals(StorFilestorConfig.Async_operation_throttler.Type.DYNAMIC, config.async_operation_throttler().type()); assertEquals(1.2, config.async_operation_throttler().window_size_decrement_factor(), 0.0001); assertEquals(0.95, config.async_operation_throttler().window_size_backoff(), 0.0001); assertEquals(20, config.async_operation_throttler().min_window_size()); diff --git a/config-provisioning/pom.xml b/config-provisioning/pom.xml index fdfbbfc9049..c7616b2187a 100644 --- a/config-provisioning/pom.xml +++ b/config-provisioning/pom.xml @@ -8,14 +8,12 @@ <version>7-SNAPSHOT</version> <relativePath>../parent/pom.xml</relativePath> </parent> - <groupId>com.yahoo.vespa</groupId> + + <name>config-provisioning</name> + <description>Provisioning APIs.</description> <artifactId>config-provisioning</artifactId> <packaging>container-plugin</packaging> - <version>7-SNAPSHOT</version> - <name>config-provisioning</name> - <description> -Provisioning APIs. - </description> + <dependencies> <dependency> <groupId>com.yahoo.vespa</groupId> @@ -69,11 +67,6 @@ Provisioning APIs. <artifactId>junit</artifactId> <scope>test</scope> </dependency> - <dependency> - <groupId>com.google.guava</groupId> - <artifactId>guava-testlib</artifactId> - <scope>test</scope> - </dependency> </dependencies> <build> <plugins> diff --git a/config-provisioning/src/main/java/com/yahoo/config/provision/ApplicationId.java b/config-provisioning/src/main/java/com/yahoo/config/provision/ApplicationId.java index aa70bf4d26a..f77a6b8d182 100644 --- a/config-provisioning/src/main/java/com/yahoo/config/provision/ApplicationId.java +++ b/config-provisioning/src/main/java/com/yahoo/config/provision/ApplicationId.java @@ -16,14 +16,11 @@ import java.util.regex.Pattern; */ public class ApplicationId implements Comparable<ApplicationId> { - // TODO: remove '.' and '*' from this pattern. - static final Pattern namePattern = Pattern.compile("(?!\\.\\.)[a-zA-Z0-9_.*-]{1,256}"); - - private static final ApplicationId global = new ApplicationId(TenantName.from("*"), - ApplicationName.from("*"), - InstanceName.from("*")) { - @Override public boolean equals(Object other) { return this == other; } - }; + static final Pattern namePattern = Pattern.compile("(?!\\.\\.)[a-zA-Z0-9_-]{1,256}"); + + private static final ApplicationId global = new ApplicationId(TenantName.from("hosted-vespa"), + ApplicationName.from("routing"), + InstanceName.from("default")) { }; private static final Comparator<ApplicationId> comparator = Comparator.comparing(ApplicationId::tenant) .thenComparing(ApplicationId::application) diff --git a/config-provisioning/src/main/java/com/yahoo/config/provision/AthenzDomain.java b/config-provisioning/src/main/java/com/yahoo/config/provision/AthenzDomain.java index 7b60d22c810..16cb3c43814 100644 --- a/config-provisioning/src/main/java/com/yahoo/config/provision/AthenzDomain.java +++ b/config-provisioning/src/main/java/com/yahoo/config/provision/AthenzDomain.java @@ -1,54 +1,23 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.config.provision; +import ai.vespa.validation.PatternedStringWrapper; + import java.util.regex.Pattern; /** * @author mortent */ -public class AthenzDomain { - - private static final Pattern PATTERN = Pattern.compile("[a-zA-Z0-9_][a-zA-Z0-9_\\-.]*[a-zA-Z0-9_]"); +public class AthenzDomain extends PatternedStringWrapper<AthenzDomain> { - private final String name; + private static final Pattern PATTERN = Pattern.compile("[a-zA-Z0-9_][a-zA-Z0-9_.-]*[a-zA-Z0-9_]"); private AthenzDomain(String name) { - // TODO bjorncs: Temporarily disable name validation - // validateName(name); - this.name = name; - } - - private static void validateName(String name) { - if (!PATTERN.matcher(name).matches()) { - throw new IllegalArgumentException("Not a valid domain name: '" + name + "'"); - } + super(name, PATTERN, "Athenz domain"); } public static AthenzDomain from(String value) { return new AthenzDomain(value); } - public String value() { return name; } - - @Override - public String toString() { - return "AthenzDomain{" + - "name='" + name + '\'' + - '}'; - } - - @Override - public boolean equals(Object o) { - if (this == o) return true; - if (o == null || getClass() != o.getClass()) return false; - - AthenzDomain that = (AthenzDomain) o; - - return name != null ? name.equals(that.name) : that.name == null; - } - - @Override - public int hashCode() { - return name != null ? name.hashCode() : 0; - } } diff --git a/config-provisioning/src/main/java/com/yahoo/config/provision/CloudName.java b/config-provisioning/src/main/java/com/yahoo/config/provision/CloudName.java index b4be3531f2a..417e381587e 100644 --- a/config-provisioning/src/main/java/com/yahoo/config/provision/CloudName.java +++ b/config-provisioning/src/main/java/com/yahoo/config/provision/CloudName.java @@ -1,29 +1,27 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.config.provision; +import ai.vespa.validation.PatternedStringWrapper; + import java.util.Objects; +import java.util.regex.Pattern; /** * Represents a cloud provider used in a hosted Vespa system. * * @author mpolden */ -public class CloudName implements Comparable<CloudName> { - - private final static CloudName defaultCloud = from("default"); +public class CloudName extends PatternedStringWrapper<CloudName> { - private final String cloud; + private static final Pattern pattern = Pattern.compile("[a-z]([a-z0-9-]*[a-z0-9])*"); + private static final CloudName defaultCloud = from("default"); private CloudName(String cloud) { - this.cloud = cloud; - } - - public String value() { - return cloud; + super(cloud, pattern, "cloud name"); } public boolean isDefault() { - return defaultName().equals(this); + return equals(defaultCloud); } public static CloudName defaultName() { @@ -34,27 +32,4 @@ public class CloudName implements Comparable<CloudName> { return new CloudName(cloud); } - @Override - public boolean equals(Object o) { - if (this == o) return true; - if (o == null || getClass() != o.getClass()) return false; - CloudName cloudName = (CloudName) o; - return Objects.equals(cloud, cloudName.cloud); - } - - @Override - public int hashCode() { - return Objects.hash(cloud); - } - - @Override - public String toString() { - return cloud; - } - - @Override - public int compareTo(CloudName o) { - return cloud.compareTo(o.cloud); - } - } diff --git a/config-provisioning/src/main/java/com/yahoo/config/provision/RegionName.java b/config-provisioning/src/main/java/com/yahoo/config/provision/RegionName.java index 73ad2181965..be431a5fd68 100644 --- a/config-provisioning/src/main/java/com/yahoo/config/provision/RegionName.java +++ b/config-provisioning/src/main/java/com/yahoo/config/provision/RegionName.java @@ -1,56 +1,35 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.config.provision; -import java.util.Objects; +import ai.vespa.validation.PatternedStringWrapper; + +import java.util.regex.Pattern; /** - * Represents an application's region, which may be any kind of string or default. This type is defined - * in order to provide a type safe API for defining regions. + * A region in a hosted Vespa system. + * A region name must be all lowercase, start with a letter, and contain letters and digits, separated by dashes. * - * @author Ulf Lilleengen - * @since 5.11 + * @author jonmv */ -public class RegionName implements Comparable<RegionName> { +public class RegionName extends PatternedStringWrapper<RegionName> { - private final String region; + private static final Pattern pattern = Pattern.compile("[a-z]([a-z0-9-]*[a-z0-9])*"); + private static final RegionName defaultName = from("default"); private RegionName(String region) { - this.region = region; - } - - @Override - public int hashCode() { - return region.hashCode(); - } - - @Override - public boolean equals(Object obj) { - if (!(obj instanceof RegionName)) return false; - return Objects.equals(((RegionName) obj).region, region); + super(region, pattern, "region name"); } - @Override - public String toString() { - return region; - } - - // TODO: Add verification of region name. public static RegionName from(String region) { return new RegionName(region); } public static RegionName defaultName() { - return new RegionName("default"); + return defaultName; } public boolean isDefault() { - return equals(RegionName.defaultName()); + return equals(defaultName()); } - public String value() { return region; } - - @Override - public int compareTo(RegionName region) { - return this.region.compareTo(region.region); - } } diff --git a/config-provisioning/src/test/java/com/yahoo/config/provision/ApplicationIdTest.java b/config-provisioning/src/test/java/com/yahoo/config/provision/ApplicationIdTest.java index 01904b5eece..622fc2527bf 100644 --- a/config-provisioning/src/test/java/com/yahoo/config/provision/ApplicationIdTest.java +++ b/config-provisioning/src/test/java/com/yahoo/config/provision/ApplicationIdTest.java @@ -1,12 +1,18 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.config.provision; -import static org.junit.Assert.assertEquals; - import com.yahoo.cloud.config.ApplicationIdConfig; import com.yahoo.test.TotalOrderTester; import org.junit.Test; -import com.google.common.testing.EqualsTester; + +import java.util.HashSet; +import java.util.List; +import java.util.Set; + +import static com.yahoo.config.provision.ApplicationId.from; +import static com.yahoo.config.provision.ApplicationId.global; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotEquals; /** * @author Ulf Lilleengen @@ -15,19 +21,11 @@ import com.google.common.testing.EqualsTester; */ public class ApplicationIdTest { - ApplicationId idFrom(String tenant, String name, String instance) { - ApplicationId.Builder b = new ApplicationId.Builder(); - b.tenant(tenant); - b.applicationName(name); - b.instanceName(instance); - return b.build(); - } - @Test public void require_that_application_id_is_set() { ApplicationId app = applicationId("application"); assertEquals("application", app.application().value()); - app = idFrom("tenant", "application", "instance"); + app = from("tenant", "application", "instance"); assertEquals("tenant", app.tenant().value()); assertEquals("application", app.application().value()); assertEquals("instance", app.instance().value()); @@ -35,22 +33,25 @@ public class ApplicationIdTest { @Test public void require_that_equals_and_hashcode_behaves_correctly() { - new EqualsTester() - .addEqualityGroup(idFrom("tenant1", "name1", "instance1"), - idFrom("tenant1", "name1", "instance1")) - .addEqualityGroup(idFrom("tenant2", "name1", "instance1")) - .addEqualityGroup(idFrom("tenant1", "name2", "instance1")) - .addEqualityGroup(idFrom("tenant1", "name1", "instance2")) - .addEqualityGroup(applicationId("onlyName1")) - .addEqualityGroup(applicationId("onlyName2")) - .testEquals(); + assertEquals(Set.of(from("tenant1", "name1", "instance1"), + from("tenant2", "name1", "instance1"), + from("tenant1", "name2", "instance1"), + from("tenant1", "name1", "instance2"), + applicationId("name1"), + applicationId("name2")), + new HashSet<>(List.of(from("tenant1", "name1", "instance1"), + from("tenant2", "name1", "instance1"), + from("tenant1", "name2", "instance1"), + from("tenant1", "name1", "instance2"), + applicationId("name1"), + applicationId("name2")))); } @Test public void require_that_value_format_is_correct() { ApplicationId id1 = applicationId("foo"); ApplicationId id2 = applicationId("bar"); - ApplicationId id3 = idFrom("tenant", "baz", "bim"); + ApplicationId id3 = from("tenant", "baz", "bim"); assertEquals("default:foo:default", id1.serializedForm()); assertEquals("default:bar:default", id2.serializedForm()); assertEquals("tenant:baz:bim", id3.serializedForm()); @@ -59,8 +60,8 @@ public class ApplicationIdTest { @Test public void require_string_formats_are_correct() { ApplicationId id1 = applicationId("foo"); - ApplicationId id2 = idFrom("bar", "baz", "default"); - ApplicationId id3 = idFrom("tenant", "baz", "bim"); + ApplicationId id2 = from("bar", "baz", "default"); + ApplicationId id3 = from("tenant", "baz", "bim"); assertEquals("default.foo", id1.toShortString()); assertEquals("default.foo.default", id1.toFullString()); assertEquals("bar.baz", id2.toShortString()); @@ -92,11 +93,11 @@ public class ApplicationIdTest { @Test public void require_that_compare_to_is_correct() { new TotalOrderTester<ApplicationId>() - .theseObjects(idFrom("tenant1", "name1", "instance1"), - idFrom("tenant1", "name1", "instance1")) - .areLessThan(idFrom("tenant2", "name1", "instance1")) - .areLessThan(idFrom("tenant2", "name2", "instance1")) - .areLessThan(idFrom("tenant2", "name2", "instance2")) + .theseObjects(from("tenant1", "name1", "instance1"), + from("tenant1", "name1", "instance1")) + .areLessThan(from("tenant2", "name1", "instance1")) + .areLessThan(from("tenant2", "name2", "instance1")) + .areLessThan(from("tenant2", "name2", "instance2")) .testOrdering(); } @@ -106,15 +107,21 @@ public class ApplicationIdTest { builder.tenant("a"); builder.application("b"); builder.instance("c"); - ApplicationId applicationId = ApplicationId.from(new ApplicationIdConfig(builder)); + ApplicationId applicationId = from(new ApplicationIdConfig(builder)); assertEquals("a", applicationId.tenant().value()); assertEquals("b", applicationId.application().value()); assertEquals("c", applicationId.instance().value()); } + @Test + public void require_that_global_is_special() { + assertEquals(global(), global()); + assertNotEquals(global(), from("hosted-vespa", "routing", "default")); + assertEquals(global().serializedForm(), from("hosted-vespa", "routing", "default").serializedForm()); + } + private ApplicationId applicationId(String applicationName) { - return ApplicationId.from(TenantName.defaultName(), - ApplicationName.from(applicationName), InstanceName.defaultName()); + return from(TenantName.defaultName(), ApplicationName.from(applicationName), InstanceName.defaultName()); } } diff --git a/config-provisioning/src/test/java/com/yahoo/config/provision/ClusterSpecTest.java b/config-provisioning/src/test/java/com/yahoo/config/provision/ClusterSpecTest.java index 0fe123a774a..dc228ee7fd2 100644 --- a/config-provisioning/src/test/java/com/yahoo/config/provision/ClusterSpecTest.java +++ b/config-provisioning/src/test/java/com/yahoo/config/provision/ClusterSpecTest.java @@ -1,13 +1,16 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.config.provision; -import com.google.common.testing.EqualsTester; import com.yahoo.component.Version; +import com.yahoo.config.provision.ClusterSpec.Group; +import com.yahoo.config.provision.ClusterSpec.Id; import org.junit.Test; +import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Optional; +import java.util.Set; import static org.junit.Assert.assertEquals; @@ -18,20 +21,14 @@ public class ClusterSpecTest { @Test public void testIdEquals() { - new EqualsTester() - .addEqualityGroup(ClusterSpec.Id.from("id1"), ClusterSpec.Id.from("id1")) - .addEqualityGroup(ClusterSpec.Id.from("id2")) - .addEqualityGroup(ClusterSpec.Id.from("id3")) - .testEquals(); + assertEquals(Set.of(Id.from("id1"), Id.from("id2"), Id.from("id3")), + new HashSet<>(List.of(Id.from("id1"), Id.from("id1"), Id.from("id2"), Id.from("id3")))); } @Test public void testGroupEquals() { - new EqualsTester() - .addEqualityGroup(ClusterSpec.Group.from(1), ClusterSpec.Group.from(1)) - .addEqualityGroup(ClusterSpec.Group.from(2)) - .addEqualityGroup(ClusterSpec.Group.from(3)) - .testEquals(); + assertEquals(Set.of(Group.from(1), Group.from(2), Group.from(3)), + new HashSet<>(List.of(Group.from(1), Group.from(1), Group.from(2), Group.from(3)))); } @Test diff --git a/config-provisioning/src/test/java/com/yahoo/config/provision/IdentifierTestBase.java b/config-provisioning/src/test/java/com/yahoo/config/provision/IdentifierTestBase.java index d4edb7a14eb..fd1a9401b81 100644 --- a/config-provisioning/src/test/java/com/yahoo/config/provision/IdentifierTestBase.java +++ b/config-provisioning/src/test/java/com/yahoo/config/provision/IdentifierTestBase.java @@ -1,9 +1,12 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.config.provision; -import com.google.common.testing.EqualsTester; import org.junit.Test; +import java.util.HashSet; +import java.util.List; +import java.util.Set; + import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertNotEquals; @@ -12,7 +15,6 @@ import static org.junit.Assert.assertTrue; /** * Generic test for identifiers such as {@link Environment} and {@link RegionName}. * @author Ulf Lilleengen - * @since 5.23 */ public abstract class IdentifierTestBase<ID_TYPE> { @@ -34,10 +36,8 @@ public abstract class IdentifierTestBase<ID_TYPE> { @Test public void testEquals() { - new EqualsTester() - .addEqualityGroup(createInstance("foo"), createInstance("foo")) - .addEqualityGroup(createInstance("bar")) - .addEqualityGroup(createInstance("baz")) - .testEquals(); + assertEquals(Set.of(createInstance("foo"), createInstance("bar"), createInstance("baz")), + new HashSet<>(List.of(createInstance("foo"), createInstance("foo"), createInstance("bar"), createInstance("baz")))); } + } diff --git a/config/src/main/java/com/yahoo/config/subscription/impl/ConfigSubscription.java b/config/src/main/java/com/yahoo/config/subscription/impl/ConfigSubscription.java index caa11b211ac..5d720a74b15 100644 --- a/config/src/main/java/com/yahoo/config/subscription/impl/ConfigSubscription.java +++ b/config/src/main/java/com/yahoo/config/subscription/impl/ConfigSubscription.java @@ -206,7 +206,9 @@ public abstract class ConfigSubscription<T extends ConfigInstance> { int sizeHint = 500; log.log(Level.WARNING, "Config has changed unexpectedly for " + key + ", generation " + generation + ", config in state :" + generator.makeSnippet(previousConfig.toString(), sizeHint) + ", new config: " + - generator.makeSnippet(config.toString(), sizeHint)); + generator.makeSnippet(config.toString(), sizeHint) + + ". This likely happened because config changed on a previous generation" + + ", look for earlier entry in log with warning about config changing without a change in config generation."); } this.config.set(new ConfigState<>(true, generation, applyOnRestart, configChanged, config, payloadChecksums)); } diff --git a/config/src/main/java/com/yahoo/config/subscription/impl/JRTConfigRequester.java b/config/src/main/java/com/yahoo/config/subscription/impl/JRTConfigRequester.java index 604c85555db..d77cfc17c92 100644 --- a/config/src/main/java/com/yahoo/config/subscription/impl/JRTConfigRequester.java +++ b/config/src/main/java/com/yahoo/config/subscription/impl/JRTConfigRequester.java @@ -6,6 +6,7 @@ import com.yahoo.config.ConfigurationRuntimeException; import com.yahoo.config.subscription.ConfigSourceSet; import com.yahoo.jrt.Request; import com.yahoo.jrt.RequestWaiter; +import com.yahoo.text.internal.SnippetGenerator; import com.yahoo.vespa.config.Connection; import com.yahoo.vespa.config.ConnectionPool; import com.yahoo.vespa.config.ErrorCode; @@ -19,6 +20,7 @@ import java.time.Duration; import java.time.Instant; import java.util.concurrent.ScheduledThreadPoolExecutor; import java.util.concurrent.TimeUnit; +import java.util.logging.Level; import java.util.logging.Logger; import static java.util.logging.Level.FINE; @@ -196,6 +198,15 @@ public class JRTConfigRequester implements RequestWaiter { log.log(FINE, () -> "OK response received in handleOkRequest: " + jrtReq); if (jrtReq.hasUpdatedGeneration()) { sub.updateConfig(jrtReq); + } else if (jrtReq.hasUpdatedConfig()) { + SnippetGenerator generator = new SnippetGenerator(); + int sizeHint = 500; + String config = jrtReq.getNewPayload().toString(); + log.log(Level.WARNING, "Config " + jrtReq.getConfigKey() + " has changed without a change in config generation: generation " + + jrtReq.getNewGeneration() + ", config: " + generator.makeSnippet(config, sizeHint) + + ". This might happen when a newly upgraded config server responds with different config when bootstrapping " + + " (changes to code generating config that are different between versions) or non-deterministic config generation" + + " (e.g. when using collections with non-deterministic iteration order)"); } scheduleNextRequest(jrtReq, sub, calculateSuccessDelay(), calculateSuccessTimeout()); } diff --git a/config/src/main/java/com/yahoo/config/subscription/impl/JRTConfigSubscription.java b/config/src/main/java/com/yahoo/config/subscription/impl/JRTConfigSubscription.java index abfbf2c6c79..23c38812d64 100644 --- a/config/src/main/java/com/yahoo/config/subscription/impl/JRTConfigSubscription.java +++ b/config/src/main/java/com/yahoo/config/subscription/impl/JRTConfigSubscription.java @@ -141,7 +141,7 @@ public class JRTConfigSubscription<T extends ConfigInstance> extends ConfigSubsc return configInstance; } - // Called by JRTConfigRequester when there is a config with new generation for this subscription + // Called by JRTConfigRequester when there is a config response for this subscription void updateConfig(JRTClientConfigRequest jrtReq) { if ( ! responseQueue.offer(jrtReq)) setException(new ConfigurationRuntimeException("Failed offering returned request to queue of subscription " + this)); diff --git a/configdefinitions/src/vespa/stor-filestor.def b/configdefinitions/src/vespa/stor-filestor.def index 531805d3039..ef4094a02bf 100644 --- a/configdefinitions/src/vespa/stor-filestor.def +++ b/configdefinitions/src/vespa/stor-filestor.def @@ -80,7 +80,7 @@ resource_usage_reporter_noise_level double default=0.001 ## - DYNAMIC uses DynamicThrottlePolicy under the hood and will block if the window ## is full (if a blocking throttler API call is invoked). ## -async_operation_throttler.type enum { UNLIMITED, DYNAMIC } default=UNLIMITED +async_operation_throttler.type enum { UNLIMITED, DYNAMIC } default=DYNAMIC ## Internal throttler tuning parameters that only apply when type == DYNAMIC: async_operation_throttler.window_size_increment int default=20 async_operation_throttler.window_size_decrement_factor double default=1.2 @@ -104,7 +104,7 @@ async_operation_throttler.throttle_individual_merge_feed_ops bool default=true ## is full (if a blocking throttler API call is invoked). ## ## TODO deprecate in favor of the async_operation_throttler struct instead. -async_operation_throttler_type enum { UNLIMITED, DYNAMIC } default=UNLIMITED +async_operation_throttler_type enum { UNLIMITED, DYNAMIC } default=DYNAMIC ## Specifies the extent the throttling window is increased by when the async throttle ## policy has decided that more concurrent operations are desirable. Also affects the diff --git a/configserver/pom.xml b/configserver/pom.xml index 110099421d1..131e4503f6d 100644 --- a/configserver/pom.xml +++ b/configserver/pom.xml @@ -184,11 +184,6 @@ </dependency> <dependency> <groupId>com.yahoo.vespa</groupId> - <artifactId>serviceview</artifactId> - <version>${project.version}</version> - </dependency> - <dependency> - <groupId>com.yahoo.vespa</groupId> <artifactId>zookeeper-server-common</artifactId> <version>${project.version}</version> <scope>provided</scope> @@ -246,7 +241,7 @@ <version>${project.version}</version> </dependency> - <!-- Jersey, needed by serviceview --> + <!-- Jersey, needed by orchestrator --> <dependency> <groupId>javax.ws.rs</groupId> <artifactId>javax.ws.rs-api</artifactId> diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/application/HttpProxy.java b/configserver/src/main/java/com/yahoo/vespa/config/server/application/HttpProxy.java index 5f7ace38a2c..66fd84e11d3 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/application/HttpProxy.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/application/HttpProxy.java @@ -17,14 +17,11 @@ import com.yahoo.vespa.config.server.http.NotFoundException; import com.yahoo.vespa.config.server.http.SimpleHttpFetcher; import java.io.ByteArrayOutputStream; -import java.io.FilterOutputStream; import java.io.IOException; import java.io.OutputStream; import java.nio.charset.Charset; -import java.nio.charset.StandardCharsets; import java.util.List; import java.util.logging.Logger; -import java.util.regex.Pattern; import static java.nio.charset.StandardCharsets.UTF_8; @@ -63,7 +60,7 @@ public class HttpProxy { .orElseThrow(() -> new NotFoundException("Failed to find HTTP state port")); HttpURL url = HttpURL.create(Scheme.http, DomainName.of(host.getHostname()), port.getPort(), path, query); - HttpResponse response = fetcher.get(new Params(2000), // 2_000 ms read timeout + HttpResponse response = fetcher.get(new Params(29_000), // 29 sec (30 sec on controller) url.asURI()); return forwardedUrl == null ? response : new UrlRewritingProxyResponse(response, url, forwardedUrl); } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/ApplicationFileManager.java b/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/ApplicationFileManager.java index 8072dab978f..727ca4c44d8 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/ApplicationFileManager.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/ApplicationFileManager.java @@ -26,10 +26,12 @@ public class ApplicationFileManager implements AddFileInterface { private final File applicationDir; private final FileDirectory fileDirectory; + private final boolean isHosted; - ApplicationFileManager(File applicationDir, FileDirectory fileDirectory) { + ApplicationFileManager(File applicationDir, FileDirectory fileDirectory, boolean isHosted) { this.applicationDir = applicationDir; this.fileDirectory = fileDirectory; + this.isHosted = isHosted; } @Override @@ -44,6 +46,7 @@ public class ApplicationFileManager implements AddFileInterface { @Override public FileReference addUri(String uri, Path path) { + if (isHosted) throw new IllegalArgumentException("URI type resources are not supported in this Vespa cloud"); try (TmpDir tmp = new TmpDir()) { return addFile(download(uri, tmp.dir, path)); } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/FileDistributionFactory.java b/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/FileDistributionFactory.java index 1027cc6a237..1d162b7af9a 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/FileDistributionFactory.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/FileDistributionFactory.java @@ -36,7 +36,7 @@ public class FileDistributionFactory implements AutoCloseable { } public AddFileInterface createFileManager(File applicationDir) { - return new ApplicationFileManager(applicationDir, new FileDirectory(getFileReferencesDir())); + return new ApplicationFileManager(applicationDir, new FileDirectory(getFileReferencesDir()), configserverConfig.hostedVespa()); } protected File getFileReferencesDir() { diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/rpc/RpcServer.java b/configserver/src/main/java/com/yahoo/vespa/config/server/rpc/RpcServer.java index ebf1fb32141..e5dede9af8a 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/rpc/RpcServer.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/rpc/RpcServer.java @@ -379,7 +379,7 @@ public class RpcServer implements Runnable, ReloadListener, TenantListener { if (completionService == null) { executorService.submit(task); } else { - completionService.submit(() -> { task.run();return true;}); + completionService.submit(() -> { task.run(); return true; }); } updateWorkQueueMetrics(); return true; diff --git a/configserver/src/main/java/com/yahoo/vespa/serviceview/Cluster.java b/configserver/src/main/java/com/yahoo/vespa/serviceview/Cluster.java deleted file mode 100644 index 7e2a83b6b9a..00000000000 --- a/configserver/src/main/java/com/yahoo/vespa/serviceview/Cluster.java +++ /dev/null @@ -1,78 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.serviceview; - -import java.util.Arrays; -import java.util.List; - -import com.google.common.collect.ImmutableList; - -/** - * Model a single cluster of services in the Vespa model. - * - * @author Steinar Knutsen - */ -public final class Cluster implements Comparable<Cluster> { - - public final String name; - public final String type; - /** - * An ordered list of the service instances in this cluster. - */ - public final ImmutableList<Service> services; - - public Cluster(String name, String type, List<Service> services) { - this.name = name; - this.type = type; - ImmutableList.Builder<Service> builder = ImmutableList.builder(); - Service[] sortingBuffer = services.toArray(new Service[0]); - Arrays.sort(sortingBuffer); - builder.add(sortingBuffer); - this.services = builder.build(); - } - - @Override - public int compareTo(Cluster other) { - int nameOrder = name.compareTo(other.name); - if (nameOrder != 0) { - return nameOrder; - } - return type.compareTo(other.type); - } - - @Override - public int hashCode() { - final int prime = 761; - int result = 1; - result = prime * result + name.hashCode(); - result = prime * result + type.hashCode(); - return result; - } - - @Override - public boolean equals(Object obj) { - if (this == obj) { - return true; - } - if (obj == null) { - return false; - } - if (getClass() != obj.getClass()) { - return false; - } - Cluster other = (Cluster) obj; - if (!name.equals(other.name)) { - return false; - } - return type.equals(other.type); - } - - @Override - public String toString() { - final int maxLen = 3; - StringBuilder builder = new StringBuilder(); - builder.append("Cluster [name=").append(name).append(", type=").append(type).append(", services=") - .append(services.subList(0, Math.min(services.size(), maxLen))).append("]"); - return builder.toString(); - } - -} diff --git a/configserver/src/main/java/com/yahoo/vespa/serviceview/ProxyErrorMapper.java b/configserver/src/main/java/com/yahoo/vespa/serviceview/ProxyErrorMapper.java deleted file mode 100644 index 545e9b3ddc8..00000000000 --- a/configserver/src/main/java/com/yahoo/vespa/serviceview/ProxyErrorMapper.java +++ /dev/null @@ -1,24 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.serviceview; - -import javax.ws.rs.WebApplicationException; -import javax.ws.rs.core.Response; -import javax.ws.rs.ext.ExceptionMapper; -import javax.ws.rs.ext.Provider; - -/** - * Convert exceptions thrown by the internal REST client into a little more helpful responses. - * - * @author Steinar Knutsen - */ -@Provider -public class ProxyErrorMapper implements ExceptionMapper<WebApplicationException> { - - @Override - public Response toResponse(WebApplicationException exception) { - StringBuilder msg = new StringBuilder("Invoking (external) web service failed: "); - msg.append(exception.getMessage()); - return Response.status(500).entity(msg.toString()).type("text/plain").build(); - } - -} diff --git a/configserver/src/main/java/com/yahoo/vespa/serviceview/Service.java b/configserver/src/main/java/com/yahoo/vespa/serviceview/Service.java deleted file mode 100644 index 280ae1fa6c1..00000000000 --- a/configserver/src/main/java/com/yahoo/vespa/serviceview/Service.java +++ /dev/null @@ -1,163 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.serviceview; - -import java.math.BigInteger; -import java.security.MessageDigest; -import java.security.NoSuchAlgorithmException; -import java.util.ArrayList; -import java.util.List; - -import com.google.common.collect.ImmutableList; -import com.yahoo.text.Utf8; - -/** - * Model a single service instance as a sortable object. - * - * @author Steinar Knutsen - */ -public final class Service implements Comparable<Service> { - - public final String serviceType; - public final String host; - public final int statePort; - public final String configId; - public final List<Integer> ports; - public final String name; - - public Service(String serviceType, String host, int statePort, String clusterName, String clusterType, - String configId, List<Integer> ports, String name) { - this.serviceType = serviceType; - this.host = host.toLowerCase(); - this.statePort = statePort; - this.configId = configId; - ImmutableList.Builder<Integer> portsBuilder = new ImmutableList.Builder<>(); - portsBuilder.addAll(ports); - this.ports = portsBuilder.build(); - this.name = name; - } - - @Override - public int compareTo(Service other) { - int serviceTypeOrder = serviceType.compareTo(other.serviceType); - if (serviceTypeOrder != 0) { - return serviceTypeOrder; - } - int hostOrder = host.compareTo(other.host); - if (hostOrder != 0) { - return hostOrder; - } - return Integer.compare(statePort, other.statePort); - } - - /** - * Generate an identifier string for one of the ports of this service - * suitable for using in an URL. - * - * @param port - * port which this identifier pertains to - * @return an opaque identifier string for this service - */ - public String getIdentifier(int port) { - StringBuilder b = new StringBuilder(serviceType); - b.append("-"); - MessageDigest md5; - try { - md5 = MessageDigest.getInstance("MD5"); - } catch (NoSuchAlgorithmException e) { - throw new RuntimeException("MD5 should by definition always be available in the JVM.", e); - } - md5.update(Utf8.toBytes(serviceType)); - md5.update(Utf8.toBytes(configId)); - md5.update(Utf8.toBytes(host)); - for (int i = 3; i >= 0; --i) { - md5.update((byte) (port >>> i)); - } - byte[] digest = md5.digest(); - BigInteger digestMarshal = new BigInteger(1, digest); - b.append(digestMarshal.toString(36)); - return b.toString(); - } - - /** - * All valid identifiers for this object. - * - * @return a list with a unique ID for each of this service's ports - */ - public List<String> getIdentifiers() { - List<String> ids = new ArrayList<>(ports.size()); - for (int port : ports) { - ids.add(getIdentifier(port)); - } - return ids; - } - - /** - * Find which port number a hash code pertains to. - * - * @param identifier a string generated from {@link #getIdentifier(int)} - * @return a port number, or 0 if no match is found - */ - public int matchIdentifierWithPort(String identifier) { - for (int port : ports) { - if (identifier.equals(getIdentifier(port))) { - return port; - } - } - throw new IllegalArgumentException("Identifier " + identifier + " matches no ports in " + this); - } - - @Override - public String toString() { - final int maxLen = 3; - StringBuilder builder = new StringBuilder(); - builder.append("Service [serviceType=").append(serviceType).append(", host=").append(host).append(", statePort=") - .append(statePort).append(", configId=").append(configId).append(", ports=") - .append(ports.subList(0, Math.min(ports.size(), maxLen))).append(", name=").append(name) - .append("]"); - return builder.toString(); - } - - @Override - public int hashCode() { - final int prime = 131; - int result = 1; - result = prime * result + configId.hashCode(); - result = prime * result + host.hashCode(); - result = prime * result + name.hashCode(); - result = prime * result + ports.hashCode(); - result = prime * result + serviceType.hashCode(); - result = prime * result + statePort; - return result; - } - - @Override - public boolean equals(Object obj) { - if (this == obj) { - return true; - } - if (obj == null) { - return false; - } - if (getClass() != obj.getClass()) { - return false; - } - Service other = (Service) obj; - if (!configId.equals(other.configId)) { - return false; - } - if (!host.equals(other.host)) { - return false; - } - if (!name.equals(other.name)) { - return false; - } - if (!ports.equals(other.ports)) { - return false; - } - if (!serviceType.equals(other.serviceType)) { - return false; - } - return statePort == other.statePort; - } - -} diff --git a/configserver/src/main/java/com/yahoo/vespa/serviceview/ServiceModel.java b/configserver/src/main/java/com/yahoo/vespa/serviceview/ServiceModel.java deleted file mode 100644 index f3e327ead32..00000000000 --- a/configserver/src/main/java/com/yahoo/vespa/serviceview/ServiceModel.java +++ /dev/null @@ -1,232 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.serviceview; - -import java.util.ArrayList; -import java.util.Collections; -import java.util.HashMap; -import java.util.List; -import java.util.Map; - -import com.google.common.collect.HashBasedTable; -import com.google.common.collect.ImmutableList; -import com.google.common.collect.ImmutableMap; -import com.google.common.collect.Table; -import com.google.common.collect.Table.Cell; -import com.yahoo.vespa.serviceview.bindings.ApplicationView; -import com.yahoo.vespa.serviceview.bindings.ClusterView; -import com.yahoo.vespa.serviceview.bindings.HostService; -import com.yahoo.vespa.serviceview.bindings.ModelResponse; -import com.yahoo.vespa.serviceview.bindings.ServicePort; -import com.yahoo.vespa.serviceview.bindings.ServiceView; - -import static com.yahoo.config.model.api.container.ContainerServiceType.CLUSTERCONTROLLER_CONTAINER; - -/** - * A transposed view for cloud.config.model. - * - * @author Steinar Knutsen - */ -public final class ServiceModel { - - private static final String CONTENT_CLUSTER_TYPENAME = "content"; - - private final Map<String, Service> servicesMap; - - /** - * An ordered list of the clusters in this config model. - */ - public final ImmutableList<Cluster> clusters; - - ServiceModel(ModelResponse modelConfig) { - Table<String, String, List<Service>> services = HashBasedTable.create(); - for (HostService h : modelConfig.hosts) { - String hostName = h.name; - for (com.yahoo.vespa.serviceview.bindings.Service s : h.services) { - addService(services, hostName, s); - } - } - List<Cluster> sortingBuffer = new ArrayList<>(); - for (Cell<String, String, List<Service>> c : services.cellSet()) { - sortingBuffer.add(new Cluster(c.getRowKey(), c.getColumnKey(), c.getValue())); - } - Collections.sort(sortingBuffer); - ImmutableList.Builder<Cluster> clustersBuilder = new ImmutableList.Builder<>(); - clustersBuilder.addAll(sortingBuffer); - clusters = clustersBuilder.build(); - Map<String, Service> seenIdentifiers = new HashMap<>(); - for (Cluster c : clusters) { - for (Service s : c.services) { - List<String> identifiers = s.getIdentifiers(); - for (String identifier : identifiers) { - if (seenIdentifiers.containsKey(identifier)) { - throw new RuntimeException("Hash collision" + " between " + - seenIdentifiers.get(identifier) + " and " + s + "."); - } - seenIdentifiers.put(identifier, s); - } - } - } - ImmutableMap.Builder<String, Service> servicesBuilder = new ImmutableMap.Builder<>(); - servicesBuilder.putAll(seenIdentifiers); - servicesMap = servicesBuilder.build(); - } - - private static void addService(Table<String, String, List<Service>> services, String hostName, - com.yahoo.vespa.serviceview.bindings.Service s) { - boolean hasStateApi = false; - int statePort = 0; - List<Integer> ports = new ArrayList<>(s.ports.size()); - for (ServicePort port : s.ports) { - ports.add(port.number); - if (!hasStateApi && port.hasTags("http", "state")) { - hasStateApi = true; - statePort = port.number; - } - } - // ignore hosts without state API - if (hasStateApi) { - Service service = new Service(s.type, hostName, statePort, s.clustername, s.clustertype, s.configid, ports, s.name); - getAndSetEntry(services, s.clustername, s.clustertype).add(service); - } - } - - private static List<Service> getAndSetEntry(Table<String, String, List<Service>> services, String clusterName, String clusterType) { - List<Service> serviceList = services.get(clusterName, clusterType); - if (serviceList == null) { - serviceList = new ArrayList<>(); - services.put(clusterName, clusterType, serviceList); - } - return serviceList; - } - - /** - * The top level view of a given application. - * - * @return a top level view of the entire application in a form suitable for - * consumption by a REST API - */ - public ApplicationView showAllClusters(String uriBase, String applicationIdentifier) { - ApplicationView response = new ApplicationView(); - List<ClusterView> clusterViews = new ArrayList<>(); - for (Cluster c : clusters) { - clusterViews.add(showCluster(c, uriBase, applicationIdentifier)); - } - response.clusters = clusterViews; - return response; - } - - private ClusterView showCluster(Cluster c, String uriBase, String applicationIdentifier) { - List<ServiceView> services = new ArrayList<>(); - for (Service s : c.services) { - ServiceView service = new ServiceView(); - StringBuilder buffer = getLinkBuilder(uriBase).append(applicationIdentifier).append('/'); - service.url = buffer.append("service/").append(s.getIdentifier(s.statePort)).append("/state/v1/").toString(); - service.serviceType = s.serviceType; - service.serviceName = s.name; - service.configId = s.configId; - service.host = s.host; - addLegacyLink(uriBase, applicationIdentifier, s, service); - services.add(service); - } - ClusterView v = new ClusterView(); - v.services = services; - v.name = c.name; - v.type = c.type; - if (CONTENT_CLUSTER_TYPENAME.equals(c.type)) { - Service s = getFirstClusterController(); - StringBuilder buffer = getLinkBuilder(uriBase).append(applicationIdentifier).append('/'); - buffer.append("service/").append(s.getIdentifier(s.statePort)).append("/cluster/v2/").append(c.name); - v.url = buffer.toString(); - } else { - v.url = null; - } - return v; - } - - private void addLegacyLink(String uriBase, String applicationIdentifier, Service s, ServiceView service) { - if (s.serviceType.equals("storagenode") || s.serviceType.equals("distributor")) { - StringBuilder legacyBuffer = getLinkBuilder(uriBase); - legacyBuffer.append("legacy/").append(applicationIdentifier).append('/'); - legacyBuffer.append("service/").append(s.getIdentifier(s.statePort)).append('/'); - service.legacyStatusPages = legacyBuffer.toString(); - } - } - - private Service getFirstServiceInstanceByType(String typeName) { - for (Cluster c : clusters) { - for (Service s : c.services) { - if (typeName.equals(s.serviceType)) { - return s; - } - } - } - throw new IllegalStateException("This installation has but no service of required type: " - + typeName + "."); - } - - private Service getFirstClusterController() { - // This is used assuming all cluster controllers know of all fleet controllers in an application - return getFirstServiceInstanceByType(CLUSTERCONTROLLER_CONTAINER.serviceName); - } - - private StringBuilder getLinkBuilder(String uriBase) { - StringBuilder buffer = new StringBuilder(uriBase); - if (!uriBase.endsWith("/")) { - buffer.append('/'); - } - return buffer; - } - - @Override - public String toString() { - final int maxLen = 3; - StringBuilder builder = new StringBuilder(); - builder.append("ServiceModel [clusters=") - .append(clusters.subList(0, Math.min(clusters.size(), maxLen))).append("]"); - return builder.toString(); - } - - - /** - * Match an identifier with a service for this cluster. - * - * @param identifier - * an opaque service identifier generated by the service - * @return the corresponding Service instance - */ - public Service getService(String identifier) { - return servicesMap.get(identifier); - } - - /** - * Find a service based on host and port. - * - * @param host - * the name of the host running the service - * @param port - * a port owned by the service - * @param self - * the service which generated the host data - * @return a service instance fullfilling the criteria - * @throws IllegalArgumentException - * if no matching service is found - */ - public Service resolve(String host, int port, Service self) { - Integer portAsObject = port; - String realHost; - if ("localhost".equals(host)) { - realHost = self.host; - } else { - realHost = host; - } - for (Cluster c : clusters) { - for (Service s : c.services) { - if (s.host.equals(realHost) && s.ports.contains(portAsObject)) { - return s; - } - } - } - throw new IllegalArgumentException("No registered service owns port " + port + " on host " + realHost + "."); - } - -} diff --git a/configserver/src/main/java/com/yahoo/vespa/serviceview/StateRequestHandler.java b/configserver/src/main/java/com/yahoo/vespa/serviceview/StateRequestHandler.java deleted file mode 100644 index 59b91a52791..00000000000 --- a/configserver/src/main/java/com/yahoo/vespa/serviceview/StateRequestHandler.java +++ /dev/null @@ -1,253 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.serviceview; - -import com.fasterxml.jackson.jaxrs.json.JacksonJsonProvider; -import com.google.inject.Inject; -import com.yahoo.cloud.config.ConfigserverConfig; -import com.yahoo.container.jdisc.ThreadedHttpRequestHandler; -import ai.vespa.http.DomainName; -import ai.vespa.http.HttpURL; -import ai.vespa.http.HttpURL.Path; -import ai.vespa.http.HttpURL.Query; -import ai.vespa.http.HttpURL.Scheme; -import com.yahoo.restapi.RestApi; -import com.yahoo.restapi.RestApiRequestHandler; -import com.yahoo.vespa.serviceview.bindings.ApplicationView; -import com.yahoo.vespa.serviceview.bindings.ConfigClient; -import com.yahoo.vespa.serviceview.bindings.HealthClient; -import com.yahoo.vespa.serviceview.bindings.ModelResponse; -import org.glassfish.jersey.client.ClientProperties; -import org.glassfish.jersey.client.proxy.WebResourceFactory; - -import javax.ws.rs.client.Client; -import javax.ws.rs.client.ClientRequestFilter; -import javax.ws.rs.client.WebTarget; -import javax.ws.rs.core.HttpHeaders; -import java.net.URI; -import java.net.URISyntaxException; -import java.util.HashMap; -import java.util.List; -import java.util.ListIterator; -import java.util.Map; - -/** - * A web service to discover and proxy Vespa service state info. - * - * @author Steinar Knutsen - * @author bjorncs - */ -public class StateRequestHandler extends RestApiRequestHandler<StateRequestHandler> { - - private static final String USER_AGENT = "service-view-config-server-client"; - private static final String SINGLE_API_LINK = "url"; - - @SuppressWarnings("removal") - private final Client client = new ai.vespa.util.http.VespaClientBuilderFactory() - .newBuilder() - .property(ClientProperties.CONNECT_TIMEOUT, 10000) - .property(ClientProperties.READ_TIMEOUT, 10000) - .register(JacksonJsonProvider.class) - .register((ClientRequestFilter) ctx -> ctx.getHeaders().put(HttpHeaders.USER_AGENT, List.of(USER_AGENT))) - .build(); - - private final int restApiPort; - - private static class GiveUpLinkRetargetingException extends Exception { - public GiveUpLinkRetargetingException(Throwable reason) { - super(reason); - } - - public GiveUpLinkRetargetingException(String message) { - super(message); - } - } - - @Inject - public StateRequestHandler(ThreadedHttpRequestHandler.Context context, - ConfigserverConfig configserverConfig) { - super(context, StateRequestHandler::createRestApiDefinition); - this.restApiPort = configserverConfig.httpport(); - } - - @Override - protected void destroy() { - client.close(); - super.destroy(); - } - - private static RestApi createRestApiDefinition(StateRequestHandler self) { - return RestApi.builder() - .addRoute(RestApi.route("/serviceview/v1") - .get(self::getDefaultUserInfo)) - .addRoute(RestApi.route("/serviceview/v1/") - .get(self::getDefaultUserInfo)) - .addRoute(RestApi.route("/serviceview/v1/tenant/{tenantName}/application/{applicationName}/environment/{environmentName}/region/{regionName}/instance/{instanceName}") - .get(self::getUserInfo)) - .addRoute(RestApi.route("/serviceview/v1/tenant/{tenantName}/application/{applicationName}/environment/{environmentName}/region/{regionName}/instance/{instanceName}/service/{serviceIdentifier}/{*}") - .get(self::singleService)) - .registerJacksonResponseEntity(HashMap.class) - .registerJacksonResponseEntity(ApplicationView.class) - .build(); - } - - private ApplicationView getDefaultUserInfo(RestApi.RequestContext context) { - return getUserInfo(context.baseRequestURL(), "default", "default", "default", "default", "default"); - } - - private ApplicationView getUserInfo(RestApi.RequestContext context) { - String tenantName = context.pathParameters().getStringOrThrow("tenantName"); - String applicationName = context.pathParameters().getStringOrThrow("applicationName"); - String environmentName = context.pathParameters().getStringOrThrow("environmentName"); - String regionName = context.pathParameters().getStringOrThrow("regionName"); - String instanceName = context.pathParameters().getStringOrThrow("instanceName"); - return getUserInfo(context.baseRequestURL(), tenantName, applicationName, environmentName, regionName, instanceName); - } - - public HashMap<?, ?> singleService(RestApi.RequestContext context) { - String tenantName = context.pathParameters().getStringOrThrow("tenantName"); - String applicationName = context.pathParameters().getStringOrThrow("applicationName"); - String environmentName = context.pathParameters().getStringOrThrow("environmentName"); - String regionName = context.pathParameters().getStringOrThrow("regionName"); - String instanceName = context.pathParameters().getStringOrThrow("instanceName"); - String identifier = context.pathParameters().getStringOrThrow("serviceIdentifier"); - Path apiParams = context.pathParameters().getRest().orElse(Path.empty()); - Query apiQuery = context.queryParameters().getFullQuery(); - return singleService(context.baseRequestURL(), tenantName, applicationName, environmentName, regionName, instanceName, identifier, apiParams, apiQuery); - } - - protected ApplicationView getUserInfo(HttpURL url, String tenantName, String applicationName, String environmentName, String regionName, String instanceName) { - ServiceModel model = new ServiceModel( - getModelConfig(tenantName, applicationName, environmentName, regionName, instanceName)); - return model.showAllClusters( - baseUri(url).toString(), - applicationIdentifier(tenantName, applicationName, environmentName, regionName, instanceName)); - } - - protected ModelResponse getModelConfig(String tenant, String application, String environment, String region, String instance) { - WebTarget target = client.target("http://localhost:" + restApiPort + "/"); - ConfigClient resource = WebResourceFactory.newResource(ConfigClient.class, target); - return resource.getServiceModel(tenant, application, environment, region, instance); - } - - protected HashMap<?, ?> singleService( - HttpURL url, String tenantName, String applicationName, String environmentName, String regionName, String instanceName, String identifier, Path path, Query query) { - ServiceModel model = new ServiceModel(getModelConfig(tenantName, applicationName, environmentName, regionName, instanceName)); - Service s = model.getService(identifier); - int requestedPort = s.matchIdentifierWithPort(identifier); - HealthClient resource = getHealthClient(path, s, requestedPort, query, client); - HashMap<?, ?> apiResult = resource.getHealthInfo(); - rewriteResourceLinks(url, apiResult, model, s, applicationIdentifier(tenantName, applicationName, environmentName, regionName, instanceName), identifier); - return apiResult; - } - - protected HealthClient getHealthClient(Path apiParams, Service s, int requestedPort, Query query, Client client) { - URI uri = HttpURL.create(Scheme.http, DomainName.of(s.host), requestedPort, apiParams, query).asURI(); - WebTarget target = client.target(uri); - return WebResourceFactory.newResource(HealthClient.class, target); - } - - private String applicationIdentifier(String tenant, String application, String environment, String region, String instance) { - return "tenant/" + tenant - + "/application/" + application - + "/environment/" + environment - + "/region/" + region - + "/instance/" + instance; - } - - private void rewriteResourceLinks(HttpURL url, - Object apiResult, - ServiceModel model, - Service self, - String applicationIdentifier, - String incomingIdentifier) { - if (apiResult instanceof List) { - for (@SuppressWarnings("unchecked") ListIterator<Object> i = ((List<Object>) apiResult).listIterator(); i.hasNext();) { - Object resource = i.next(); - if (resource instanceof String) { - try { - StringBuilder buffer = linkBuffer(url, applicationIdentifier); - // if it points to a port and host not part of the application, rewriting will not occur, so this is kind of safe - retarget(model, self, buffer, (String) resource); - i.set(buffer.toString()); - } catch (GiveUpLinkRetargetingException e) { - break; // assume relatively homogenous lists when doing rewrites to avoid freezing up on scanning long lists - } - } else { - rewriteResourceLinks(url, resource, model, self, applicationIdentifier, incomingIdentifier); - } - } - } else if (apiResult instanceof Map) { - @SuppressWarnings("unchecked") - Map<Object, Object> api = (Map<Object, Object>) apiResult; - for (Map.Entry<Object, Object> entry : api.entrySet()) { - if (SINGLE_API_LINK.equals(entry.getKey()) && entry.getValue() instanceof String) { - try { - rewriteSingleLink(entry, model, self, linkBuffer(url, applicationIdentifier)); - } catch (GiveUpLinkRetargetingException e) { - // NOP - } - } else if ("link".equals(entry.getKey()) && entry.getValue() instanceof String) { - buildSingleLink(entry, linkBuffer(url, applicationIdentifier), incomingIdentifier); - } else { - rewriteResourceLinks(url, entry.getValue(), model, self, applicationIdentifier, incomingIdentifier); - } - } - } - } - - private void buildSingleLink(Map.Entry<Object, Object> entry, - StringBuilder newUri, - String incomingIdentifier) { - newUri.append("/service/") - .append(incomingIdentifier); - newUri.append(entry.getValue()); - entry.setValue(newUri.toString()); - } - - private void addQuery(String query, StringBuilder newUri) { - if (query != null && query.length() > 0) { - newUri.append('?').append(query); - } - } - - private StringBuilder linkBuffer(HttpURL url, String applicationIdentifier) { - return new StringBuilder(baseUri(url).appendPath(Path.parse(applicationIdentifier)).toString()); - } - - private void rewriteSingleLink(Map.Entry<Object, Object> entry, - ServiceModel model, - Service self, - StringBuilder newUri) throws GiveUpLinkRetargetingException { - String url = (String) entry.getValue(); - retarget(model, self, newUri, url); - entry.setValue(newUri.toString()); - } - - private void retarget(ServiceModel model, Service self, StringBuilder newUri, String url) throws GiveUpLinkRetargetingException { - URI link; - try { - link = new URI(url); - } catch (URISyntaxException e) { - throw new GiveUpLinkRetargetingException(e); - } - if (!link.isAbsolute()) { - throw new GiveUpLinkRetargetingException("This rewriting only supports absolute URIs."); - } - int linkPort = link.getPort(); - if (linkPort == -1) { - linkPort = 80; - } - Service s; - try { - s = model.resolve(link.getHost(), linkPort, self); - } catch (IllegalArgumentException e) { - throw new GiveUpLinkRetargetingException(e); - } - newUri.append("/service/").append(s.getIdentifier(linkPort)); - newUri.append(link.getRawPath()); - } - - private static HttpURL baseUri(HttpURL url) { - return url.withPath(Path.parse("/serviceview/v1/")); - } -} diff --git a/configserver/src/main/java/com/yahoo/vespa/serviceview/package-info.java b/configserver/src/main/java/com/yahoo/vespa/serviceview/package-info.java deleted file mode 100644 index 72cfaf0d05d..00000000000 --- a/configserver/src/main/java/com/yahoo/vespa/serviceview/package-info.java +++ /dev/null @@ -1,13 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -/** - * Home of the centralised service view implementation. The service view is a - * REST API for discovering and accessing the state API for any service in a - * Vespa cluster. - * - * <p>Do note this package is in its prototyping stage and classes <i>will</i> - * be renamed and moved around a little.</p> - */ -@ExportPackage -package com.yahoo.vespa.serviceview; - -import com.yahoo.osgi.annotation.ExportPackage; diff --git a/configserver/src/main/resources/configserver-app/services.xml b/configserver/src/main/resources/configserver-app/services.xml index b3a37c6f669..3536cfc7942 100644 --- a/configserver/src/main/resources/configserver-app/services.xml +++ b/configserver/src/main/resources/configserver-app/services.xml @@ -83,10 +83,6 @@ <binding>http://*/orchestrator/v1/instances</binding> <binding>http://*/orchestrator/v1/instances/*</binding> </handler> - <handler id="com.yahoo.vespa.serviceview.StateRequestHandler" bundle="configserver"> - <binding>http://*/serviceview/v1</binding> - <binding>http://*/serviceview/v1/*</binding> - </handler> <handler id='com.yahoo.vespa.config.server.http.HttpGetConfigHandler' bundle='configserver'> <binding>http://*/config/v1/*/*</binding> <binding>http://*/config/v1/*</binding> diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/application/HttpProxyTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/application/HttpProxyTest.java index 80e998521a9..e3820ff99be 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/application/HttpProxyTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/application/HttpProxyTest.java @@ -2,13 +2,13 @@ package com.yahoo.vespa.config.server.application; import ai.vespa.http.HttpURL; +import ai.vespa.http.HttpURL.Path; import ai.vespa.http.HttpURL.Query; import com.yahoo.config.model.api.HostInfo; import com.yahoo.config.model.api.Model; import com.yahoo.config.model.api.ServiceInfo; import com.yahoo.config.provision.ClusterSpec; import com.yahoo.container.jdisc.HttpResponse; -import ai.vespa.http.HttpURL.Path; import com.yahoo.slime.Slime; import com.yahoo.slime.SlimeUtils; import com.yahoo.vespa.config.server.http.HttpFetcher; @@ -21,8 +21,6 @@ import org.mockito.ArgumentCaptor; import java.io.ByteArrayOutputStream; import java.io.IOException; import java.net.URI; -import java.net.URL; -import java.nio.charset.StandardCharsets; import java.util.Arrays; import java.util.Collections; @@ -30,7 +28,6 @@ import static com.yahoo.config.model.api.container.ContainerServiceType.CLUSTERC import static com.yahoo.vespa.config.server.application.MockModel.createServiceInfo; import static java.nio.charset.StandardCharsets.UTF_8; import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertNotSame; import static org.junit.Assert.assertSame; import static org.mockito.ArgumentMatchers.any; import static org.mockito.Mockito.doAnswer; @@ -53,7 +50,7 @@ public class HttpProxyTest { } @Test - public void testNormalGet() throws Exception { + public void testNormalGet() { ArgumentCaptor<HttpFetcher.Params> actualParams = ArgumentCaptor.forClass(HttpFetcher.Params.class); ArgumentCaptor<URI> actualUrl = ArgumentCaptor.forClass(URI.class); HttpResponse response = new StaticResponse(200, "application/json", "body"); @@ -64,7 +61,7 @@ public class HttpProxyTest { Query.parse("foo=bar")); assertEquals(1, actualParams.getAllValues().size()); - assertEquals(2000, actualParams.getValue().readTimeoutMs); + assertEquals(29000, actualParams.getValue().readTimeoutMs); assertEquals(1, actualUrl.getAllValues().size()); assertEquals(URI.create("http://" + hostname + ":" + port + "/clustercontroller-status/v1/clusterName?foo=bar"), diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/filedistribution/FileDBRegistryTestCase.java b/configserver/src/test/java/com/yahoo/vespa/config/server/filedistribution/FileDBRegistryTestCase.java index 6d7074aef3c..7ed5388129d 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/filedistribution/FileDBRegistryTestCase.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/filedistribution/FileDBRegistryTestCase.java @@ -14,6 +14,7 @@ import java.nio.charset.StandardCharsets; import java.util.Set; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertThrows; import static org.junit.Assert.fail; /** @@ -32,10 +33,18 @@ public class FileDBRegistryTestCase { private static final FileReference FOO_REF = new FileReference("b5ce94ca1feae86c"); @Test + public void uriResourcesNotSupportedWhenHosted() { + assertEquals("URI type resources are not supported in this Vespa cloud", + assertThrows(IllegalArgumentException.class, + () -> new ApplicationFileManager(null, null, true).addUri(null, null)) + .getMessage()); + } + + @Test public void importAndExport() throws IOException { TemporaryFolder tmpDir = new TemporaryFolder(); tmpDir.create(); - AddFileInterface fileManager = new ApplicationFileManager(new File(APP), new FileDirectory(tmpDir.newFolder())); + AddFileInterface fileManager = new ApplicationFileManager(new File(APP), new FileDirectory(tmpDir.newFolder()), false); FileRegistry fileRegistry = new FileDBRegistry(fileManager); assertEquals(FOO_REF, fileRegistry.addFile(FOO_FILE)); try { diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/filedistribution/MockFileRegistry.java b/configserver/src/test/java/com/yahoo/vespa/config/server/filedistribution/MockFileRegistry.java index 9a9c224627a..5cda7521188 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/filedistribution/MockFileRegistry.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/filedistribution/MockFileRegistry.java @@ -23,7 +23,7 @@ public class MockFileRegistry implements FileRegistry { public MockFileRegistry(File applicationDir, File rootPath) { FileDirectory fileDirectory = new FileDirectory(rootPath); - this.addFileInterface = new ApplicationFileManager(applicationDir, fileDirectory); + this.addFileInterface = new ApplicationFileManager(applicationDir, fileDirectory, false); } public FileReference addFile(String relativePath) { diff --git a/configserver/src/test/java/com/yahoo/vespa/serviceview/ServiceModelTest.java b/configserver/src/test/java/com/yahoo/vespa/serviceview/ServiceModelTest.java deleted file mode 100644 index 44bc2c15e8e..00000000000 --- a/configserver/src/test/java/com/yahoo/vespa/serviceview/ServiceModelTest.java +++ /dev/null @@ -1,111 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.serviceview; - -import com.yahoo.vespa.defaults.Defaults; -import com.yahoo.vespa.serviceview.bindings.ApplicationView; -import com.yahoo.vespa.serviceview.bindings.HostService; -import com.yahoo.vespa.serviceview.bindings.ModelResponse; -import com.yahoo.vespa.serviceview.bindings.ServicePort; -import com.yahoo.vespa.serviceview.bindings.ServiceView; -import org.junit.After; -import org.junit.Before; -import org.junit.Test; - -import java.util.Arrays; -import java.util.Collections; - -import static com.yahoo.config.model.api.container.ContainerServiceType.CLUSTERCONTROLLER_CONTAINER; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertNotNull; - -/** - * Functional tests for the programmatic view of cloud.config.model. - * - * @author Steinar Knutsen - */ -public class ServiceModelTest { - - private ServiceModel model; - - @Before - public void setUp() { - ModelResponse model = syntheticModelResponse(); - this.model = new ServiceModel(model); - } - - static ModelResponse syntheticModelResponse() { - ModelResponse model = new ModelResponse(); - HostService h = new HostService(); - h.name = "vespa.yahoo.com"; - com.yahoo.vespa.serviceview.bindings.Service service0 = new com.yahoo.vespa.serviceview.bindings.Service(); - { - service0.clustername = "examplecluster"; - service0.clustertype = "somethingservers"; - service0.index = 1L; - service0.type = "something"; - service0.name = "examplename"; - service0.configid = "blblb/lbl.0"; - ServicePort port = new ServicePort(); - port.number = Defaults.getDefaults().vespaWebServicePort(); - port.tags = "state http"; - service0.ports = Collections.singletonList(port); - } - com.yahoo.vespa.serviceview.bindings.Service service1 = new com.yahoo.vespa.serviceview.bindings.Service(); - { - service1.clustername = "examplecluster"; - service1.clustertype = "somethingservers"; - service1.index = 2L; - service1.type = CLUSTERCONTROLLER_CONTAINER.serviceName; - service1.name = "clustercontroller"; - service1.configid = "clustercontroller/lbl.0"; - ServicePort port = new ServicePort(); - port.number = 4090; - port.tags = "state http"; - service1.ports = Collections.singletonList(port); - } - com.yahoo.vespa.serviceview.bindings.Service service2 = new com.yahoo.vespa.serviceview.bindings.Service(); - { - service2.clustername = "tralala"; - service2.clustertype = "admin"; - service2.index = 3L; - service2.type = "configserver"; - service2.name = "configservername"; - service2.configid = "clustercontroller/lbl.0"; - ServicePort port = new ServicePort(); - port.number = 5000; - port.tags = "state http"; - service2.ports = Collections.singletonList(port); - } - h.services = Arrays.asList(service0, service1, service2); - model.hosts = Collections.singletonList(h); - return model; - } - - @After - public void tearDown() { - model = null; - } - - @Test - public final void test() { - final String uriBase = "http://configserver:5000/"; - ApplicationView x = model.showAllClusters(uriBase, "/tenant/default/application/default"); - assertEquals(2, x.clusters.size()); - String urlTracking = null; - for (com.yahoo.vespa.serviceview.bindings.ClusterView c : x.clusters) { - for (ServiceView s : c.services) { - if ("examplename".equals(s.serviceName)) { - assertEquals("something", s.serviceType); - urlTracking = s.url; - break; - } - } - } - assertNotNull(urlTracking); - final String serviceIdentifier = urlTracking.substring(urlTracking.indexOf("something"), - urlTracking.length() - "/state/v1/".length()); - Service y = model.getService(serviceIdentifier); - assertEquals("examplename", y.name); - } - -} diff --git a/configserver/src/test/java/com/yahoo/vespa/serviceview/StateRequestHandlerTest.java b/configserver/src/test/java/com/yahoo/vespa/serviceview/StateRequestHandlerTest.java deleted file mode 100644 index 79ed38b2025..00000000000 --- a/configserver/src/test/java/com/yahoo/vespa/serviceview/StateRequestHandlerTest.java +++ /dev/null @@ -1,93 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.serviceview; - -import ai.vespa.http.HttpURL; -import ai.vespa.http.HttpURL.Query; -import com.yahoo.cloud.config.ConfigserverConfig; -import com.yahoo.jdisc.test.MockMetric; -import ai.vespa.http.HttpURL.Path; -import com.yahoo.vespa.serviceview.bindings.ApplicationView; -import com.yahoo.vespa.serviceview.bindings.HealthClient; -import com.yahoo.vespa.serviceview.bindings.ModelResponse; -import org.junit.After; -import org.junit.Before; -import org.junit.Test; -import org.mockito.Mockito; - -import javax.ws.rs.client.Client; -import java.net.URI; -import java.util.Collections; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.concurrent.Executors; - -import static org.junit.Assert.assertEquals; - -/** - * Functional test for {@link StateRequestHandler}. - * - * @author Steinar Knutsen - * @author bjorncs - */ -public class StateRequestHandlerTest { - - private static final String EXTERNAL_BASE_URI = "http://someserver:8080/serviceview/v1/"; - - private static class TestHandler extends StateRequestHandler { - private static final String BASE_URI = "http://vespa.yahoo.com:8080/state/v1"; - - TestHandler(ConfigserverConfig config) { - super(new Context(Executors.newSingleThreadExecutor(), new MockMetric()), config); - } - - @Override - protected ModelResponse getModelConfig(String tenant, String application, String environment, String region, String instance) { - return ServiceModelTest.syntheticModelResponse(); - } - - @Override - protected HealthClient getHealthClient(Path apiParams, Service s, int requestedPort, Query query, Client client) { - HealthClient healthClient = Mockito.mock(HealthClient.class); - HashMap<Object, Object> dummyHealthData = new HashMap<>(); - HashMap<String, String> dummyLink = new HashMap<>(); - dummyLink.put("url", BASE_URI); - dummyHealthData.put("resources", Collections.singletonList(dummyLink)); - Mockito.when(healthClient.getHealthInfo()).thenReturn(dummyHealthData); - return healthClient; - } - } - - private StateRequestHandler testHandler; - private ServiceModel correspondingModel; - - @Before - public void setUp() throws Exception { - testHandler = new TestHandler(new ConfigserverConfig(new ConfigserverConfig.Builder())); - correspondingModel = new ServiceModel(ServiceModelTest.syntheticModelResponse()); - } - - @After - public void tearDown() { - testHandler = null; - correspondingModel = null; - } - - @Test - public final void test() { - Service s = correspondingModel.resolve("vespa.yahoo.com", 8080, null); - String api = "/state/v1"; - HashMap<?, ?> boom = testHandler.singleService(HttpURL.from(URI.create("http://someserver:8080")), "default", "default", "default", "default", "default", s.getIdentifier(8080), Path.parse(api), Query.empty().add("foo", "bar")); - assertEquals(EXTERNAL_BASE_URI + "tenant/default/application/default/environment/default/region/default/instance/default/service/" + s.getIdentifier(8080) + api, - ((Map<?, ?>) ((List<?>) boom.get("resources")).get(0)).get("url")); - } - - @Test - public final void testLinkEquality() { - ApplicationView explicitParameters = testHandler.getUserInfo(HttpURL.from(URI.create("http://someserver:8080")), "default", "default", "default", "default", "default"); - assertEquals(EXTERNAL_BASE_URI + "tenant/default/application/default/environment/default/region/default/instance" + - "/default/service/container-clustercontroller-2ul67p8psr451t3w8kdd0qwgg/state/v1/", - explicitParameters.clusters.get(0).services.get(0).url); - } - -} diff --git a/container-core/src/main/java/com/yahoo/container/handler/ThreadPoolProvider.java b/container-core/src/main/java/com/yahoo/container/handler/ThreadPoolProvider.java index bc3c35cb78e..1818a3d97b4 100644 --- a/container-core/src/main/java/com/yahoo/container/handler/ThreadPoolProvider.java +++ b/container-core/src/main/java/com/yahoo/container/handler/ThreadPoolProvider.java @@ -6,15 +6,16 @@ import com.yahoo.component.AbstractComponent; import com.yahoo.container.di.componentgraph.Provider; import com.yahoo.container.handler.threadpool.ContainerThreadPool; import com.yahoo.container.handler.threadpool.ContainerThreadpoolConfig; -import com.yahoo.container.handler.threadpool.DefaultContainerThreadpool; +import com.yahoo.container.handler.threadpool.ContainerThreadpoolImpl; import com.yahoo.container.protect.ProcessTerminator; import com.yahoo.jdisc.Metric; import java.util.concurrent.Executor; /** - * A configurable thread pool provider. This provides the worker threads used for normal request processing. - * Request an Executor injected in your component constructor if you want to use it. + * A configurable thread pool provider for the jdisc default threadpool. + * This provides the worker threads used for normal request processing. + * Request an {@link Executor} injected in your component constructor if you want to use it. * * @author Steinar Knutsen * @author baldersheim @@ -26,11 +27,11 @@ public class ThreadPoolProvider extends AbstractComponent implements Provider<Ex @Inject public ThreadPoolProvider(ThreadpoolConfig config, Metric metric) { - this.threadpool = new DefaultContainerThreadpool(translateConfig(config), metric); + this.threadpool = new ContainerThreadpoolImpl(translateConfig(config), metric); } public ThreadPoolProvider(ThreadpoolConfig config, Metric metric, ProcessTerminator processTerminator) { - this.threadpool = new DefaultContainerThreadpool(translateConfig(config), metric, processTerminator); + this.threadpool = new ContainerThreadpoolImpl(translateConfig(config), metric, processTerminator); } /** diff --git a/container-core/src/main/java/com/yahoo/container/handler/threadpool/DefaultContainerThreadpool.java b/container-core/src/main/java/com/yahoo/container/handler/threadpool/ContainerThreadpoolImpl.java index 638336e51d8..73845c13fe8 100644 --- a/container-core/src/main/java/com/yahoo/container/handler/threadpool/DefaultContainerThreadpool.java +++ b/container-core/src/main/java/com/yahoo/container/handler/threadpool/ContainerThreadpoolImpl.java @@ -22,25 +22,25 @@ import java.util.logging.Logger; * @author bratseth * @author bjorncs */ -public class DefaultContainerThreadpool extends AbstractComponent implements AutoCloseable, ContainerThreadPool { +public class ContainerThreadpoolImpl extends AbstractComponent implements AutoCloseable, ContainerThreadPool { - private static final Logger log = Logger.getLogger(DefaultContainerThreadpool.class.getName()); + private static final Logger log = Logger.getLogger(ContainerThreadpoolImpl.class.getName()); private static final int MIN_QUEUE_SIZE = 650; private static final int MIN_THREADS_WHEN_SCALE_FACTOR = 8; private final ExecutorServiceWrapper threadpool; @Inject - public DefaultContainerThreadpool(ContainerThreadpoolConfig config, Metric metric) { + public ContainerThreadpoolImpl(ContainerThreadpoolConfig config, Metric metric) { this(config, metric, new ProcessTerminator()); } - public DefaultContainerThreadpool(ContainerThreadpoolConfig config, Metric metric, ProcessTerminator processTerminator) { + public ContainerThreadpoolImpl(ContainerThreadpoolConfig config, Metric metric, ProcessTerminator processTerminator) { this(config, metric, processTerminator, Runtime.getRuntime().availableProcessors()); } - DefaultContainerThreadpool(ContainerThreadpoolConfig config, Metric metric, ProcessTerminator processTerminator, - int cpus) { + ContainerThreadpoolImpl(ContainerThreadpoolConfig config, Metric metric, ProcessTerminator processTerminator, + int cpus) { String name = config.name(); int maxThreads = maxThreads(config, cpus); int minThreads = minThreads(config, maxThreads, cpus); diff --git a/container-core/src/main/java/com/yahoo/container/handler/threadpool/ExecutorServiceWrapper.java b/container-core/src/main/java/com/yahoo/container/handler/threadpool/ExecutorServiceWrapper.java index 99f49f10526..6dd6b2d122e 100644 --- a/container-core/src/main/java/com/yahoo/container/handler/threadpool/ExecutorServiceWrapper.java +++ b/container-core/src/main/java/com/yahoo/container/handler/threadpool/ExecutorServiceWrapper.java @@ -43,9 +43,6 @@ class ExecutorServiceWrapper extends ForwardingExecutorService { this.queueCapacity = threadPoolIsOnlyQ ? wrapped.getMaximumPoolSize() : maxQueueCapacity; - - metric.reportThreadPoolSize(wrapped.getPoolSize()); - metric.reportActiveThreads(wrapped.getActiveCount()); reportMetrics(); metricReporter = new Thread(this::reportMetricsRegularly); metricReporter.setName(name + "-threadpool-metric-reporter"); @@ -55,6 +52,7 @@ class ExecutorServiceWrapper extends ForwardingExecutorService { private void reportMetrics() { int activeThreads = wrapped.getActiveCount(); metric.reportThreadPoolSize(wrapped.getPoolSize()); + metric.reportMaxAllowedThreadPoolSize(wrapped.getMaximumPoolSize()); metric.reportActiveThreads(activeThreads); int queueSize = threadPoolIsOnlyQ ? activeThreads : wrapped.getQueue().size(); metric.reportWorkQueueSize(queueSize); @@ -81,7 +79,6 @@ class ExecutorServiceWrapper extends ForwardingExecutorService { @Override public void shutdown() { - super.shutdown(); synchronized (closed) { closed.set(true); closed.notify(); @@ -89,6 +86,7 @@ class ExecutorServiceWrapper extends ForwardingExecutorService { try { metricReporter.join(); } catch (InterruptedException e) {} + super.shutdown(); } /** diff --git a/container-core/src/main/java/com/yahoo/container/handler/threadpool/ThreadPoolMetric.java b/container-core/src/main/java/com/yahoo/container/handler/threadpool/ThreadPoolMetric.java index cc70abac59b..1c7a1cc4ebe 100644 --- a/container-core/src/main/java/com/yahoo/container/handler/threadpool/ThreadPoolMetric.java +++ b/container-core/src/main/java/com/yahoo/container/handler/threadpool/ThreadPoolMetric.java @@ -22,9 +22,23 @@ class ThreadPoolMetric { this.defaultContext = metric.createContext(Map.of(THREAD_POOL_NAME_DIMENSION, threadPoolName)); } - void reportRejectRequest() { metric.add("serverRejectedRequests", 1L, defaultContext); } - void reportThreadPoolSize(long size) { metric.set("serverThreadPoolSize", size, defaultContext); } - void reportActiveThreads(long threads) { metric.set("serverActiveThreads", threads, defaultContext); } + void reportRejectRequest() { + metric.add("serverRejectedRequests", 1L, defaultContext); + metric.add("jdisc.thread_pool.rejected_tasks", 1L, defaultContext); + } + + void reportThreadPoolSize(long size) { + metric.set("serverThreadPoolSize", size, defaultContext); + metric.set("jdisc.thread_pool.size", size, defaultContext); + } + + void reportMaxAllowedThreadPoolSize(long size) { metric.set("jdisc.thread_pool.max_allowed_size", size, defaultContext); } + + void reportActiveThreads(long threads) { + metric.set("serverActiveThreads", threads, defaultContext); + metric.set("jdisc.thread_pool.active_threads", threads, defaultContext); + } + void reportWorkQueueCapacity(long capacity) { metric.set("jdisc.thread_pool.work_queue.capacity", capacity, defaultContext); } void reportWorkQueueSize(long size) { metric.set("jdisc.thread_pool.work_queue.size", size, defaultContext); } void reportUnhandledException(Throwable t) { diff --git a/container-core/src/main/java/com/yahoo/jdisc/http/server/jetty/JettyConnectionLogger.java b/container-core/src/main/java/com/yahoo/jdisc/http/server/jetty/JettyConnectionLogger.java index 4e3fd3f29b3..2e2eb257b6a 100644 --- a/container-core/src/main/java/com/yahoo/jdisc/http/server/jetty/JettyConnectionLogger.java +++ b/container-core/src/main/java/com/yahoo/jdisc/http/server/jetty/JettyConnectionLogger.java @@ -311,7 +311,7 @@ class JettyConnectionLogger extends AbstractLifeCycle implements Connection.List this.sslSubjectAlternativeNames = X509CertificateUtils.getSubjectAlternativeNames(peerCertificate).stream() .map(SubjectAlternativeName::getValue) .collect(Collectors.toList()); - this.sslPeerIssuerSubject = peerCertificate.getIssuerDN().getName(); + this.sslPeerIssuerSubject = peerCertificate.getIssuerX500Principal().getName(); this.sslPeerEncodedCertificate = peerCertificate.getEncoded(); } catch (SSLPeerUnverifiedException | CertificateEncodingException e) { // Throw if peer is not authenticated (e.g when client auth is disabled) diff --git a/container-core/src/main/resources/configdefinitions/jdisc.http.jdisc.http.server.def b/container-core/src/main/resources/configdefinitions/jdisc.http.jdisc.http.server.def index 7e1404bbc5e..aed92b1459f 100644 --- a/container-core/src/main/resources/configdefinitions/jdisc.http.jdisc.http.server.def +++ b/container-core/src/main/resources/configdefinitions/jdisc.http.jdisc.http.server.def @@ -37,10 +37,12 @@ defaultFilters[].localPort int strictFiltering bool default = false # Max number of threads in underlying Jetty pool -maxWorkerThreads int default = 200 +# Defaults to 16+vCPU +maxWorkerThreads int default = -1 # Min number of threads in underlying Jetty pool -minWorkerThreads int default = 8 +# Defaults to 16+vCPU +minWorkerThreads int default = -1 # Stop timeout in seconds. The maximum allowed time to process in-flight requests during server shutdown. Setting it to 0 disable graceful shutdown. stopTimeout double default = 30.0 diff --git a/container-core/src/test/java/com/yahoo/container/handler/threadpool/DefaultContainerThreadPoolTest.java b/container-core/src/test/java/com/yahoo/container/handler/threadpool/ContainerThreadPoolImplTest.java index b56d89cafb3..c3f91000296 100644 --- a/container-core/src/test/java/com/yahoo/container/handler/threadpool/DefaultContainerThreadPoolTest.java +++ b/container-core/src/test/java/com/yahoo/container/handler/threadpool/ContainerThreadPoolImplTest.java @@ -20,7 +20,7 @@ import static org.junit.Assert.fail; * @author Steinar Knutsen * @author bjorncs */ -public class DefaultContainerThreadPoolTest { +public class ContainerThreadPoolImplTest { private static final int CPUS = 16; @@ -28,7 +28,7 @@ public class DefaultContainerThreadPoolTest { public final void testThreadPool() throws InterruptedException { Metric metrics = new MetricMock(); ContainerThreadpoolConfig config = new ContainerThreadpoolConfig(new ContainerThreadpoolConfig.Builder().maxThreads(1)); - ContainerThreadPool threadPool = new DefaultContainerThreadpool(config, metrics); + ContainerThreadPool threadPool = new ContainerThreadpoolImpl(config, metrics); Executor exec = threadPool.executor(); Tuple2<Receiver.MessageState, Boolean> reply; FlipIt command = new FlipIt(); @@ -66,7 +66,7 @@ public class DefaultContainerThreadPoolTest { .maxThreads(maxThreads) .minThreads(maxThreads) .queueSize(queueSize)); - ContainerThreadPool threadPool = new DefaultContainerThreadpool( + ContainerThreadPool threadPool = new ContainerThreadpoolImpl( config, metric, new MockProcessTerminator(), CPUS); ExecutorServiceWrapper wrapper = (ExecutorServiceWrapper) threadPool.executor(); WorkerCompletionTimingThreadPoolExecutor executor = (WorkerCompletionTimingThreadPoolExecutor)wrapper.delegate(); @@ -79,8 +79,9 @@ public class DefaultContainerThreadPoolTest { ThreadPoolExecutor executor = createPool(metrics, 3, 1200); assertEquals(3, executor.getMaximumPoolSize()); assertEquals(1200, executor.getQueue().remainingCapacity()); - assertEquals(4, metrics.innvocations().size()); + assertEquals(7, metrics.innvocations().size()); assertEquals(3L, metrics.innvocations().get("serverThreadPoolSize").val); + assertEquals(3L, metrics.innvocations().get("jdisc.thread_pool.max_allowed_size").val); assertEquals(0L, metrics.innvocations().get("serverActiveThreads").val); assertEquals(1200L, metrics.innvocations().get("jdisc.thread_pool.work_queue.capacity").val); assertEquals(0L, metrics.innvocations().get("jdisc.thread_pool.work_queue.size").val); @@ -91,8 +92,9 @@ public class DefaultContainerThreadPoolTest { ThreadPoolExecutor executor = createPool(metrics, 0, 0); assertEquals(CPUS*4, executor.getMaximumPoolSize()); assertEquals(0, executor.getQueue().remainingCapacity()); - assertEquals(4, metrics.innvocations().size()); + assertEquals(7, metrics.innvocations().size()); assertEquals(64L, metrics.innvocations().get("serverThreadPoolSize").val); + assertEquals(64L, metrics.innvocations().get("jdisc.thread_pool.max_allowed_size").val); assertEquals(0L, metrics.innvocations().get("serverActiveThreads").val); assertEquals(64L, metrics.innvocations().get("jdisc.thread_pool.work_queue.capacity").val); assertEquals(0L, metrics.innvocations().get("jdisc.thread_pool.work_queue.size").val); @@ -128,7 +130,7 @@ public class DefaultContainerThreadPoolTest { .maxThreadExecutionTimeSeconds(1)); MockProcessTerminator terminator = new MockProcessTerminator(); Metric metrics = new MetricMock(); - ContainerThreadPool threadPool = new DefaultContainerThreadpool(config, metrics, terminator); + ContainerThreadPool threadPool = new ContainerThreadpoolImpl(config, metrics, terminator); // No dying when threads hang shorter than max thread execution time threadPool.executor().execute(new Hang(500)); diff --git a/container-search/src/main/java/com/yahoo/search/grouping/vespa/ResultBuilder.java b/container-search/src/main/java/com/yahoo/search/grouping/vespa/ResultBuilder.java index 695707bfc50..7f006b098cd 100644 --- a/container-search/src/main/java/com/yahoo/search/grouping/vespa/ResultBuilder.java +++ b/container-search/src/main/java/com/yahoo/search/grouping/vespa/ResultBuilder.java @@ -269,7 +269,7 @@ class ResultBuilder { } private long correctExpressionCountEstimate(long count, int tag) { - int actualGroupCount = group.getChildren().size(); + int actualGroupCount = group.getNumChildren(); // Use actual group count if estimate differ. If max is present, only use actual group count if less than max. // NOTE: If the actual group count is 0, estimate is also 0. if (actualGroupCount > 0 && count != actualGroupCount) { @@ -325,10 +325,11 @@ class ResultBuilder { void addGroup(com.yahoo.searchlib.aggregation.Group execGroup) { GroupBuilder groupBuilder = getOrCreateGroup(execGroup); - if (!execGroup.getChildren().isEmpty()) { - boolean ranked = execGroup.getChildren().get(0).isRankedByRelevance(); + if (execGroup.getNumChildren() > 0) { execGroup.sortChildrenByRank(); - for (com.yahoo.searchlib.aggregation.Group childGroup : execGroup.getChildren()) { + List<com.yahoo.searchlib.aggregation.Group> children = execGroup.getChildren(); + boolean ranked = children.get(0).isRankedByRelevance(); + for (com.yahoo.searchlib.aggregation.Group childGroup : children) { GroupListBuilder childList = groupBuilder.getOrCreateChildList(childGroup.getTag(), ranked); childList.addGroup(childGroup); } diff --git a/controller-api/pom.xml b/controller-api/pom.xml index 5b81b793534..dc2ec981e66 100644 --- a/controller-api/pom.xml +++ b/controller-api/pom.xml @@ -34,13 +34,6 @@ <dependency> <groupId>com.yahoo.vespa</groupId> - <artifactId>serviceview</artifactId> - <scope>provided</scope> - <version>${project.version}</version> - </dependency> - - <dependency> - <groupId>com.yahoo.vespa</groupId> <artifactId>config-provisioning</artifactId> <scope>provided</scope> <version>${project.version}</version> diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/application/v4/EnvironmentResource.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/application/v4/EnvironmentResource.java index 389cda605af..7953fbc4055 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/application/v4/EnvironmentResource.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/application/v4/EnvironmentResource.java @@ -70,9 +70,6 @@ public interface EnvironmentResource { @PathParam("instanceId") InstanceId instanceId, @QueryParam("timeout") long timeoutInSeconds); - @Path("{environmentId}/region/{regionId}/instance/{instanceId}/service") - ServiceViewResource service(); - @PUT @Path("{environmentId}/region/{regionId}/instance/{instanceId}/global-rotation/override") String setRotationOut(@PathParam("tenantId") TenantId tenantId, diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/application/v4/ServiceViewResource.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/application/v4/ServiceViewResource.java deleted file mode 100644 index 1d2860f5dd3..00000000000 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/application/v4/ServiceViewResource.java +++ /dev/null @@ -1,32 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.controller.api.application.v4; - -import com.yahoo.vespa.serviceview.bindings.ApplicationView; - -import javax.ws.rs.GET; -import javax.ws.rs.Path; -import javax.ws.rs.PathParam; -import javax.ws.rs.Produces; -import javax.ws.rs.core.MediaType; -import java.util.HashMap; - -/** - * @author Stian Kristoffersen - */ -@Path("") -@Produces(MediaType.APPLICATION_JSON) -public interface ServiceViewResource { - - @GET - @Path("") - @Produces(MediaType.APPLICATION_JSON) - ApplicationView getUserInfo(); - - @GET - @Path("{serviceIdentifier}/{apiParams: .*}") - @Produces(MediaType.APPLICATION_JSON) - @SuppressWarnings("rawtypes") - HashMap singleService(@PathParam("serviceIdentifier") String identifier, - @PathParam("apiParams") String apiParams); - -} diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/ConfigServer.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/ConfigServer.java index 0b23f1c2f3f..97128d4c980 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/ConfigServer.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/ConfigServer.java @@ -18,7 +18,6 @@ import com.yahoo.vespa.hosted.controller.api.integration.deployment.TestReport; import com.yahoo.vespa.hosted.controller.api.integration.deployment.TesterCloud; import com.yahoo.vespa.hosted.controller.api.integration.noderepository.RestartFilter; import com.yahoo.vespa.hosted.controller.api.integration.secrets.TenantSecretStore; -import com.yahoo.vespa.serviceview.bindings.ApplicationView; import java.io.InputStream; import java.net.URI; @@ -53,10 +52,6 @@ public interface ConfigServer { boolean isSuspended(DeploymentId deployment); - ApplicationView getApplicationView(String tenantName, String applicationName, String instanceName, String environment, String region); - - Map<?,?> getServiceApiResponse(DeploymentId deployment, String serviceName, Path restPath); - /** Returns a proxied response from a given path running on a given service and node */ ProxyResponse getServiceNodePage(DeploymentId deployment, String serviceName, DomainName node, Path subPath, Query query); diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/Node.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/Node.java index c5d3fd1374f..bb97349e5dd 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/Node.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/Node.java @@ -61,6 +61,7 @@ public class Node { private final boolean wantToRetire; private final boolean wantToDeprovision; private final boolean wantToRebuild; + private final boolean down; private final Optional<TenantName> reservedTo; private final Optional<ApplicationId> exclusiveTo; private final Map<String, String> reports; @@ -79,7 +80,7 @@ public class Node { long restartGeneration, long wantedRestartGeneration, long rebootGeneration, long wantedRebootGeneration, int cost, int failCount, Optional<String> flavor, String clusterId, ClusterType clusterType, String group, boolean retired, boolean wantToRetire, boolean wantToDeprovision, - boolean wantToRebuild, Optional<TenantName> reservedTo, Optional<ApplicationId> exclusiveTo, + boolean wantToRebuild, boolean down, Optional<TenantName> reservedTo, Optional<ApplicationId> exclusiveTo, DockerImage wantedDockerImage, DockerImage currentDockerImage, Map<String, String> reports, List<Event> history, Set<String> ipAddresses, Set<String> additionalIpAddresses, Set<String> additionalHostnames, Optional<String> switchHostname, @@ -117,6 +118,7 @@ public class Node { this.wantedDockerImage = Objects.requireNonNull(wantedDockerImage, "wantedDockerImage must be non-null"); this.currentDockerImage = Objects.requireNonNull(currentDockerImage, "currentDockerImage must be non-null"); this.wantToRebuild = wantToRebuild; + this.down = down; this.reports = Map.copyOf(Objects.requireNonNull(reports, "reports must be non-null")); this.history = List.copyOf(Objects.requireNonNull(history, "history must be non-null")); this.ipAddresses = Set.copyOf(Objects.requireNonNull(ipAddresses, "ipAddresses must be non-null")); @@ -280,6 +282,11 @@ public class Node { return wantToRebuild; } + /** Whether this node is currently down */ + public boolean down() { + return down; + } + /** The tenant this has been reserved to, if any */ public Optional<TenantName> reservedTo() { return reservedTo; } @@ -467,6 +474,7 @@ public class Node { private boolean wantToRetire = false; private boolean wantToDeprovision = false; private boolean wantToRebuild = false; + private boolean down = false; private Optional<TenantName> reservedTo = Optional.empty(); private Optional<ApplicationId> exclusiveTo = Optional.empty(); private Map<String, String> reports = Map.of(); @@ -512,6 +520,7 @@ public class Node { this.wantToRetire = node.wantToRetire; this.wantToDeprovision = node.wantToDeprovision; this.wantToRebuild = node.wantToRebuild; + this.down = node.down; this.reservedTo = node.reservedTo; this.exclusiveTo = node.exclusiveTo; this.reports = node.reports; @@ -687,6 +696,11 @@ public class Node { return this; } + public Builder down(boolean down) { + this.down = down; + return this; + } + public Builder reservedTo(TenantName tenant) { this.reservedTo = Optional.of(tenant); return this; @@ -742,7 +756,7 @@ public class Node { currentOsVersion, wantedOsVersion, currentFirmwareCheck, wantedFirmwareCheck, serviceState, suspendedSince, restartGeneration, wantedRestartGeneration, rebootGeneration, wantedRebootGeneration, cost, failCount, flavor, clusterId, clusterType, group, retired, - wantToRetire, wantToDeprovision, wantToRebuild, reservedTo, exclusiveTo, wantedDockerImage, + wantToRetire, wantToDeprovision, wantToRebuild, down, reservedTo, exclusiveTo, wantedDockerImage, currentDockerImage, reports, history, ipAddresses, additionalIpAddresses, additionalHostnames, switchHostname, modelName, environment); } diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/deployment/JobId.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/deployment/JobId.java index 846f3784657..0ef9807270e 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/deployment/JobId.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/deployment/JobId.java @@ -29,7 +29,7 @@ public class JobId { if (o == null || getClass() != o.getClass()) return false; JobId jobId = (JobId) o; return application.equals(jobId.application) && - type == jobId.type; + type.equals(jobId.type); } @Override diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/deployment/JobType.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/deployment/JobType.java index e9231f80091..4505fe3ceb5 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/deployment/JobType.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/deployment/JobType.java @@ -41,140 +41,91 @@ public final class JobType implements Comparable<JobType> { PublicCd, ZoneId.from("staging", "aws-us-east-1c"), Public , ZoneId.from("staging", "aws-us-east-1c"))); - public static final JobType productionUsEast3 = of("production-us-east-3", - Map.of(main, ZoneId.from("prod" , "us-east-3"))); + public static final JobType productionUsEast3 = prod("us-east-3"); - public static final JobType testUsEast3 = of("test-us-east-3", - Map.of(main, ZoneId.from("prod" , "us-east-3")), true); + public static final JobType testUsEast3 = test("us-east-3"); - public static final JobType productionUsWest1 = of("production-us-west-1", - Map.of(main, ZoneId.from("prod" , "us-west-1"))); + public static final JobType productionUsWest1 = prod("us-west-1"); - public static final JobType testUsWest1 = of("test-us-west-1", - Map.of(main, ZoneId.from("prod" , "us-west-1")), true); + public static final JobType testUsWest1 = test("us-west-1"); - public static final JobType productionUsCentral1 = of("production-us-central-1", - Map.of(main, ZoneId.from("prod" , "us-central-1"))); + public static final JobType productionUsCentral1 = prod("us-central-1"); - public static final JobType testUsCentral1 = of("test-us-central-1", - Map.of(main, ZoneId.from("prod" , "us-central-1")), true); + public static final JobType testUsCentral1 = test("us-central-1"); - public static final JobType productionApNortheast1 = of("production-ap-northeast-1", - Map.of(main, ZoneId.from("prod" , "ap-northeast-1"))); + public static final JobType productionApNortheast1 = prod("ap-northeast-1"); - public static final JobType testApNortheast1 = of("test-ap-northeast-1", - Map.of(main, ZoneId.from("prod" , "ap-northeast-1")), true); + public static final JobType testApNortheast1 = test("ap-northeast-1"); - public static final JobType productionApNortheast2 = of("production-ap-northeast-2", - Map.of(main, ZoneId.from("prod" , "ap-northeast-2"))); + public static final JobType productionApNortheast2 = prod("ap-northeast-2"); - public static final JobType testApNortheast2 = of("test-ap-northeast-2", - Map.of(main, ZoneId.from("prod" , "ap-northeast-2")), true); + public static final JobType testApNortheast2 = test("ap-northeast-2"); - public static final JobType productionApSoutheast1 = of("production-ap-southeast-1", - Map.of(main, ZoneId.from("prod" , "ap-southeast-1"))); + public static final JobType productionApSoutheast1 = prod("ap-southeast-1"); - public static final JobType testApSoutheast1 = of("test-ap-southeast-1", - Map.of(main, ZoneId.from("prod" , "ap-southeast-1")), true); + public static final JobType testApSoutheast1 = test("ap-southeast-1"); - public static final JobType productionEuWest1 = of("production-eu-west-1", - Map.of(main, ZoneId.from("prod" , "eu-west-1"))); + public static final JobType productionEuWest1 = prod("eu-west-1"); - public static final JobType testEuWest1 = of("test-eu-west-1", - Map.of(main, ZoneId.from("prod" , "eu-west-1")), true); + public static final JobType testEuWest1 = test("eu-west-1"); - public static final JobType productionAwsUsEast1a= of("production-aws-us-east-1a", - Map.of(main, ZoneId.from("prod" , "aws-us-east-1a"))); + public static final JobType productionAwsUsEast1a= prod("aws-us-east-1a"); - public static final JobType testAwsUsEast1a = of("test-aws-us-east-1a", - Map.of(main, ZoneId.from("prod" , "aws-us-east-1a")), true); + public static final JobType testAwsUsEast1a = test("aws-us-east-1a"); - public static final JobType productionAwsUsEast1c= of("production-aws-us-east-1c", - Map.of(PublicCd, ZoneId.from("prod", "aws-us-east-1c"), - Public, ZoneId.from("prod", "aws-us-east-1c"))); + public static final JobType productionAwsUsEast1c= prod("aws-us-east-1c"); - public static final JobType testAwsUsEast1c = of("test-aws-us-east-1c", - Map.of(PublicCd, ZoneId.from("prod", "aws-us-east-1c"), - Public, ZoneId.from("prod", "aws-us-east-1c")), true); + public static final JobType testAwsUsEast1c = test("aws-us-east-1c"); - public static final JobType productionAwsApNortheast1a= of("production-aws-ap-northeast-1a", - Map.of(Public, ZoneId.from("prod", "aws-ap-northeast-1a"))); + public static final JobType productionAwsApNortheast1a= prod("aws-ap-northeast-1a"); - public static final JobType testAwsApNortheast1a = of("test-aws-ap-northeast-1a", - Map.of(Public, ZoneId.from("prod", "aws-ap-northeast-1a")), true); + public static final JobType testAwsApNortheast1a = test("aws-ap-northeast-1a"); - public static final JobType productionAwsEuWest1a= of("production-aws-eu-west-1a", - Map.of(Public, ZoneId.from("prod", "aws-eu-west-1a"))); + public static final JobType productionAwsEuWest1a= prod("aws-eu-west-1a"); - public static final JobType testAwsEuWest1a = of("test-aws-eu-west-1a", - Map.of(Public, ZoneId.from("prod", "aws-eu-west-1a")), true); + public static final JobType testAwsEuWest1a = test("aws-eu-west-1a"); - public static final JobType productionAwsUsWest2a= of("production-aws-us-west-2a", - Map.of(main, ZoneId.from("prod", "aws-us-west-2a"), - Public, ZoneId.from("prod", "aws-us-west-2a"))); + public static final JobType productionAwsUsWest2a= prod("aws-us-west-2a"); - public static final JobType testAwsUsWest2a = of("test-aws-us-west-2a", - Map.of(main, ZoneId.from("prod", "aws-us-west-2a"), - Public, ZoneId.from("prod", "aws-us-west-2a")), true); + public static final JobType testAwsUsWest2a = test("aws-us-west-2a"); - public static final JobType productionAwsUsEast1b= of("production-aws-us-east-1b", - Map.of(main, ZoneId.from("prod" , "aws-us-east-1b"))); + public static final JobType productionAwsUsEast1b= prod("aws-us-east-1b"); - public static final JobType testAwsUsEast1b = of("test-aws-us-east-1b", - Map.of(main, ZoneId.from("prod" , "aws-us-east-1b")), true); + public static final JobType testAwsUsEast1b = test("aws-us-east-1b"); - public static final JobType devUsEast1 = of("dev-us-east-1", - Map.of(main, ZoneId.from("dev" , "us-east-1"))); + public static final JobType devUsEast1 = dev("us-east-1"); - public static final JobType devAwsUsEast2a = of("dev-aws-us-east-2a", - Map.of(main, ZoneId.from("dev" , "aws-us-east-2a"))); + public static final JobType devAwsUsEast2a = dev("aws-us-east-2a"); - public static final JobType productionCdAwsUsEast1a = of("production-cd-aws-us-east-1a", - Map.of(cd , ZoneId.from("prod" , "cd-aws-us-east-1a"))); + public static final JobType productionCdAwsUsEast1a = prod("cd-aws-us-east-1a"); - public static final JobType testCdAwsUsEast1a = of("test-cd-aws-us-east-1a", - Map.of(cd , ZoneId.from("prod" , "cd-aws-us-east-1a")), true); + public static final JobType testCdAwsUsEast1a = test("cd-aws-us-east-1a"); - public static final JobType productionCdUsCentral1 = of("production-cd-us-central-1", - Map.of(cd , ZoneId.from("prod" , "cd-us-central-1"))); + public static final JobType productionCdUsCentral1 = prod("cd-us-central-1"); - public static final JobType testCdUsCentral1 = of("test-cd-us-central-1", - Map.of(cd , ZoneId.from("prod" , "cd-us-central-1")), true); + public static final JobType testCdUsCentral1 = test("cd-us-central-1"); - // TODO: Cannot remove production-cd-us-central-2 until we know there are no serialized data in controller referencing it - public static final JobType productionCdUsCentral2 = of("production-cd-us-central-2", - Map.of(cd , ZoneId.from("prod" , "cd-us-central-2"))); + public static final JobType productionCdUsCentral2 = prod("cd-us-central-2"); - public static final JobType testCdUsCentral2 = of("test-cd-us-central-2", - Map.of(cd , ZoneId.from("prod" , "cd-us-central-2")), true); + public static final JobType testCdUsCentral2 = test("cd-us-central-2"); - public static final JobType productionCdUsEast1= of("production-cd-us-east-1", - Map.of(cd , ZoneId.from("prod" , "cd-us-east-1"))); + public static final JobType productionCdUsEast1= prod("cd-us-east-1"); - public static final JobType testCdUsEast1 = of("test-cd-us-east-1", - Map.of(cd , ZoneId.from("prod" , "cd-us-east-1")), true); + public static final JobType testCdUsEast1 = test("cd-us-east-1"); - public static final JobType productionCdUsWest1= of("production-cd-us-west-1", - Map.of(cd , ZoneId.from("prod" , "cd-us-west-1"))); + public static final JobType productionCdUsWest1= prod("cd-us-west-1"); - public static final JobType testCdUsWest1 = of("test-cd-us-west-1", - Map.of(cd , ZoneId.from("prod" , "cd-us-west-1")), true); + public static final JobType testCdUsWest1 = test("cd-us-west-1"); - public static final JobType devCdUsCentral1 = of("dev-cd-us-central-1", - Map.of(cd , ZoneId.from("dev" , "cd-us-central-1"))); + public static final JobType devCdUsCentral1 = dev("cd-us-central-1"); - public static final JobType devCdUsWest1 = of("dev-cd-us-west-1", - Map.of(cd , ZoneId.from("dev" , "cd-us-west-1"))); + public static final JobType devCdUsWest1 = dev("cd-us-west-1"); - public static final JobType devAwsUsEast1c = of("dev-aws-us-east-1c", - Map.of(Public, ZoneId.from("dev", "aws-us-east-1c"), - PublicCd, ZoneId.from("dev", "aws-us-east-1c"))); + public static final JobType devAwsUsEast1c = dev("aws-us-east-1c"); - public static final JobType perfAwsUsEast1c = of("perf-aws-us-east-1c", - Map.of(Public, ZoneId.from("perf", "aws-us-east-1c"))); + public static final JobType perfAwsUsEast1c = perf("aws-us-east-1c"); - public static final JobType perfUsEast3 = of("perf-us-east-3", - Map.of(main, ZoneId.from("perf" , "us-east-3"))); + public static final JobType perfUsEast3 = perf("us-east-3"); private static final JobType[] values = new JobType[] { systemTest, @@ -247,6 +198,11 @@ public final class JobType implements Comparable<JobType> { public String jobName() { return jobName; } + /** Returns the zone for this job in the given system. */ + public ZoneId zone() { + throw new UnsupportedOperationException(); + } + /** Returns the zone for this job in the given system, or throws if this job does not have a zone */ public ZoneId zone(SystemName system) { if ( ! zones.containsKey(system)) @@ -255,10 +211,12 @@ public final class JobType implements Comparable<JobType> { return zones.get(system); } + /** A system test in a test zone, or throws if no test zones are present.. */ public static JobType systemTest(ZoneRegistry zones) { return testIn(test, zones); } + /** A staging test in a staging zone, or throws if no staging zones are present. */ public static JobType stagingTest(ZoneRegistry zones){ return testIn(staging, zones); } @@ -268,18 +226,22 @@ public final class JobType implements Comparable<JobType> { .findFirst().orElseThrow(() -> new IllegalArgumentException("no zones in " + environment + " among " + zones.zones().controllerUpgraded().zones())); } + /** A deployment to the given dev region. */ public static JobType dev(String region) { return deploymentTo(ZoneId.from("dev", region)); } + /** A deployment to the given perf region. */ public static JobType perf(String region) { return deploymentTo(ZoneId.from("perf", region)); } + /** A deployment to the given prod region. */ public static JobType prod(String region) { return deploymentTo(ZoneId.from("prod", region)); } + /** A production test in the given region. */ public static JobType test(String region) { return productionTestOf(ZoneId.from("prod", region)); } @@ -311,20 +273,16 @@ public final class JobType implements Comparable<JobType> { throw new IllegalArgumentException("illegal serialized job type '" + raw + "'"); } - // TODO jonmv: use for serialisation - public String serialized() { - throw new UnsupportedOperationException(); - // return zone.environment().value() + "." + zone.region().value() + (isProductionTest ? ".test"); + public String serialized(SystemName system) { + ZoneId zone = zone(system); + return zone.environment().value() + "." + zone.region().value() + (isProductionTest ? ".test" : ""); } public static List<JobType> allIn(ZoneRegistry zones) { - return Stream.of(values).filter(job -> job.zones.containsKey(zones.system())).collect(Collectors.toUnmodifiableList()); - /* return zones.zones().controllerUpgraded().zones().stream() - .flatMap(zone -> zone.getEnvironment().isProduction() ? Stream.of(of(zone.getId()), ofTest(zone.getId())) - : Stream.of(of(zone.getId()))) + .flatMap(zone -> zone.getEnvironment().isProduction() ? Stream.of(deploymentTo(zone.getId()), productionTestOf(zone.getId())) + : Stream.of(deploymentTo(zone.getId()))) .collect(Collectors.toUnmodifiableList()); - */ } static JobType[] values() { @@ -367,6 +325,21 @@ public final class JobType implements Comparable<JobType> { .orElseThrow(() -> new IllegalArgumentException("Unknown job name '" + jobName + "'")); } + // TODO jonmv: require zones + public static JobType fromJobName(String jobName, ZoneRegistry zones) { + String[] parts = jobName.split("-", 2); + if (parts.length != 2) throw new IllegalArgumentException("job names must be 'system-test', 'staging-test', or environment and region parts, separated by '-', but got: " + jobName); + switch (parts[0]) { + case "system": return systemTest(zones); + case "staging": return stagingTest(zones); + case "production": return prod(parts[1]); + case "test": return test(parts[1]); + case "dev": return dev(parts[1]); + case "perf": return perf(parts[1]); + default: throw new IllegalArgumentException("job names must begin with one of: system, staging, production, test, dev, perf; but got: " + jobName); + } + } + /** Returns the job type for the given zone */ public static Optional<JobType> from(SystemName system, ZoneId zone, boolean isTest) { return Stream.of(values) @@ -412,7 +385,12 @@ public final class JobType implements Comparable<JobType> { @Override public int hashCode() { - return Objects.hash(jobName); + return jobName.hashCode(); + } + + @Override + public String toString() { + return jobName; } } diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/noderepository/NodeRepositoryNode.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/noderepository/NodeRepositoryNode.java index 71f08f7318d..8964e1b5127 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/noderepository/NodeRepositoryNode.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/noderepository/NodeRepositoryNode.java @@ -85,6 +85,8 @@ public class NodeRepositoryNode { private Boolean wantToDeprovision; @JsonProperty("wantToRebuild") private Boolean wantToRebuild; + @JsonProperty("down") + private Boolean down; @JsonProperty("cost") private Integer cost; @JsonProperty("history") @@ -318,6 +320,14 @@ public class NodeRepositoryNode { this.wantToRebuild = wantToRebuild; } + public Boolean getDown() { + return down; + } + + public void setDown(Boolean down) { + this.down = down; + } + public Integer getCost() { return cost; } @@ -465,6 +475,7 @@ public class NodeRepositoryNode { ", wantToRetire=" + wantToRetire + ", wantToDeprovision=" + wantToDeprovision + ", wantToRebuild=" + wantToRebuild + + ", down=" + down + ", cost=" + cost + ", history=" + history + ", orchestratorStatus='" + orchestratorStatus + '\'' + diff --git a/controller-api/src/test/java/com/yahoo/vespa/hosted/controller/api/integration/deployment/JobTypeTest.java b/controller-api/src/test/java/com/yahoo/vespa/hosted/controller/api/integration/deployment/JobTypeTest.java index 990e8911e91..64f4d6150dd 100644 --- a/controller-api/src/test/java/com/yahoo/vespa/hosted/controller/api/integration/deployment/JobTypeTest.java +++ b/controller-api/src/test/java/com/yahoo/vespa/hosted/controller/api/integration/deployment/JobTypeTest.java @@ -30,32 +30,37 @@ public class JobTypeTest { } } - assertEquals(JobType.testUsEast3, JobType.fromJobName("prod.us-east-3.test")); - assertEquals(JobType.devAwsUsEast1c, JobType.fromJobName("dev.aws-us-east-1c")); + assertEquals(JobType.testUsEast3, JobType.ofSerialized("prod.us-east-3.test")); + assertEquals(JobType.devAwsUsEast1c, JobType.ofSerialized("dev.aws-us-east-1c")); - assertFalse(JobType.dev("snøhetta").isTest()); - assertTrue(JobType.dev("snøhetta").isDeployment()); - assertFalse(JobType.dev("snøhetta").isProduction()); + assertEquals(JobType.fromJobName("production-my-zone", null), JobType.prod("my-zone")); + assertEquals(JobType.fromJobName("test-my-zone", null), JobType.test("my-zone")); + assertEquals(JobType.fromJobName("dev-my-zone", null), JobType.dev("my-zone")); + assertEquals(JobType.fromJobName("perf-my-zone", null), JobType.perf("my-zone")); - assertFalse(JobType.perf("snøhetta").isTest()); - assertTrue(JobType.perf("snøhetta").isDeployment()); - assertFalse(JobType.perf("snøhetta").isProduction()); + assertFalse(JobType.dev("snohetta").isTest()); + assertTrue(JobType.dev("snohetta").isDeployment()); + assertFalse(JobType.dev("snohetta").isProduction()); - assertTrue(JobType.deploymentTo(ZoneId.from("test", "snøhetta")).isTest()); - assertTrue(JobType.deploymentTo(ZoneId.from("test", "snøhetta")).isDeployment()); - assertFalse(JobType.deploymentTo(ZoneId.from("test", "snøhetta")).isProduction()); + assertFalse(JobType.perf("snohetta").isTest()); + assertTrue(JobType.perf("snohetta").isDeployment()); + assertFalse(JobType.perf("snohetta").isProduction()); - assertTrue(JobType.deploymentTo(ZoneId.from("staging", "snøhetta")).isTest()); - assertTrue(JobType.deploymentTo(ZoneId.from("staging", "snøhetta")).isDeployment()); - assertFalse(JobType.deploymentTo(ZoneId.from("staging", "snøhetta")).isProduction()); + assertTrue(JobType.deploymentTo(ZoneId.from("test", "snohetta")).isTest()); + assertTrue(JobType.deploymentTo(ZoneId.from("test", "snohetta")).isDeployment()); + assertFalse(JobType.deploymentTo(ZoneId.from("test", "snohetta")).isProduction()); - assertFalse(JobType.prod("snøhetta").isTest()); - assertTrue(JobType.prod("snøhetta").isDeployment()); - assertTrue(JobType.prod("snøhetta").isProduction()); + assertTrue(JobType.deploymentTo(ZoneId.from("staging", "snohetta")).isTest()); + assertTrue(JobType.deploymentTo(ZoneId.from("staging", "snohetta")).isDeployment()); + assertFalse(JobType.deploymentTo(ZoneId.from("staging", "snohetta")).isProduction()); - assertTrue(JobType.test("snøhetta").isTest()); - assertFalse(JobType.test("snøhetta").isDeployment()); - assertTrue(JobType.test("snøhetta").isProduction()); + assertFalse(JobType.prod("snohetta").isTest()); + assertTrue(JobType.prod("snohetta").isDeployment()); + assertTrue(JobType.prod("snohetta").isProduction()); + + assertTrue(JobType.test("snohetta").isTest()); + assertFalse(JobType.test("snohetta").isDeployment()); + assertTrue(JobType.test("snohetta").isProduction()); } } diff --git a/controller-server/pom.xml b/controller-server/pom.xml index 5697d849b0f..5cf53929a98 100644 --- a/controller-server/pom.xml +++ b/controller-server/pom.xml @@ -56,13 +56,6 @@ <dependency> <groupId>com.yahoo.vespa</groupId> - <artifactId>serviceview</artifactId> - <version>${project.version}</version> - <scope>provided</scope> - </dependency> - - <dependency> - <groupId>com.yahoo.vespa</groupId> <artifactId>config-provisioning</artifactId> <version>${project.version}</version> <scope>provided</scope> diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java index 3be5345b377..0f7cbcee4ab 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java @@ -287,7 +287,7 @@ public class ApplicationController { if (oldest == null || version.isBefore(oldest)) oldest = version; - if (run.status() == RunStatus.success) + if (run.hasSucceeded()) return Optional.of(oldest); } // If no successful run was found, ask the node repository in the relevant zone. diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Controller.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Controller.java index dc9e696ce51..48e663e7feb 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Controller.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Controller.java @@ -34,7 +34,6 @@ import com.yahoo.vespa.hosted.controller.versions.OsVersionTarget; import com.yahoo.vespa.hosted.controller.versions.VersionStatus; import com.yahoo.vespa.hosted.controller.versions.VespaVersion; import com.yahoo.vespa.hosted.rotation.config.RotationsConfig; -import com.yahoo.vespa.serviceview.bindings.ApplicationView; import com.yahoo.yolean.concurrent.Sleeper; import java.time.Clock; @@ -170,11 +169,6 @@ public class Controller extends AbstractComponent { public ControllerConfig controllerConfig() { return controllerConfig; } - public ApplicationView getApplicationView(String tenantName, String applicationName, String instanceName, - String environment, String region) { - return serviceRegistry.configServer().getApplicationView(tenantName, applicationName, instanceName, environment, region); - } - /** Replace the current version status by a new one */ public void updateVersionStatus(VersionStatus newStatus) { VersionStatus currentStatus = readVersionStatus(); diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/pkg/ApplicationPackage.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/pkg/ApplicationPackage.java index 258884a4d11..ed794a9d929 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/pkg/ApplicationPackage.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/pkg/ApplicationPackage.java @@ -197,11 +197,11 @@ public class ApplicationPackage { RegionName.defaultName()) .run(); // Populates the zip archive cache with files that would be included. } - catch (RuntimeException e) { + catch (IllegalArgumentException e) { throw e; } catch (Exception e) { - throw new RuntimeException(e); + throw new IllegalArgumentException(e); } } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentStatus.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentStatus.java index 7b53d9c5d99..cb2958745d0 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentStatus.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentStatus.java @@ -321,7 +321,7 @@ public class DeploymentStatus { .type(type).asList().stream() .flatMap(status -> RunList.from(status) .on(versions) - .status(RunStatus.success) + .matching(Run::hasSucceeded) .asList().stream() .map(Run::start)) .min(naturalOrder()); @@ -486,7 +486,7 @@ public class DeploymentStatus { if ( job.type().isProduction() && job.type().isDeployment() && allJobs.successOn(productionJob.versions()).type(testType).isEmpty() && testJobs.keySet().stream() - .noneMatch(test -> test.type() == testType + .noneMatch(test -> test.type().equals(testType) && testJobs.get(test).stream().anyMatch(testJob -> testJob.versions().equals(productionJob.versions())))) { JobId testJob = firstDeclaredOrElseImplicitTest(testType); testJobs.merge(testJob, @@ -860,7 +860,7 @@ public class DeploymentStatus { Optional<Instant> end = Optional.empty(); for (Run run : job.runs().descendingMap().values()) { if (run.versions().targetsMatch(change)) { - if (run.status() == RunStatus.success) end = run.end(); + if (run.hasSucceeded()) end = run.end(); } else if (dependent.equals(job())) // If strict completion, consider only last time this change was deployed. break; @@ -887,7 +887,7 @@ public class DeploymentStatus { Optional<Instant> deployedAt = status.jobSteps().get(prodId).completedAt(change, Optional.of(prodId)); return (dependent.equals(job()) ? job.lastTriggered().filter(run -> deployedAt.map(at -> ! run.start().isBefore(at)).orElse(false)).stream() : job.runs().values().stream()) - .filter(run -> run.status() == RunStatus.success) + .filter(Run::hasSucceeded) .filter(run -> run.versions().targetsMatch(change)) .flatMap(run -> run.end().stream()).findFirst(); } @@ -907,7 +907,7 @@ public class DeploymentStatus { status.systemVersion))) .orElseGet(() -> (change.platform().isEmpty() || change.platform().get().equals(run.versions().targetPlatform())) && (change.revision().isEmpty() || change.revision().get().equals(run.versions().targetRevision())))) - .status(RunStatus.success) + .matching(Run::hasSucceeded) .asList().stream() .map(run -> run.end().get()) .max(naturalOrder()); diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java index ce4bb70c174..28d9439b457 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java @@ -23,8 +23,6 @@ import com.yahoo.security.SignatureAlgorithm; import com.yahoo.security.X509CertificateBuilder; import com.yahoo.security.X509CertificateUtils; import com.yahoo.text.Text; -import com.yahoo.vespa.flags.FetchVector; -import com.yahoo.vespa.flags.Flags; import com.yahoo.vespa.hosted.controller.Application; import com.yahoo.vespa.hosted.controller.Controller; import com.yahoo.vespa.hosted.controller.Instance; @@ -36,7 +34,6 @@ import com.yahoo.vespa.hosted.controller.api.integration.configserver.Node; import com.yahoo.vespa.hosted.controller.api.integration.configserver.NodeFilter; import com.yahoo.vespa.hosted.controller.api.integration.configserver.PrepareResponse; import com.yahoo.vespa.hosted.controller.api.integration.configserver.ServiceConvergence; -import com.yahoo.vespa.hosted.controller.api.integration.deployment.ApplicationVersion; import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType; import com.yahoo.vespa.hosted.controller.api.integration.deployment.RevisionId; import com.yahoo.vespa.hosted.controller.api.integration.deployment.RunId; @@ -90,6 +87,7 @@ import static com.yahoo.vespa.hosted.controller.api.integration.configserver.Nod import static com.yahoo.vespa.hosted.controller.deployment.RunStatus.deploymentFailed; import static com.yahoo.vespa.hosted.controller.deployment.RunStatus.error; import static com.yahoo.vespa.hosted.controller.deployment.RunStatus.installationFailed; +import static com.yahoo.vespa.hosted.controller.deployment.RunStatus.noTests; import static com.yahoo.vespa.hosted.controller.deployment.RunStatus.nodeAllocationFailure; import static com.yahoo.vespa.hosted.controller.deployment.RunStatus.reset; import static com.yahoo.vespa.hosted.controller.deployment.RunStatus.running; @@ -638,6 +636,7 @@ public class InternalStepRunner implements StepRunner { return Optional.of(running); } + @SuppressWarnings("fallthrough") private Optional<RunStatus> endTests(RunId id, boolean isSetup, DualLogger logger) { Optional<Deployment> deployment = deployment(id.application(), id.type()); if (deployment.isEmpty()) { @@ -679,12 +678,14 @@ public class InternalStepRunner implements StepRunner { controller.jobController().updateTestReport(id); return Optional.of(error); case NO_TESTS: - TesterCloud.Suite suite = TesterCloud.Suite.of(id.type(), isSetup); - logger.log(INFO, "No tests were found in the test package, for test suite '" + suite + "'"); - logger.log(INFO, "The test package must either contain basic HTTP tests under 'tests/<suite-name>/', " + - "or a Java test bundle under 'components/' with at least one test with the annotation " + - "for this suite. See docs.vespa.ai/en/testing.html for details."); - return Optional.of(allowNoTests(id.application()) ? running : testFailure); + if ( ! isSetup) { // TODO: consider changing this Later™ + TesterCloud.Suite suite = TesterCloud.Suite.of(id.type(), isSetup); + logger.log(INFO, "No tests were found in the test package, for test suite '" + suite + "'"); + logger.log(INFO, "The test package should either contain basic HTTP tests under 'tests/<suite-name>/', " + + "or a Java test bundle under 'components/' with at least one test with the annotation " + + "for this suite. See docs.vespa.ai/en/testing.html for details."); + return Optional.of(noTests); + } case SUCCESS: logger.log("Tests completed successfully."); controller.jobController().updateTestReport(id); @@ -694,12 +695,6 @@ public class InternalStepRunner implements StepRunner { } } - private boolean allowNoTests(ApplicationId appId) { - return Flags.ALLOW_NO_TESTS.bindTo(controller.flagSource()) - .with(FetchVector.Dimension.TENANT_ID, appId.tenant().value()) - .value(); - } - private Optional<RunStatus> copyVespaLogs(RunId id, DualLogger logger) { if (deployment(id.application(), id.type()).isPresent()) try { @@ -835,6 +830,10 @@ public class InternalStepRunner implements StepRunner { case testFailure: updater.accept("one or more verification tests against the deployment failed. Please review test output in the deployment job log."); return; + case noTests: + controller.notificationsDb().setNotification(source, Notification.Type.deployment, Notification.Level.warning, + "no tests were found for this job type. Please review test output in the deployment job log."); + return; case error: case endpointCertificateTimeout: break; @@ -849,6 +848,7 @@ public class InternalStepRunner implements StepRunner { switch (run.status()) { case running: case aborted: + case noTests: case success: return Optional.empty(); case nodeAllocationFailure: @@ -861,11 +861,11 @@ public class InternalStepRunner implements StepRunner { return Optional.of(mails.testFailure(run.id(), recipients)); case error: case endpointCertificateTimeout: - return Optional.of(mails.systemError(run.id(), recipients)); + break; default: logger.log(WARNING, "Don't know what mail to send for run status '" + run.status() + "'"); - return Optional.of(mails.systemError(run.id(), recipients)); } + return Optional.of(mails.systemError(run.id(), recipients)); } /** Returns the deployment of the real application in the zone of the given job, if it exists. */ diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java index 74730a78e31..b9ce094c1c0 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java @@ -400,7 +400,7 @@ public class JobController { locked(id.application(), id.type(), runs -> { runs.put(run.id(), finishedRun); long last = id.number(); - long successes = runs.values().stream().filter(old -> old.status() == RunStatus.success).count(); + long successes = runs.values().stream().filter(Run::hasSucceeded).count(); var oldEntries = runs.entrySet().iterator(); for (var old = oldEntries.next(); old.getKey().number() <= last - historyLength @@ -409,7 +409,7 @@ public class JobController { // Make sure we keep the last success and the first failing if ( successes == 1 - && old.getValue().status() == RunStatus.success + && old.getValue().hasSucceeded() && ! old.getValue().start().isBefore(controller.clock().instant().minus(maxHistoryAge))) { oldEntries.next(); continue; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobList.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobList.java index 639702128d3..387ea755414 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobList.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobList.java @@ -120,7 +120,7 @@ public class JobList extends AbstractFilteringList<JobStatus, JobList> { /** Returns the jobs with successful runs matching the given versions — targets only for system test, everything present otherwise. */ public JobList successOn(Versions versions) { - return matching(job -> ! RunList.from(job).status(RunStatus.success).on(versions).isEmpty()); + return matching(job -> ! RunList.from(job).matching(Run::hasSucceeded).on(versions).isEmpty()); } // ----------------------------------- JobRun filtering diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobMetrics.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobMetrics.java index 874b1828f5f..e95e515685f 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobMetrics.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobMetrics.java @@ -21,6 +21,7 @@ public class JobMetrics { public static final String deploymentFailure = "deployment.deploymentFailure"; public static final String convergenceFailure = "deployment.convergenceFailure"; public static final String testFailure = "deployment.testFailure"; + public static final String noTests = "deployment.noTests"; public static final String error = "deployment.error"; public static final String abort = "deployment.abort"; public static final String success = "deployment.success"; @@ -56,6 +57,7 @@ public class JobMetrics { case deploymentFailed: return deploymentFailure; case installationFailed: return convergenceFailure; case testFailure: return testFailure; + case noTests: return noTests; case error: return error; case aborted: return abort; case success: return success; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobStatus.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobStatus.java index aad5d510261..45bf508f026 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobStatus.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobStatus.java @@ -60,7 +60,7 @@ public class JobStatus { } public boolean isSuccess() { - return lastStatus().isPresent() && lastStatus().get() == RunStatus.success; + return lastCompleted.map(last -> ! last.hasFailed()).orElse(false); } public boolean isRunning() { @@ -90,18 +90,17 @@ public class JobStatus { static Optional<Run> lastSuccess(NavigableMap<RunId, Run> runs) { return runs.descendingMap().values().stream() - .filter(run -> run.status() == RunStatus.success) + .filter(Run::hasSucceeded) .findFirst(); } static Optional<Run> firstFailing(NavigableMap<RunId, Run> runs) { Run failed = null; - loop: for (Run run : runs.descendingMap().values()) - switch (run.status()) { - case running: continue loop; - case success: break loop; - default: failed = run; - } + for (Run run : runs.descendingMap().values()) { + if ( ! run.hasEnded()) continue; + if ( ! run.hasFailed()) break; + failed = run; + } return Optional.ofNullable(failed); } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/RetriggerEntrySerializer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/RetriggerEntrySerializer.java index 6f456d2e217..063167647d5 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/RetriggerEntrySerializer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/RetriggerEntrySerializer.java @@ -3,6 +3,7 @@ package com.yahoo.vespa.hosted.controller.deployment; import com.yahoo.config.provision.ApplicationId; +import com.yahoo.config.provision.SystemName; import com.yahoo.slime.Cursor; import com.yahoo.slime.Inspector; import com.yahoo.slime.Slime; @@ -23,29 +24,35 @@ public class RetriggerEntrySerializer { private static final String JOB_TYPE_KEY = "jobType"; private static final String MIN_REQUIRED_RUN_ID_KEY = "minimumRunId"; - public static List<RetriggerEntry> fromSlime(Slime slime) { + private final SystemName system; + + public RetriggerEntrySerializer(SystemName system) { + this.system = system; + } + + public List<RetriggerEntry> fromSlime(Slime slime) { return SlimeUtils.entriesStream(slime.get().field("entries")) - .map(RetriggerEntrySerializer::deserializeEntry) + .map(this::deserializeEntry) .collect(Collectors.toList()); } - public static Slime toSlime(List<RetriggerEntry> entryList) { + public Slime toSlime(List<RetriggerEntry> entryList) { Slime slime = new Slime(); Cursor root = slime.setObject(); Cursor entries = root.setArray("entries"); - entryList.forEach(e -> RetriggerEntrySerializer.serializeEntry(entries, e)); + entryList.forEach(e -> serializeEntry(entries, e)); return slime; } - private static void serializeEntry(Cursor array, RetriggerEntry entry) { + private void serializeEntry(Cursor array, RetriggerEntry entry) { Cursor root = array.addObject(); Cursor jobid = root.setObject(JOB_ID_KEY); jobid.setString(APPLICATION_ID_KEY, entry.jobId().application().serializedForm()); - jobid.setString(JOB_TYPE_KEY, entry.jobId().type().jobName()); + jobid.setString(JOB_TYPE_KEY, entry.jobId().type().serialized(system)); root.setLong(MIN_REQUIRED_RUN_ID_KEY, entry.requiredRun()); } - private static RetriggerEntry deserializeEntry(Inspector inspector) { + private RetriggerEntry deserializeEntry(Inspector inspector) { Inspector jobid = inspector.field(JOB_ID_KEY); ApplicationId applicationId = ApplicationId.fromSerializedForm(require(jobid, APPLICATION_ID_KEY).asString()); JobType jobType = JobType.fromJobName(require(jobid, JOB_TYPE_KEY).asString()); @@ -53,11 +60,12 @@ public class RetriggerEntrySerializer { return new RetriggerEntry(new JobId(applicationId, jobType), minRequiredRunId); } - private static Inspector require(Inspector inspector, String fieldName) { + private Inspector require(Inspector inspector, String fieldName) { Inspector field = inspector.field(fieldName); if (!field.valid()) { throw new IllegalStateException("Could not deserialize, field not found in json: " + fieldName); } return field; } + } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/Run.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/Run.java index e73d3f52e1f..03cc6c6ba8d 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/Run.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/Run.java @@ -13,6 +13,7 @@ import java.util.Optional; import java.util.stream.Collectors; import static com.yahoo.vespa.hosted.controller.deployment.RunStatus.aborted; +import static com.yahoo.vespa.hosted.controller.deployment.RunStatus.noTests; import static com.yahoo.vespa.hosted.controller.deployment.RunStatus.running; import static com.yahoo.vespa.hosted.controller.deployment.RunStatus.success; import static com.yahoo.vespa.hosted.controller.deployment.Step.Status.succeeded; @@ -80,8 +81,9 @@ public class Run { EnumMap<Step, StepInfo> steps = new EnumMap<>(this.steps); steps.put(step.get(), stepInfo.with(Step.Status.of(status))); - return new Run(id, steps, versions, isRedeployment, start, end, sleepUntil, this.status == running ? status : this.status, - lastTestRecord, lastVespaLogTimestamp, noNodesDownSince, convergenceSummary, testerCertificate, dryRun, reason); + RunStatus newStatus = hasFailed() || status == running ? this.status : status; + return new Run(id, steps, versions, isRedeployment, start, end, sleepUntil, newStatus, lastTestRecord, + lastVespaLogTimestamp, noNodesDownSince, convergenceSummary, testerCertificate, dryRun, reason); } /** Returns a new Run with a new start time*/ @@ -210,7 +212,7 @@ public class Run { /** Returns whether the run has failed, and should switch to its run-always steps. */ public boolean hasFailed() { - return status != running && status != success; + return status != running && status != success && status != noTests; } /** Returns whether the run has ended, i.e., has become inactive, and can no longer be updated. */ @@ -218,6 +220,8 @@ public class Run { return end.isPresent(); } + public boolean hasSucceeded() { return hasEnded() && ! hasFailed(); } + /** Returns the target, and possibly source, versions for this run. */ public Versions versions() { return versions; @@ -297,7 +301,7 @@ public class Run { return steps.entrySet().stream() .filter(entry -> entry.getValue().status() == unfinished && entry.getKey().prerequisites().stream() - .allMatch(step -> steps.get(step) == null + .allMatch(step -> steps.get(step) == null || steps.get(step).status() == succeeded)) .map(Map.Entry::getKey) .collect(Collectors.toUnmodifiableList()); @@ -310,7 +314,7 @@ public class Run { && entry.getKey().alwaysRun() && entry.getKey().prerequisites().stream() .filter(Step::alwaysRun) - .allMatch(step -> steps.get(step) == null + .allMatch(step -> steps.get(step) == null || steps.get(step).status() != unfinished)) .map(Map.Entry::getKey) .collect(Collectors.toUnmodifiableList()); diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/RunStatus.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/RunStatus.java index 0bb4a30425e..9ca634b19fd 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/RunStatus.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/RunStatus.java @@ -26,6 +26,9 @@ public enum RunStatus { /** The verification tests failed. */ testFailure, + /** No tests, for a test job. */ + noTests, + /** An unexpected error occurred. */ error, diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/Step.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/Step.java index 14d6b35f6ad..82d154dcf03 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/Step.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/Step.java @@ -113,6 +113,7 @@ public enum Step { case success : throw new AssertionError("Unexpected run status '" + status + "'!"); case reset : case aborted : return unfinished; + case noTests : case running : return succeeded; default : return failed; } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/VcmrMaintainer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/VcmrMaintainer.java index 39234973af3..d0b3b9f4c7f 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/VcmrMaintainer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/VcmrMaintainer.java @@ -246,7 +246,7 @@ public class VcmrMaintainer extends ControllerMaintainer { } private boolean hasRetired(Node node, HostAction hostAction) { - return hostAction.getState() == State.RETIRING && + return List.of(State.RETIRING, State.REQUIRES_OPERATOR_ACTION).contains(hostAction.getState()) && node.state() == Node.State.parked; } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializer.java index 4b9df825951..e853dbc0d5a 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializer.java @@ -54,7 +54,6 @@ import java.util.Optional; import java.util.OptionalInt; import java.util.OptionalLong; import java.util.Set; -import java.util.function.Function; /** * Serializes {@link Application}s to/from slime. @@ -245,7 +244,7 @@ public class ApplicationSerializer { revisions.development().forEach((job, devRevisions) -> { Cursor devRevisionsObject = devRevisionsArray.addObject(); devRevisionsObject.setString(instanceNameField, job.application().instance().value()); - devRevisionsObject.setString(jobTypeField, job.type().jobName()); + devRevisionsObject.setString(jobTypeField, job.type().serialized(system)); revisionsToSlime(devRevisions, devRevisionsObject.setArray(versionsField)); }); } @@ -285,7 +284,7 @@ public class ApplicationSerializer { Cursor jobStatusArray = cursor.setArray(jobStatusField); jobPauses.forEach((type, until) -> { Cursor jobPauseObject = jobStatusArray.addObject(); - jobPauseObject.setString(jobTypeField, type.jobName()); + jobPauseObject.setString(jobTypeField, type.serialized(system)); jobPauseObject.setLong(pausedUntilField, until.toEpochMilli()); }); } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/CuratorDb.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/CuratorDb.java index 6190d58e0a1..45b762b1b9c 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/CuratorDb.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/CuratorDb.java @@ -107,7 +107,6 @@ public class CuratorDb { private final ControllerVersionSerializer controllerVersionSerializer = new ControllerVersionSerializer(); private final ConfidenceOverrideSerializer confidenceOverrideSerializer = new ConfidenceOverrideSerializer(); private final TenantSerializer tenantSerializer = new TenantSerializer(); - private final RunSerializer runSerializer = new RunSerializer(); private final OsVersionSerializer osVersionSerializer = new OsVersionSerializer(); private final OsVersionTargetSerializer osVersionTargetSerializer = new OsVersionTargetSerializer(osVersionSerializer); private final OsVersionStatusSerializer osVersionStatusSerializer = new OsVersionStatusSerializer(osVersionSerializer, nodeVersionSerializer); @@ -116,6 +115,9 @@ public class CuratorDb { private final AuditLogSerializer auditLogSerializer = new AuditLogSerializer(); private final NameServiceQueueSerializer nameServiceQueueSerializer = new NameServiceQueueSerializer(); private final ApplicationSerializer applicationSerializer; + private final RunSerializer runSerializer; + private final RetriggerEntrySerializer retriggerEntrySerializer; + private final NotificationsSerializer notificationsSerializer; private final Curator curator; private final Duration tryLockTimeout; @@ -138,6 +140,9 @@ public class CuratorDb { this.tryLockTimeout = tryLockTimeout; this.lockScheme = Flags.CONTROLLER_LOCK_SCHEME.bindTo(flagSource); this.applicationSerializer = new ApplicationSerializer(system); + this.runSerializer = new RunSerializer(system); + this.retriggerEntrySerializer = new RetriggerEntrySerializer(system); + this.notificationsSerializer = new NotificationsSerializer(system); } /** Returns all hostnames configured to be part of this ZooKeeper cluster */ @@ -683,7 +688,7 @@ public class CuratorDb { public List<Notification> readNotifications(TenantName tenantName) { return readSlime(notificationsPath(tenantName)) - .map(slime -> NotificationsSerializer.fromSlime(tenantName, slime)).orElseGet(List::of); + .map(slime -> notificationsSerializer.fromSlime(tenantName, slime)).orElseGet(List::of); } @@ -694,7 +699,7 @@ public class CuratorDb { } public void writeNotifications(TenantName tenantName, List<Notification> notifications) { - curator.set(notificationsPath(tenantName), asJson(NotificationsSerializer.toSlime(notifications))); + curator.set(notificationsPath(tenantName), asJson(notificationsSerializer.toSlime(notifications))); } public void deleteNotifications(TenantName tenantName) { @@ -715,11 +720,11 @@ public class CuratorDb { // -------------- Job Retrigger entries ----------------------------------- public List<RetriggerEntry> readRetriggerEntries() { - return readSlime(deploymentRetriggerPath()).map(RetriggerEntrySerializer::fromSlime).orElseGet(List::of); + return readSlime(deploymentRetriggerPath()).map(retriggerEntrySerializer::fromSlime).orElseGet(List::of); } public void writeRetriggerEntries(List<RetriggerEntry> retriggerEntries) { - curator.set(deploymentRetriggerPath(), asJson(RetriggerEntrySerializer.toSlime(retriggerEntries))); + curator.set(deploymentRetriggerPath(), asJson(retriggerEntrySerializer.toSlime(retriggerEntries))); } // -------------- Paths --------------------------------------------------- diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/NotificationsSerializer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/NotificationsSerializer.java index 10763e1f22c..1d5f6d70ca5 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/NotificationsSerializer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/NotificationsSerializer.java @@ -4,6 +4,7 @@ package com.yahoo.vespa.hosted.controller.persistence; import com.yahoo.config.provision.ApplicationName; import com.yahoo.config.provision.ClusterSpec; import com.yahoo.config.provision.InstanceName; +import com.yahoo.config.provision.SystemName; import com.yahoo.config.provision.TenantName; import com.yahoo.config.provision.zone.ZoneId; import com.yahoo.slime.Cursor; @@ -43,7 +44,13 @@ public class NotificationsSerializer { private static final String jobTypeField = "jobId"; private static final String runNumberField = "runNumber"; - public static Slime toSlime(List<Notification> notifications) { + private final SystemName system; + + NotificationsSerializer(SystemName system) { + this.system = system; + } + + public Slime toSlime(List<Notification> notifications) { Slime slime = new Slime(); Cursor notificationsArray = slime.setObject().setArray(notificationsFieldName); @@ -59,20 +66,20 @@ public class NotificationsSerializer { notification.source().instance().ifPresent(instance -> notificationObject.setString(instanceField, instance.value())); notification.source().zoneId().ifPresent(zoneId -> notificationObject.setString(zoneField, zoneId.value())); notification.source().clusterId().ifPresent(clusterId -> notificationObject.setString(clusterIdField, clusterId.value())); - notification.source().jobType().ifPresent(jobType -> notificationObject.setString(jobTypeField, jobType.jobName())); + notification.source().jobType().ifPresent(jobType -> notificationObject.setString(jobTypeField, jobType.serialized(system))); notification.source().runNumber().ifPresent(runNumber -> notificationObject.setLong(runNumberField, runNumber)); } return slime; } - public static List<Notification> fromSlime(TenantName tenantName, Slime slime) { + public List<Notification> fromSlime(TenantName tenantName, Slime slime) { return SlimeUtils.entriesStream(slime.get().field(notificationsFieldName)) .map(inspector -> fromInspector(tenantName, inspector)) .collect(Collectors.toUnmodifiableList()); } - private static Notification fromInspector(TenantName tenantName, Inspector inspector) { + private Notification fromInspector(TenantName tenantName, Inspector inspector) { return new Notification( SlimeUtils.instant(inspector.field(atFieldName)), typeFrom(inspector.field(typeField)), @@ -125,4 +132,5 @@ public class NotificationsSerializer { default: throw new IllegalArgumentException("Unknown serialized notification level value '" + field.asString() + "'"); } } + } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/RunSerializer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/RunSerializer.java index 32ba583321c..143aaaeabb8 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/RunSerializer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/RunSerializer.java @@ -3,6 +3,7 @@ package com.yahoo.vespa.hosted.controller.persistence; import com.yahoo.component.Version; import com.yahoo.config.provision.ApplicationId; +import com.yahoo.config.provision.SystemName; import com.yahoo.security.X509CertificateUtils; import com.yahoo.slime.ArrayTraverser; import com.yahoo.slime.Cursor; @@ -10,12 +11,9 @@ import com.yahoo.slime.Inspector; import com.yahoo.slime.ObjectTraverser; import com.yahoo.slime.Slime; import com.yahoo.slime.SlimeUtils; -import com.yahoo.vespa.hosted.controller.Application; -import com.yahoo.vespa.hosted.controller.api.integration.deployment.ApplicationVersion; import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType; import com.yahoo.vespa.hosted.controller.api.integration.deployment.RevisionId; import com.yahoo.vespa.hosted.controller.api.integration.deployment.RunId; -import com.yahoo.vespa.hosted.controller.api.integration.deployment.SourceRevision; import com.yahoo.vespa.hosted.controller.deployment.ConvergenceSummary; import com.yahoo.vespa.hosted.controller.deployment.Run; import com.yahoo.vespa.hosted.controller.deployment.RunStatus; @@ -30,15 +28,14 @@ import java.util.Collections; import java.util.EnumMap; import java.util.NavigableMap; import java.util.Optional; -import java.util.OptionalLong; import java.util.TreeMap; -import java.util.function.Function; import static com.yahoo.vespa.hosted.controller.deployment.RunStatus.aborted; import static com.yahoo.vespa.hosted.controller.deployment.RunStatus.deploymentFailed; import static com.yahoo.vespa.hosted.controller.deployment.RunStatus.endpointCertificateTimeout; import static com.yahoo.vespa.hosted.controller.deployment.RunStatus.error; import static com.yahoo.vespa.hosted.controller.deployment.RunStatus.installationFailed; +import static com.yahoo.vespa.hosted.controller.deployment.RunStatus.noTests; import static com.yahoo.vespa.hosted.controller.deployment.RunStatus.nodeAllocationFailure; import static com.yahoo.vespa.hosted.controller.deployment.RunStatus.reset; import static com.yahoo.vespa.hosted.controller.deployment.RunStatus.running; @@ -101,6 +98,12 @@ class RunSerializer { private static final String isDryRunField = "isDryRun"; private static final String reasonField = "reason"; + private final SystemName system; + + RunSerializer(SystemName system) { + this.system = system; + } + Run runFromSlime(Slime slime) { return runFromSlime(slime.get()); } @@ -209,7 +212,7 @@ class RunSerializer { private void toSlime(Run run, Cursor runObject) { runObject.setString(applicationField, run.id().application().serializedForm()); - runObject.setString(jobTypeField, run.id().type().jobName()); + runObject.setString(jobTypeField, run.id().type().serialized(system)); runObject.setBool(isRedeploymentField, run.isRedeployment()); runObject.setLong(numberField, run.id().number()); runObject.setLong(startField, run.start().toEpochMilli()); @@ -340,6 +343,7 @@ class RunSerializer { case deploymentFailed : return "deploymentFailed"; case installationFailed : return "installationFailed"; case testFailure : return "testFailure"; + case noTests : return "noTests"; case error : return "error"; case success : return "success"; case aborted : return "aborted"; @@ -352,11 +356,11 @@ class RunSerializer { static RunStatus runStatusOf(String status) { switch (status) { case "running" : return running; - case "outOfCapacity" : return nodeAllocationFailure; // TODO: Remove after March 2022 case "nodeAllocationFailure" : return nodeAllocationFailure; case "endpointCertificateTimeout" : return endpointCertificateTimeout; case "deploymentFailed" : return deploymentFailed; case "installationFailed" : return installationFailed; + case "noTests" : return noTests; case "testFailure" : return testFailure; case "error" : return error; case "success" : return success; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java index 8b43de58cbe..98579b661c8 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java @@ -119,7 +119,6 @@ import com.yahoo.vespa.hosted.controller.tenant.TenantContacts; import com.yahoo.vespa.hosted.controller.tenant.TenantInfo; import com.yahoo.vespa.hosted.controller.versions.VersionStatus; import com.yahoo.vespa.hosted.controller.versions.VespaVersion; -import com.yahoo.vespa.serviceview.bindings.ApplicationView; import com.yahoo.yolean.Exceptions; import javax.ws.rs.ForbiddenException; @@ -268,8 +267,6 @@ public class ApplicationApiHandler extends AuditLoggingRequestHandler { if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/environment/{environment}/region/{region}")) return deployment(path.get("tenant"), path.get("application"), path.get("instance"), path.get("environment"), path.get("region"), request); if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/environment/{environment}/region/{region}/reindexing")) return getReindexing(path.get("tenant"), path.get("application"), path.get("instance"), path.get("environment"), path.get("region"), request); if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/environment/{environment}/region/{region}/suspended")) return suspended(path.get("tenant"), path.get("application"), path.get("instance"), path.get("environment"), path.get("region"), request); - if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/environment/{environment}/region/{region}/service")) return services(path.get("tenant"), path.get("application"), path.get("instance"), path.get("environment"), path.get("region"), request); - if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/environment/{environment}/region/{region}/service/{service}/state/v1/{*}")) return service(path.get("tenant"), path.get("application"), path.get("instance"), path.get("environment"), path.get("region"), path.get("service"), path.getRest(), request); if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/environment/{environment}/region/{region}/service/{service}/{host}/status/{*}")) return status(path.get("tenant"), path.get("application"), path.get("instance"), path.get("environment"), path.get("region"), path.get("service"), path.get("host"), path.getRest(), request); if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/environment/{environment}/region/{region}/service/{service}/{host}/state/v1/{*}")) return stateV1(path.get("tenant"), path.get("application"), path.get("instance"), path.get("environment"), path.get("region"), path.get("service"), path.get("host"), path.getRest(), request); if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/environment/{environment}/region/{region}/orchestrator")) return orchestrator(path.get("tenant"), path.get("application"), path.get("instance"), path.get("environment"), path.get("region")); @@ -285,8 +282,6 @@ public class ApplicationApiHandler extends AuditLoggingRequestHandler { if (path.matches("/application/v4/tenant/{tenant}/application/{application}/environment/{environment}/region/{region}/instance/{instance}")) return deployment(path.get("tenant"), path.get("application"), path.get("instance"), path.get("environment"), path.get("region"), request); if (path.matches("/application/v4/tenant/{tenant}/application/{application}/environment/{environment}/region/{region}/instance/{instance}")) return deployment(path.get("tenant"), path.get("application"), path.get("instance"), path.get("environment"), path.get("region"), request); if (path.matches("/application/v4/tenant/{tenant}/application/{application}/environment/{environment}/region/{region}/instance/{instance}/suspended")) return suspended(path.get("tenant"), path.get("application"), path.get("instance"), path.get("environment"), path.get("region"), request); - if (path.matches("/application/v4/tenant/{tenant}/application/{application}/environment/{environment}/region/{region}/instance/{instance}/service")) return services(path.get("tenant"), path.get("application"), path.get("instance"), path.get("environment"), path.get("region"), request); - if (path.matches("/application/v4/tenant/{tenant}/application/{application}/environment/{environment}/region/{region}/instance/{instance}/service/{service}/state/v1/{*}")) return service(path.get("tenant"), path.get("application"), path.get("instance"), path.get("environment"), path.get("region"), path.get("service"), path.getRest(), request); if (path.matches("/application/v4/tenant/{tenant}/application/{application}/environment/{environment}/region/{region}/instance/{instance}/service/{service}/{host}/status/{*}")) return status(path.get("tenant"), path.get("application"), path.get("instance"), path.get("environment"), path.get("region"), path.get("service"), path.get("host"), path.getRest(), request); if (path.matches("/application/v4/tenant/{tenant}/application/{application}/environment/{environment}/region/{region}/instance/{instance}/nodes")) return nodes(path.get("tenant"), path.get("application"), path.get("instance"), path.get("environment"), path.get("region")); if (path.matches("/application/v4/tenant/{tenant}/application/{application}/environment/{environment}/region/{region}/instance/{instance}/clusters")) return clusters(path.get("tenant"), path.get("application"), path.get("instance"), path.get("environment"), path.get("region")); @@ -713,7 +708,7 @@ public class ApplicationApiHandler extends AuditLoggingRequestHandler { propertyEquals(request, "application", ApplicationName::from, notification.source().application()) && propertyEquals(request, "instance", InstanceName::from, notification.source().instance()) && propertyEquals(request, "zone", ZoneId::from, notification.source().zoneId()) && - propertyEquals(request, "job", JobType::fromJobName, notification.source().jobType()) && + propertyEquals(request, "job", job -> JobType.fromJobName(job, controller.zoneRegistry()), notification.source().jobType()) && propertyEquals(request, "type", Notification.Type::valueOf, Optional.of(notification.type())) && propertyEquals(request, "level", Notification.Level::valueOf, Optional.of(notification.level()))) .forEach(notification -> toSlime(notificationsArray.addObject(), notification, includeTenantFieldInResponse, excludeMessages)); @@ -1107,6 +1102,12 @@ public class ApplicationApiHandler extends AuditLoggingRequestHandler { Slime slime = new Slime(); Cursor nodesArray = slime.setObject().setArray("nodes"); for (Node node : nodes) { + Optional<Instant> downAt = node.history().stream() + .filter(event -> "down".equals(event.name())) + .map(Node.Event::at) + .findFirst(); + boolean isUp = downAt.isEmpty() || node.history().stream() + .anyMatch(event -> "up".equals(event.name()) && event.at().isAfter(downAt.get())); Cursor nodeObject = nodesArray.addObject(); nodeObject.setString("hostname", node.hostname().value()); nodeObject.setString("state", valueOf(node.state())); @@ -1117,7 +1118,8 @@ public class ApplicationApiHandler extends AuditLoggingRequestHandler { toSlime(node.resources(), nodeObject); nodeObject.setString("clusterId", node.clusterId()); nodeObject.setString("clusterType", valueOf(node.clusterType())); - nodeObject.setBool("down", node.history().stream().anyMatch(event -> "down".equals(event.name()))); + nodeObject.setBool("down", !isUp); +// nodeObject.setBool("down", node.down()); // TODO (valerijf): Enable when all configservers expose this nodeObject.setBool("retired", node.retired() || node.wantToRetire()); nodeObject.setBool("restarting", node.wantedRestartGeneration() > node.restartGeneration()); nodeObject.setBool("rebooting", node.wantedRebootGeneration() > node.rebootGeneration()); @@ -1780,17 +1782,6 @@ public class ApplicationApiHandler extends AuditLoggingRequestHandler { return new SlimeJsonResponse(slime); } - private HttpResponse services(String tenantName, String applicationName, String instanceName, String environment, String region, HttpRequest request) { - ApplicationView applicationView = controller.getApplicationView(tenantName, applicationName, instanceName, environment, region); - ZoneId zone = requireZone(environment, region); - ServiceApiResponse response = new ServiceApiResponse(zone, - new ApplicationId.Builder().tenant(tenantName).applicationName(applicationName).instanceName(instanceName).build(), - List.of(controller.zoneRegistry().getConfigServerVipUri(zone)), - request.getUri()); - response.setResponse(applicationView); - return response; - } - private HttpResponse status(String tenantName, String applicationName, String instanceName, String environment, String region, String serviceName, String host, HttpURL.Path restPath, HttpRequest request) { DeploymentId deploymentId = new DeploymentId(ApplicationId.from(tenantName, applicationName, instanceName), requireZone(environment, region)); return controller.serviceRegistry().configServer().getServiceNodePage(deploymentId, @@ -1813,17 +1804,6 @@ public class ApplicationApiHandler extends AuditLoggingRequestHandler { deploymentId, serviceName, DomainName.of(host), HttpURL.Path.parse("/state/v1").append(rest), query); } - private HttpResponse service(String tenantName, String applicationName, String instanceName, String environment, String region, String serviceName, HttpURL.Path restPath, HttpRequest request) { - DeploymentId deploymentId = new DeploymentId(ApplicationId.from(tenantName, applicationName, instanceName), requireZone(environment, region)); - Map<?,?> result = controller.serviceRegistry().configServer().getServiceApiResponse(deploymentId, serviceName, restPath); - ServiceApiResponse response = new ServiceApiResponse(deploymentId.zoneId(), - deploymentId.applicationId(), - List.of(controller.zoneRegistry().getConfigServerVipUri(deploymentId.zoneId())), - request.getUri()); - response.setResponse(result, serviceName, HttpURL.Path.parse("/state/v1").append(restPath)); - return response; - } - private HttpResponse content(String tenantName, String applicationName, String instanceName, String environment, String region, HttpURL.Path restPath, HttpRequest request) { DeploymentId deploymentId = new DeploymentId(ApplicationId.from(tenantName, applicationName, instanceName), requireZone(environment, region)); return controller.serviceRegistry().configServer().getApplicationPackageContent(deploymentId, restPath, request.getUri()); @@ -2715,11 +2695,11 @@ public class ApplicationApiHandler extends AuditLoggingRequestHandler { return ApplicationId.from(path.get("tenant"), path.get("application"), path.get("instance")); } - private static JobType jobTypeFromPath(Path path) { - return JobType.fromJobName(path.get("jobtype")); + private JobType jobTypeFromPath(Path path) { + return JobType.fromJobName(path.get("jobtype"), controller.zoneRegistry()); } - private static RunId runIdFromPath(Path path) { + private RunId runIdFromPath(Path path) { long number = Long.parseLong(path.get("number")); return new RunId(appIdFromPath(path), jobTypeFromPath(path), number); } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ServiceApiResponse.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ServiceApiResponse.java deleted file mode 100644 index ee4ed1b31f9..00000000000 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ServiceApiResponse.java +++ /dev/null @@ -1,197 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.controller.restapi.application; - -import ai.vespa.http.HttpURL; -import ai.vespa.http.HttpURL.Path; -import ai.vespa.http.HttpURL.Query; -import com.yahoo.config.provision.ApplicationId; -import com.yahoo.config.provision.zone.ZoneId; -import com.yahoo.container.jdisc.HttpResponse; -import com.yahoo.slime.Cursor; -import com.yahoo.slime.JsonFormat; -import com.yahoo.slime.Slime; -import com.yahoo.vespa.serviceview.bindings.ApplicationView; -import com.yahoo.vespa.serviceview.bindings.ClusterView; -import com.yahoo.vespa.serviceview.bindings.ServiceView; - -import java.io.IOException; -import java.io.OutputStream; -import java.net.URI; -import java.util.List; -import java.util.Map; -import java.util.regex.Matcher; -import java.util.regex.Pattern; - -/** - * A response containing a service view for an application deployment. - * This does not define the API response but merely proxies the API response provided by Vespa, with URLs - * rewritten to include zone and application information allow proxying through the controller - * - * @author Steinar Knutsen - * @author bratseth - */ -class ServiceApiResponse extends HttpResponse { - - private final ZoneId zone; - private final ApplicationId application; - private final List<URI> configServerURIs; - private final Slime slime; - private final HttpURL requestUri; - - // Only set for one of the setResponse calls - private String serviceName = null; - private Path restPath = null; - - public ServiceApiResponse(ZoneId zone, ApplicationId application, List<URI> configServerURIs, URI requestUri) { - super(200); - this.zone = zone; - this.application = application; - this.configServerURIs = configServerURIs; - this.slime = new Slime(); - this.requestUri = HttpURL.from(requestUri).withQuery(Query.empty()); - } - - public void setResponse(ApplicationView applicationView) { - Cursor clustersArray = slime.setObject().setArray("clusters"); - for (ClusterView clusterView : applicationView.clusters) { - Cursor clusterObject = clustersArray.addObject(); - clusterObject.setString("name", clusterView.name); - clusterObject.setString("type", clusterView.type); - setNullableString("url", rewriteIfUrl(clusterView.url, requestUri), clusterObject); - Cursor servicesArray = clusterObject.setArray("services"); - for (ServiceView serviceView : clusterView.services) { - Cursor serviceObject = servicesArray.addObject(); - setNullableString("url", rewriteIfUrl(serviceView.url, requestUri), serviceObject); - serviceObject.setString("serviceType", serviceView.serviceType); - serviceObject.setString("serviceName", serviceView.serviceName); - serviceObject.setString("configId", serviceView.configId); - serviceObject.setString("host", serviceView.host); - } - } - } - - public void setResponse(Map<?,?> responseData, String serviceName, Path restPath) { - this.serviceName = serviceName; - this.restPath = restPath; - mapToSlime(responseData, slime.setObject()); - } - - @Override - public void render(OutputStream stream) throws IOException { - new JsonFormat(true).encode(stream, slime); - } - - @Override - public String getContentType() { - return "application/json"; - } - - @SuppressWarnings("unchecked") - private void mapToSlime(Map<?,?> data, Cursor object) { - for (Map.Entry<String, Object> entry : ((Map<String, Object>)data).entrySet()) - fieldToSlime(entry.getKey(), entry.getValue(), object); - } - - private void fieldToSlime(String key, Object value, Cursor object) { - if (value instanceof String) { - if (key.equals("url") || key.equals("link")) - value = rewriteIfUrl((String)value, generateLocalLinkPrefix(serviceName, restPath)); - setNullableString(key, (String)value, object); - } - else if (value instanceof Integer) { - object.setLong(key, (int)value); - } - else if (value instanceof Long) { - object.setLong(key, (long)value); - } - else if (value instanceof Float) { - object.setDouble(key, (double)value); - } - else if (value instanceof Double) { - object.setDouble(key, (double)value); - } - else if (value instanceof List) { - listToSlime((List)value, object.setArray(key)); - } - else if (value instanceof Map) { - mapToSlime((Map<?,?>)value, object.setObject(key)); - } - } - - private void listToSlime(List<?> list, Cursor array) { - for (Object entry : list) - entryToSlime(entry, array); - } - - private void entryToSlime(Object entry, Cursor array) { - if (entry instanceof String) - addNullableString(rewriteIfUrl((String)entry, generateLocalLinkPrefix(serviceName, restPath)), array); - else if (entry instanceof Integer) - array.addLong((long)entry); - else if (entry instanceof Long) - array.addLong((long)entry); - else if (entry instanceof Float) - array.addDouble((double)entry); - else if (entry instanceof Double) - array.addDouble((double)entry); - else if (entry instanceof List) - listToSlime((List)entry, array.addArray()); - else if (entry instanceof Map) - mapToSlime((Map)entry, array.addObject()); - } - - private String rewriteIfUrl(String urlOrAnyString, HttpURL requestUri) { - if (urlOrAnyString == null) return null; - - String hostPattern = "(" + - String.join( - "|", configServerURIs.stream() - .map(URI::toString) - .map(s -> s.substring(0, s.length() -1)) - .map(Pattern::quote) - .toArray(String[]::new)) - + ")"; - - String remoteServicePath = "/serviceview/" - + "v1/tenant/" + application.tenant().value() - + "/application/" + application.application().value() - + "/environment/" + zone.environment().value() - + "/region/" + zone.region().value() - + "/instance/" + application.instance() - + "/service/"; - - Pattern remoteServiceResourcePattern = Pattern.compile("^(" + hostPattern + Pattern.quote(remoteServicePath) + ")"); - Matcher matcher = remoteServiceResourcePattern.matcher(urlOrAnyString); - - if (matcher.find()) { - String proxiedPath = urlOrAnyString.substring(matcher.group().length()); - return requestUri.withPath(requestUri.path().append(Path.parse(proxiedPath))).asURI().toString(); - } else { - return urlOrAnyString; // not a service url - } - } - - private HttpURL generateLocalLinkPrefix(String identifier, Path restPath) { - Path proxiedPath = Path.parse(identifier).append(restPath); - if (requestUri.path().tail(proxiedPath.length()).equals(proxiedPath)) { - return requestUri.withPath(requestUri.path().cut(proxiedPath.length())); - } else { - throw new IllegalStateException("Expected the resource " + requestUri.path() + " to end with " + proxiedPath); - } - } - - private void setNullableString(String key, String valueOrNull, Cursor receivingObject) { - if (valueOrNull == null) - receivingObject.setNix(key); - else - receivingObject.setString(key, valueOrNull); - } - - private void addNullableString(String valueOrNull, Cursor receivingArray) { - if (valueOrNull == null) - receivingArray.addNix(); - else - receivingArray.addString(valueOrNull); - } - -} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/billing/BillingApiHandlerV2.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/billing/BillingApiHandlerV2.java index e95a8e74dff..db9c6845183 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/billing/BillingApiHandlerV2.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/billing/BillingApiHandlerV2.java @@ -32,6 +32,7 @@ import java.time.Instant; import java.time.LocalDate; import java.time.ZoneOffset; import java.time.format.DateTimeFormatter; +import java.time.temporal.ChronoUnit; import java.util.Comparator; import java.util.List; import java.util.Optional; @@ -178,7 +179,7 @@ public class BillingApiHandlerV2 extends RestApiRequestHandler<BillingApiHandler private Slime tenantUsage(RestApi.RequestContext requestContext) { var tenantName = TenantName.from(requestContext.pathParameters().getStringOrThrow("tenant")); var tenant = tenants.require(tenantName, CloudTenant.class); - var untilAt = untilParameter(requestContext).orElseGet(clock::instant); + var untilAt = untilParameter(requestContext).orElseGet(this::startOfDayTomorrowUTC); var usage = billing.createUncommittedBill(tenant.name(), untilAt.atZone(ZoneOffset.UTC).toLocalDate()); var slime = new Slime(); @@ -189,7 +190,7 @@ public class BillingApiHandlerV2 extends RestApiRequestHandler<BillingApiHandler // --------- ACCOUNTANT API ---------- private Slime accountant(RestApi.RequestContext requestContext) { - var untilAt = untilParameter(requestContext).orElseGet(clock::instant); + var untilAt = untilParameter(requestContext).orElseGet(this::startOfDayTomorrowUTC); var usagePerTenant = billing.createUncommittedBills(untilAt.atZone(ZoneOffset.UTC).toLocalDate()); var response = new Slime(); @@ -328,6 +329,10 @@ public class BillingApiHandlerV2 extends RestApiRequestHandler<BillingApiHandler return LocalDate.now(clock.withZone(ZoneOffset.UTC)).atStartOfDay(ZoneOffset.UTC).toInstant(); } + private Instant startOfDayTomorrowUTC() { + return startOfDayTodayUTC().plus(1, ChronoUnit.DAYS); + } + private static String getInspectorFieldOrThrow(Inspector inspector, String field) { if (!inspector.field(field).valid()) throw new BadRequestException("Field " + field + " cannot be null"); diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/deployment/BadgeApiHandler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/deployment/BadgeApiHandler.java index eb74f931b2c..3078eb3cb24 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/deployment/BadgeApiHandler.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/deployment/BadgeApiHandler.java @@ -90,7 +90,7 @@ public class BadgeApiHandler extends ThreadedHttpRequestHandler { /** Returns a URI which points to a history badge for the given application and job type. */ private HttpResponse historyBadge(String tenant, String application, String instance, String jobName, String historyLength) { ApplicationId id = ApplicationId.from(tenant, application, instance); - JobType type = JobType.fromJobName(jobName); + JobType type = JobType.fromJobName(jobName, controller.zoneRegistry()); int length = historyLength == null ? 5 : Math.min(32, Math.max(0, Integer.parseInt(historyLength))); return cachedResponse(new Key(id, type, length), controller.clock().instant(), @@ -135,7 +135,7 @@ public class BadgeApiHandler extends ThreadedHttpRequestHandler { if (this == o) return true; if (o == null || getClass() != o.getClass()) return false; Key key = (Key) o; - return historyLength == key.historyLength && id.equals(key.id) && type == key.type; + return historyLength == key.historyLength && id.equals(key.id) && type.equals(key.type); } @Override diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/deployment/Badges.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/deployment/Badges.java index 1fe5ebfa9a9..7b4f2fec853 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/deployment/Badges.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/deployment/Badges.java @@ -15,6 +15,7 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; import java.util.List; +import java.util.Optional; import static java.util.stream.Collectors.toList; @@ -51,14 +52,18 @@ public class Badges { return widthOf(text, 11); } - static String colorOf(Run run, Boolean wasOk) { + static String colorOf(Run run, Optional<RunStatus> previous) { switch (run.status()) { - case running: - return wasOk ? "url(#run-on-success)" : "url(#run-on-failure)"; - case success: - return success; - default: - return failure; + case running: switch (previous.orElse(RunStatus.success)) { + case success: return "url(#run-on-success)"; + case aborted: + case noTests: return "url(#run-on-warning)"; + default: return "url(#run-on-failure)"; + } + case success: return success; + case aborted: + case noTests: return warning; + default: return failure; } } @@ -71,9 +76,10 @@ public class Badges { static final double xPad = 6; static final double logoSize = 16; static final String dark = "#404040"; - static final String success = "#00f244"; + static final String success = "#00f844"; static final String running = "#ab83ff"; static final String failure = "#bf103c"; + static final String warning = "#bd890b"; static void addText(List<String> texts, String text, double x, double width) { addText(texts, text, x, width, 11); @@ -116,13 +122,11 @@ public class Badges { .limit(length) .collect(toList()); - boolean isOk = status.lastCompleted().map(run -> run.status() == RunStatus.success).orElse(true); - text = lastTriggered.id().type().jobName(); textWidth = widthOf(text); dx = xPad + textWidth + xPad; addShade(sections, x, dx); - sections.add(" <rect x='" + (x - 6) + "' rx='3' width='" + (dx + 6) + "' height='20' fill='" + colorOf(lastTriggered, isOk) + "'/>\n"); + sections.add(" <rect x='" + (x - 6) + "' rx='3' width='" + (dx + 6) + "' height='20' fill='" + colorOf(lastTriggered, status.lastStatus()) + "'/>\n"); addShadow(sections, x + dx); addText(texts, text, x + dx / 2, textWidth); x += dx; @@ -130,7 +134,7 @@ public class Badges { dx = xPad * (192.0 / (32 + runs.size())); // Broader sections with shorter history. for (Run run : runs) { addShade(sections, x, dx); - sections.add(" <rect x='" + (x - 6) + "' rx='3' width='" + (dx + 6) + "' height='20' fill='" + colorOf(run, null) + "'/>\n"); + sections.add(" <rect x='" + (x - 6) + "' rx='3' width='" + (dx + 6) + "' height='20' fill='" + colorOf(run, Optional.empty()) + "'/>\n"); addShadow(sections, x + dx); dx *= Math.pow(0.3, 1.0 / (runs.size() + 8)); // Gradually narrowing sections with age. x += dx; @@ -179,7 +183,7 @@ public class Badges { text = nameOf(run.id().type()); textWidth = widthOf(text, isTest ? 9 : 11); dx = xPad + textWidth + (isTest ? 0 : xPad); - boolean wasOk = jobs.get(run.id().job()).flatMap(JobStatus::lastStatus).map(RunStatus.success::equals).orElse(true); + Optional<RunStatus> previous = jobs.get(run.id().job()).flatMap(JobStatus::lastStatus); addText(texts, text, x + (dx - (isTest ? xPad : 0)) / 2, textWidth, isTest ? 9 : 11); @@ -197,10 +201,10 @@ public class Badges { // Add colored section for job ... if (test == null) - sections.add(" <rect x='" + (x - 16) + "' rx='3' width='" + (dx + 16) + "' height='20' fill='" + colorOf(run, wasOk) + "'/>\n"); + sections.add(" <rect x='" + (x - 16) + "' rx='3' width='" + (dx + 16) + "' height='20' fill='" + colorOf(run, previous) + "'/>\n"); // ... with a slant if a test is next. else - sections.add(" <polygon points='" + (x - 6) + " 0 " + (x - 6) + " 20 " + (x + dx - 7) + " 20 " + (x + dx + 1) + " 0' fill='" + colorOf(run, wasOk) + "'/>\n"); + sections.add(" <polygon points='" + (x - 6) + " 0 " + (x - 6) + " 20 " + (x + dx - 7) + " 20 " + (x + dx + 1) + " 0' fill='" + colorOf(run, previous) + "'/>\n"); // Cast a shadow onto the next zone ... if (test == null) @@ -255,6 +259,13 @@ public class Badges { " <animate attributeName='x1' values='-110%;150%;20%;-110%' dur='6s' repeatCount='indefinite' />\n" + " <animate attributeName='x2' values='-10%;250%;120%;-10%' dur='6s' repeatCount='indefinite' />\n" + " </linearGradient>\n" + + // Running color sloshing back and forth on top of the warning color. + " <linearGradient id='run-on-warning' x1='40%' x2='80%' y2='0%'>\n" + + " <stop offset='0' stop-color='" + running + "' />\n" + + " <stop offset='1' stop-color='" + warning + "' />\n" + + " <animate attributeName='x1' values='-110%;150%;20%;-110%' dur='6s' repeatCount='indefinite' />\n" + + " <animate attributeName='x2' values='-10%;250%;120%;-10%' dur='6s' repeatCount='indefinite' />\n" + + " </linearGradient>\n" + // Running color sloshing back and forth on top of the success color. " <linearGradient id='run-on-success' x1='40%' x2='80%' y2='0%'>\n" + " <stop offset='0' stop-color='" + running + "' />\n" + diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/filter/AthenzRoleFilter.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/filter/AthenzRoleFilter.java index a7472ced09c..43037322f22 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/filter/AthenzRoleFilter.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/filter/AthenzRoleFilter.java @@ -21,6 +21,7 @@ import com.yahoo.vespa.hosted.controller.TenantController; import com.yahoo.vespa.hosted.controller.api.integration.athenz.ApplicationAction; import com.yahoo.vespa.hosted.controller.api.integration.athenz.AthenzClientFactory; import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType; +import com.yahoo.vespa.hosted.controller.api.integration.zone.ZoneRegistry; import com.yahoo.vespa.hosted.controller.api.role.Role; import com.yahoo.vespa.hosted.controller.api.role.SecurityContext; import com.yahoo.vespa.hosted.controller.athenz.impl.AthenzFacade; @@ -62,6 +63,7 @@ public class AthenzRoleFilter extends JsonSecurityRequestFilterBase { private final TenantController tenants; private final ExecutorService executor; private final SystemName systemName; + private final ZoneRegistry zones; @Inject public AthenzRoleFilter(AthenzClientFactory athenzClientFactory, Controller controller) { @@ -69,6 +71,7 @@ public class AthenzRoleFilter extends JsonSecurityRequestFilterBase { this.tenants = controller.tenants(); this.executor = Executors.newCachedThreadPool(); this.systemName = controller.system(); + this.zones = controller.zoneRegistry(); } @Override @@ -108,8 +111,7 @@ public class AthenzRoleFilter extends JsonSecurityRequestFilterBase { } else if(path.matches("/application/v4/tenant/{tenant}/application/{application}/environment/{environment}/region/{region}/{*}")) { zone = Optional.of(ZoneId.from(path.get("environment"), path.get("region"))); } else if(path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/deploy/{jobname}")) { - var jobtype= JobType.fromJobName(path.get("jobname")); - zone = Optional.of(jobtype.zone(systemName)); + zone = Optional.of(JobType.fromJobName(path.get("jobname"), zones).zone(systemName)); } else { zone = Optional.empty(); } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/user/UserApiHandler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/user/UserApiHandler.java index 33e6632b8e1..90f71e412f6 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/user/UserApiHandler.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/user/UserApiHandler.java @@ -172,7 +172,6 @@ public class UserApiHandler extends ThreadedHttpRequestHandler { toSlime(root.setObject("user"), user); Cursor tenants = root.setObject("tenants"); - InstanceName userInstance = InstanceName.from(user.nickname()); tenantRolesByTenantName.keySet().stream() .sorted() .forEach(tenant -> { diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentContext.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentContext.java index 899e567d7cc..3765f815e49 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentContext.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentContext.java @@ -28,6 +28,7 @@ import com.yahoo.vespa.hosted.controller.api.integration.deployment.RevisionId; import com.yahoo.vespa.hosted.controller.api.integration.deployment.RunId; import com.yahoo.vespa.hosted.controller.api.integration.deployment.SourceRevision; import com.yahoo.vespa.hosted.controller.api.integration.deployment.TesterCloud; +import com.yahoo.vespa.hosted.controller.api.integration.deployment.TesterCloud.Status; import com.yahoo.vespa.hosted.controller.api.integration.deployment.TesterId; import com.yahoo.vespa.hosted.controller.application.Deployment; import com.yahoo.vespa.hosted.controller.application.EndpointId; @@ -402,8 +403,34 @@ public class DeploymentContext { return runJob(type, instanceId); } + /** Runs the job, failing tests with noTests status, or with regular testFailure. */ + public DeploymentContext failTests(JobType type, boolean noTests) { + if ( ! type.isTest()) throw new IllegalArgumentException(type + " does not run tests"); + var job = new JobId(instanceId, type); + triggerJobs(); + doDeploy(job); + if (job.type().isDeployment()) { + doUpgrade(job); + doConverge(job); + if (job.type().environment().isManuallyDeployed()) + return this; + } + + RunId id = currentRun(job).id(); + ZoneId zone = zone(job); + + assertEquals(unfinished, jobs.run(id).get().stepStatuses().get(Step.endTests)); + tester.cloud().set(noTests ? Status.NO_TESTS : Status.FAILURE); + runner.advance(currentRun(job)); + assertTrue(jobs.run(id).get().hasEnded()); + assertEquals(noTests, jobs.run(id).get().hasSucceeded()); + assertTrue(configServer().nodeRepository().list(zone, NodeFilter.all().applications(TesterId.of(instanceId).id())).isEmpty()); + + return this; + } + /** Pulls the ready job trigger, and then runs the whole of job for the given instance, successfully. */ - public DeploymentContext runJob(JobType type, ApplicationId instance) { + private DeploymentContext runJob(JobType type, ApplicationId instance) { var job = new JobId(instance, type); triggerJobs(); doDeploy(job); @@ -476,7 +503,7 @@ public class DeploymentContext { } Run run = jobs.active().stream() - .filter(r -> r.id().type() == type) + .filter(r -> r.id().type().equals(type)) .findAny() .orElseThrow(() -> new AssertionError(type + " is not among the active: " + jobs.active())); return run.id(); @@ -499,12 +526,12 @@ public class DeploymentContext { public void assertRunning(JobType type) { assertTrue(jobId(type) + " should be among the active: " + jobs.active(), - jobs.active().stream().anyMatch(run -> run.id().application().equals(instanceId) && run.id().type() == type)); + jobs.active().stream().anyMatch(run -> run.id().application().equals(instanceId) && run.id().type().equals(type))); } public void assertNotRunning(JobType type) { assertFalse(jobId(type) + " should not be among the active: " + jobs.active(), - jobs.active().stream().anyMatch(run -> run.id().application().equals(instanceId) && run.id().type() == type)); + jobs.active().stream().anyMatch(run -> run.id().application().equals(instanceId) && run.id().type().equals(type))); } /** Deploys tester and real app, and completes tester and initial staging installation first if needed. */ @@ -522,7 +549,7 @@ public class DeploymentContext { if (job.type().isTest()) doInstallTester(job); - if (job.type() == JobType.stagingTest) { // Do the initial deployment and installation of the real application. + if (job.type().equals(JobType.stagingTest)) { // Do the initial deployment and installation of the real application. assertEquals(unfinished, jobs.run(id).get().stepStatuses().get(Step.installInitialReal)); tester.configServer().nodeRepository().doUpgrade(deployment, Optional.empty(), tester.configServer().application(job.application(), zone).get().version().get()); configServer().convergeServices(id.application(), zone); @@ -555,7 +582,7 @@ public class DeploymentContext { /** Returns the current run for the given job type, and verifies it is still running normally. */ private Run currentRun(JobId job) { Run run = jobs.last(job) - .filter(r -> r.id().type() == job.type()) + .filter(r -> r.id().type().equals(job.type())) .orElseThrow(() -> new AssertionError(job.type() + " is not among the active: " + jobs.active())); assertFalse(run.id() + " should not have failed yet: " + run, run.hasFailed()); assertFalse(run.id() + " should not have ended yet: " + run, run.hasEnded()); diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunnerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunnerTest.java index 98cbf33fb2b..15f729e7a55 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunnerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunnerTest.java @@ -24,6 +24,7 @@ import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType; import com.yahoo.vespa.hosted.controller.api.integration.deployment.RunId; import com.yahoo.vespa.hosted.controller.api.integration.deployment.TestReport; import com.yahoo.vespa.hosted.controller.api.integration.deployment.TesterCloud; +import com.yahoo.vespa.hosted.controller.api.integration.deployment.TesterCloud.Status; import com.yahoo.vespa.hosted.controller.api.integration.stubs.MockMailer; import com.yahoo.vespa.hosted.controller.application.SystemApplication; import com.yahoo.vespa.hosted.controller.application.pkg.ApplicationPackage; @@ -47,6 +48,8 @@ import java.util.List; import java.util.Optional; import java.util.concurrent.Executors; import java.util.concurrent.Future; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.function.Supplier; import static com.yahoo.vespa.hosted.controller.api.integration.LogEntry.Type.error; import static com.yahoo.vespa.hosted.controller.api.integration.LogEntry.Type.info; @@ -55,6 +58,7 @@ import static com.yahoo.vespa.hosted.controller.deployment.DeploymentContext.app import static com.yahoo.vespa.hosted.controller.deployment.DeploymentTester.instanceId; import static com.yahoo.vespa.hosted.controller.deployment.RunStatus.deploymentFailed; import static com.yahoo.vespa.hosted.controller.deployment.RunStatus.installationFailed; +import static com.yahoo.vespa.hosted.controller.deployment.RunStatus.noTests; import static com.yahoo.vespa.hosted.controller.deployment.RunStatus.running; import static com.yahoo.vespa.hosted.controller.deployment.RunStatus.success; import static com.yahoo.vespa.hosted.controller.deployment.Step.Status.failed; @@ -62,6 +66,7 @@ import static com.yahoo.vespa.hosted.controller.deployment.Step.Status.succeeded import static com.yahoo.vespa.hosted.controller.deployment.Step.Status.unfinished; import static java.time.temporal.ChronoUnit.SECONDS; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertNotEquals; import static org.junit.Assert.assertTrue; @@ -123,22 +128,11 @@ public class InternalStepRunnerTest { } @Test - // TODO jonmv: Change to only wait for restarts, and remove triggering of restarts from runner. public void restartsServicesAndWaitsForRestartAndReboot() { RunId id = app.newRun(JobType.productionUsCentral1); ZoneId zone = id.type().zone(system()); HostName host = tester.configServer().hostFor(instanceId, zone); - tester.configServer().setConfigChangeActions(new ConfigChangeActions(List.of(new RestartAction("cluster", - "container", - "search", - List.of(new ServiceInfo("queries", - "search", - "config", - host.value())), - List.of("Restart it!"))), - List.of(), - List.of())); tester.runner().run(); assertEquals(succeeded, tester.jobs().run(id).get().stepStatuses().get(Step.deployReal)); @@ -269,6 +263,27 @@ public class InternalStepRunnerTest { } @Test + public void noTestsThenErrorIsError() { + RunId id = app.startSystemTestTests(); + Run run = tester.jobs().run(id).get(); + run = run.with(noTests, new LockedStep(() -> { }, Step.endTests)); + assertFalse(run.hasFailed()); + run = run.with(RunStatus.error, new LockedStep(() -> { }, Step.deactivateReal)); + assertTrue(run.hasFailed()); + assertEquals(RunStatus.error, run.status()); + } + + @Test + public void noTestsThenSuccessIsNoTests() { + RunId id = app.startSystemTestTests(); + tester.cloud().set(Status.NO_TESTS); + tester.runner().run(); + assertEquals(succeeded, tester.jobs().run(id).get().stepStatuses().get(Step.endTests)); + Run run = tester.jobs().run(id).get(); + assertEquals(noTests, run.status()); + } + + @Test public void testsFailIfTesterRestarts() { RunId id = app.startSystemTestTests(); tester.cloud().set(TesterCloud.Status.NOT_STARTED); diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ConfigServerMock.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ConfigServerMock.java index a9937fad01e..1ed84659f58 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ConfigServerMock.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ConfigServerMock.java @@ -45,9 +45,6 @@ import com.yahoo.vespa.hosted.controller.api.integration.noderepository.RestartF import com.yahoo.vespa.hosted.controller.api.integration.secrets.TenantSecretStore; import com.yahoo.vespa.hosted.controller.application.SystemApplication; import com.yahoo.vespa.hosted.controller.application.pkg.ApplicationPackage; -import com.yahoo.vespa.serviceview.bindings.ApplicationView; -import com.yahoo.vespa.serviceview.bindings.ClusterView; -import com.yahoo.vespa.serviceview.bindings.ServiceView; import java.io.ByteArrayInputStream; import java.io.InputStream; @@ -101,7 +98,6 @@ public class ConfigServerMock extends AbstractComponent implements ConfigServer private Version lastPrepareVersion = null; private Consumer<ApplicationId> prepareException = null; - private ConfigChangeActions configChangeActions = null; private Supplier<String> log = () -> "INFO - All good"; @Inject @@ -109,11 +105,6 @@ public class ConfigServerMock extends AbstractComponent implements ConfigServer bootstrap(zoneRegistry.zones().all().ids(), SystemApplication.notController()); } - /** Sets the ConfigChangeActions that will be returned on next deployment. */ - public void setConfigChangeActions(ConfigChangeActions configChangeActions) { - this.configChangeActions = configChangeActions; - } - /** Assigns a reserved tenant node to the given deployment, with initial versions. */ public void provision(ZoneId zone, ApplicationId application, ClusterSpec.Id clusterId) { var current = new ClusterResources(2, 1, new NodeResources(2, 8, 50, 1, slow, remote)); @@ -431,10 +422,7 @@ public class ConfigServerMock extends AbstractComponent implements ConfigServer PrepareResponse prepareResponse = new PrepareResponse(); prepareResponse.message = "foo"; - prepareResponse.configChangeActions = configChangeActions != null - ? configChangeActions - : new ConfigChangeActions(List.of(), List.of(), List.of()); - setConfigChangeActions(null); + prepareResponse.configChangeActions = new ConfigChangeActions(List.of(), List.of(), List.of()); prepareResponse.tenant = new TenantId("tenant"); prepareResponse.log = warnings.getOrDefault(id, Collections.emptyList()); return prepareResponse; @@ -487,32 +475,6 @@ public class ConfigServerMock extends AbstractComponent implements ConfigServer removeLoadBalancers(deployment.applicationId(), deployment.zoneId()); } - // Returns a canned example response - @Override - public ApplicationView getApplicationView(String tenantName, String applicationName, String instanceName, - String environment, String region) { - String cfgHostname = Text.format("https://cfg.%s.%s.test.vip:4443", environment, region); - String cfgServiceUrlPrefix = Text.format("%s/serviceview/v1/tenant/%s/application/%s/environment/%s/region/%s/instance/%s/service", - cfgHostname, tenantName, applicationName, - environment, region, instanceName); - ApplicationView applicationView = new ApplicationView(); - ClusterView cluster = new ClusterView(); - cluster.name = "cluster1"; - cluster.type = "content"; - cluster.url = cfgServiceUrlPrefix + "/container-clustercontroller-6s8slgtps7ry8uh6lx21ejjiv/cluster/v2/cluster1"; - ServiceView service = new ServiceView(); - service.configId = "cluster1/storage/0"; - service.host = "host1"; - service.serviceName = "storagenode"; - service.serviceType = "storagenode"; - service.url = cfgServiceUrlPrefix + "/storagenode-awe3slno6mmq2fye191y324jl/state/v1/"; - cluster.services = new ArrayList<>(); - cluster.services.add(service); - applicationView.clusters = new ArrayList<>(); - applicationView.clusters.add(cluster); - return applicationView; - } - @Override public List<ClusterMetrics> getDeploymentMetrics(DeploymentId deployment) { return Collections.unmodifiableList(clusterMetrics.getOrDefault(deployment, List.of())); @@ -523,18 +485,6 @@ public class ConfigServerMock extends AbstractComponent implements ConfigServer return this.protonMetrics; } - // Returns a canned example response - @Override - public Map<?,?> getServiceApiResponse(DeploymentId deployment, String serviceName, Path restPath) { - Map<String,List<?>> root = new HashMap<>(); - List<Map<?,?>> resources = new ArrayList<>(); - Map<String,String> resource = new HashMap<>(); - resource.put("url", "http://localhost:8080/application/v4/tenant/tenant1/application/application1/environment/prod/region/us-west-1/instance/default/service/filedistributorservice-dud1f4w037qdxdrn0ovxfdtgw/state/v1/config"); - resources.add(resource); - root.put("resources", resources); - return root; - } - @Override public ProxyResponse getServiceNodePage(DeploymentId deployment, String serviceName, DomainName node, Path subPath, Query query) { return new ProxyResponse((subPath + " and " + query).getBytes(UTF_8), "text/html", 200); diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/CloudTrialExpirerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/CloudTrialExpirerTest.java index a4221ff3564..7f2799b6f58 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/CloudTrialExpirerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/CloudTrialExpirerTest.java @@ -83,7 +83,7 @@ public class CloudTrialExpirerTest { .withSystem(tester.zoneRegistry().system()) .withId("prod." + regionName) .build(); - tester.zoneRegistry().setZones(zone); + tester.zoneRegistry().setZones(ZoneApiMock.fromId("test.aws-us-east-1c"), ZoneApiMock.fromId("staging.aws-us-east-1c"), zone); var app = tester.createApplication(tenantName, appName, instanceName); var ctx = deploymentTester.newDeploymentContext(tenantName, appName, instanceName); var pkg = new ApplicationPackageBuilder() diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/JobRunnerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/JobRunnerTest.java index 8b155644fb4..610d8d4ca9a 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/JobRunnerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/JobRunnerTest.java @@ -86,7 +86,7 @@ public class JobRunnerTest { public void multiThreadedExecutionFinishes() { DeploymentTester tester = new DeploymentTester(); JobController jobs = tester.controller().jobController(); - StepRunner stepRunner = (step, id) -> id.type() == stagingTest && step.get() == startTests? Optional.of(error) : Optional.of(running); + StepRunner stepRunner = (step, id) -> id.type().equals(stagingTest) && step.get() == startTests? Optional.of(error) : Optional.of(running); Phaser phaser = new Phaser(1); JobRunner runner = new JobRunner(tester.controller(), Duration.ofDays(1), phasedExecutor(phaser), stepRunner); @@ -413,6 +413,7 @@ public class JobRunnerTest { assertEquals(1, metric.getMetric(context::equals, JobMetrics.nodeAllocationFailure).get().intValue()); assertEquals(1, metric.getMetric(context::equals, JobMetrics.endpointCertificateTimeout).get().intValue()); assertEquals(1, metric.getMetric(context::equals, JobMetrics.testFailure).get().intValue()); + assertEquals(1, metric.getMetric(context::equals, JobMetrics.noTests).get().intValue()); } private void start(JobController jobs, ApplicationId id, JobType type) { diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/NotificationsSerializerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/NotificationsSerializerTest.java index cbb595d2a3b..0c8a773a132 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/NotificationsSerializerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/NotificationsSerializerTest.java @@ -16,6 +16,7 @@ import java.io.IOException; import java.time.Instant; import java.util.List; +import static com.yahoo.config.provision.SystemName.main; import static org.junit.Assert.assertEquals; /** @@ -25,6 +26,7 @@ public class NotificationsSerializerTest { @Test public void serialization_test() throws IOException { + NotificationsSerializer serializer = new NotificationsSerializer(main); TenantName tenantName = TenantName.from("tenant1"); List<Notification> notifications = List.of( new Notification(Instant.ofEpochSecond(1234), @@ -38,7 +40,7 @@ public class NotificationsSerializerTest { NotificationSource.from(new RunId(ApplicationId.from(tenantName.value(), "app1", "instance1"), JobType.systemTest, 12)), List.of("Failed to deploy: Node allocation failure"))); - Slime serialized = NotificationsSerializer.toSlime(notifications); + Slime serialized = serializer.toSlime(notifications); assertEquals("{\"notifications\":[" + "{" + "\"at\":1234000," + @@ -53,11 +55,12 @@ public class NotificationsSerializerTest { "\"messages\":[\"Failed to deploy: Node allocation failure\"]," + "\"application\":\"app1\"," + "\"instance\":\"instance1\"," + - "\"jobId\":\"system-test\"," + + "\"jobId\":\"test.us-east-1\"," + "\"runNumber\":12" + "}]}", new String(SlimeUtils.toJsonBytes(serialized))); - List<Notification> deserialized = NotificationsSerializer.fromSlime(tenantName, serialized); + List<Notification> deserialized = serializer.fromSlime(tenantName, serialized); assertEquals(notifications, deserialized); } + }
\ No newline at end of file diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/RunSerializerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/RunSerializerTest.java index c63af87c08c..a3b7932197b 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/RunSerializerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/RunSerializerTest.java @@ -31,6 +31,7 @@ import java.util.Map; import java.util.Optional; import java.util.OptionalLong; +import static com.yahoo.config.provision.SystemName.main; import static com.yahoo.vespa.hosted.controller.deployment.RunStatus.aborted; import static com.yahoo.vespa.hosted.controller.deployment.RunStatus.running; import static com.yahoo.vespa.hosted.controller.deployment.Step.Status.failed; @@ -58,7 +59,7 @@ import static org.junit.Assert.assertTrue; public class RunSerializerTest { - private static final RunSerializer serializer = new RunSerializer(); + private static final RunSerializer serializer = new RunSerializer(main); private static final Path runFile = Paths.get("src/test/java/com/yahoo/vespa/hosted/controller/persistence/testdata/run-status.json"); private static final RunId id = new RunId(ApplicationId.from("tenant", "application", "default"), JobType.productionUsEast3, diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java index cc5af52282d..992ef59d9a5 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java @@ -694,15 +694,6 @@ public class ApplicationApiTest extends ControllerContainerTest { .userIdentity(USER_ID), new File("suspended.json")); - // GET services - tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/environment/prod/region/us-central-1/instance/instance1/service", GET) - .userIdentity(USER_ID), - new File("services.json")); - - // GET service - tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/environment/prod/region/us-central-1/instance/instance1/service/storagenode-awe3slno6mmq2fye191y324jl/state/v1/", GET) - .userIdentity(USER_ID), - new File("service.json")); // GET service/state/v1 tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/instance/instance1/environment/prod/region/us-central-1/service/storagenode/host.com/state/v1/?foo=bar", GET) @@ -1645,32 +1636,6 @@ public class ApplicationApiTest extends ControllerContainerTest { } @Test - public void testServiceView() { - createAthenzDomainWithAdmin(ATHENZ_TENANT_DOMAIN, USER_ID); - String serviceApi="/application/v4/tenant/tenant1/application/application1/environment/prod/region/us-central-1/instance/instance1/service"; - // Not allowed to request apis not listed in feature flag allowed-service-view-apis. e.g /document/v1 - tester.assertResponse(request(serviceApi + "/storagenode-awe3slno6mmq2fye191y324jl/document/v1/", GET) - .userIdentity(USER_ID) - .oAuthCredentials(OKTA_CREDENTIALS), - "{\"error-code\":\"NOT_FOUND\",\"message\":\"Nothing at path '/application/v4/tenant/tenant1/application/application1/environment/prod/region/us-central-1/instance/instance1/service/storagenode-awe3slno6mmq2fye191y324jl/document/v1/'\"}", - 404); - - // Test path traversal - tester.assertResponse(request(serviceApi + "/storagenode-awe3slno6mmq2fye191y324jl/state/v1/../../document/v1/", GET) - .userIdentity(USER_ID) - .oAuthCredentials(OKTA_CREDENTIALS), - "{\"error-code\":\"NOT_FOUND\",\"message\":\"Nothing at path '/application/v4/tenant/tenant1/application/application1/environment/prod/region/us-central-1/instance/instance1/service/storagenode-awe3slno6mmq2fye191y324jl/document/v1/'\"}", - 404); - - // Test urlencoded path traversal - tester.assertResponse(request(serviceApi + "/storagenode-awe3slno6mmq2fye191y324jl/state%2Fv1%2F..%2F..%2Fdocument%2Fv1%2F", GET) - .userIdentity(USER_ID) - .oAuthCredentials(OKTA_CREDENTIALS), - accessDenied, - 403); - } - - @Test public void create_application_on_deploy() { // Setup createAthenzDomainWithAdmin(ATHENZ_TENANT_DOMAIN, USER_ID); diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ServiceApiResponseTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ServiceApiResponseTest.java deleted file mode 100644 index c69cd51e20d..00000000000 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ServiceApiResponseTest.java +++ /dev/null @@ -1,95 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.controller.restapi.application; - -import com.yahoo.config.provision.ApplicationId; -import com.yahoo.config.provision.Environment; -import com.yahoo.config.provision.RegionName; -import com.yahoo.config.provision.zone.ZoneId; -import com.yahoo.io.IOUtils; -import com.yahoo.slime.Slime; -import com.yahoo.slime.SlimeUtils; -import com.yahoo.vespa.serviceview.bindings.ApplicationView; -import com.yahoo.vespa.serviceview.bindings.ClusterView; -import com.yahoo.vespa.serviceview.bindings.ServiceView; -import org.junit.Test; - -import java.io.ByteArrayOutputStream; -import java.io.File; -import java.io.IOException; -import java.net.URI; -import java.net.URISyntaxException; -import java.nio.charset.StandardCharsets; -import java.util.Collections; - -import static org.junit.Assert.assertEquals; - -/** - * @author bratseth - */ -public class ServiceApiResponseTest { - - private final static String responseFiles = "src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/"; - - @Test - public void testServiceViewResponse() throws URISyntaxException, IOException { - ServiceApiResponse response = new ServiceApiResponse(ZoneId.from(Environment.prod, RegionName.from("us-west-1")), - ApplicationId.from("tenant1", "application1", "default"), - Collections.singletonList(new URI("config-server1")), - new URI("http://server1:4080/request/path?foo=bar")); - ApplicationView applicationView = new ApplicationView(); - ClusterView clusterView = new ClusterView(); - clusterView.type = "container"; - clusterView.name = "cluster1"; - clusterView.url = "cluster-url"; - ServiceView serviceView = new ServiceView(); - serviceView.url = null; - serviceView.serviceType = "container"; - serviceView.serviceName = "service1"; - serviceView.configId = "configId1"; - serviceView.host = "host1"; - serviceView.legacyStatusPages = "legacyPages"; - clusterView.services = Collections.singletonList(serviceView); - applicationView.clusters = Collections.singletonList(clusterView); - response.setResponse(applicationView); - ByteArrayOutputStream stream = new ByteArrayOutputStream(); - response.render(stream); - Slime responseSlime = SlimeUtils.jsonToSlime(stream.toByteArray()); - Slime expectedSlime = SlimeUtils.jsonToSlime(IOUtils.readFile(new File(responseFiles + "service-api-response.json")).getBytes(StandardCharsets.UTF_8)); - - assertEquals("service-api-response.json", - new String(SlimeUtils.toJsonBytes(expectedSlime), StandardCharsets.UTF_8), - new String(SlimeUtils.toJsonBytes(responseSlime), StandardCharsets.UTF_8)); - } - - @Test - public void testServiceViewResponseWithURLs() throws URISyntaxException, IOException { - ServiceApiResponse response = new ServiceApiResponse(ZoneId.from(Environment.prod, RegionName.from("us-west-1")), - ApplicationId.from("tenant2", "application2", "default"), - Collections.singletonList(new URI("http://cfg1.test/")), - new URI("http://cfg1.test/serviceview/v1/tenant/tenant2/application/application2/environment/prod/region/us-west-1/instance/default/service/searchnode-9dujk1pa0vufxrj6n4yvmi8uc/state/v1")); - ApplicationView applicationView = new ApplicationView(); - ClusterView clusterView = new ClusterView(); - clusterView.type = "container"; - clusterView.name = "cluster1"; - clusterView.url = "http://cfg1.test/serviceview/v1/tenant/tenant2/application/application2/environment/prod/region/us-west-1/instance/default/service/searchnode-9dujk1pa0vufxrj6n4yvmi8uc/state/v1/health"; - ServiceView serviceView = new ServiceView(); - serviceView.url = null; - serviceView.serviceType = "container"; - serviceView.serviceName = "service1"; - serviceView.configId = "configId1"; - serviceView.host = "host1"; - serviceView.legacyStatusPages = "legacyPages"; - clusterView.services = Collections.singletonList(serviceView); - applicationView.clusters = Collections.singletonList(clusterView); - response.setResponse(applicationView); - ByteArrayOutputStream stream = new ByteArrayOutputStream(); - response.render(stream); - Slime responseSlime = SlimeUtils.jsonToSlime(stream.toByteArray()); - Slime expectedSlime = SlimeUtils.jsonToSlime(IOUtils.readFile(new File(responseFiles + "service-api-response-with-urls.json")).getBytes(StandardCharsets.UTF_8)); - - assertEquals("service-api-response.json", - new String(SlimeUtils.toJsonBytes(expectedSlime), StandardCharsets.UTF_8), - new String(SlimeUtils.toJsonBytes(responseSlime), StandardCharsets.UTF_8)); - } - -} diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/service-api-response-with-urls.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/service-api-response-with-urls.json deleted file mode 100644 index 0e610c4d4b2..00000000000 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/service-api-response-with-urls.json +++ /dev/null @@ -1,18 +0,0 @@ -{ - "clusters": [ - { - "name": "cluster1", - "type": "container", - "url": "http://cfg1.test/serviceview/v1/tenant/tenant2/application/application2/environment/prod/region/us-west-1/instance/default/service/searchnode-9dujk1pa0vufxrj6n4yvmi8uc/state/v1/searchnode-9dujk1pa0vufxrj6n4yvmi8uc/state/v1/health", - "services": [ - { - "url": null, - "serviceType": "container", - "serviceName": "service1", - "configId": "configId1", - "host": "host1" - } - ] - } - ] -} diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/service-api-response.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/service-api-response.json deleted file mode 100644 index 3380eb26911..00000000000 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/service-api-response.json +++ /dev/null @@ -1,18 +0,0 @@ -{ - "clusters": [ - { - "name": "cluster1", - "type": "container", - "url": "cluster-url", - "services": [ - { - "url": null, - "serviceType": "container", - "serviceName": "service1", - "configId": "configId1", - "host": "host1" - } - ] - } - ] -}
\ No newline at end of file diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/service.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/service.json deleted file mode 100644 index 81892fd547e..00000000000 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/service.json +++ /dev/null @@ -1,7 +0,0 @@ -{ - "resources": [ - { - "url": "http://localhost:8080/application/v4/tenant/tenant1/application/application1/environment/prod/region/us-west-1/instance/default/service/filedistributorservice-dud1f4w037qdxdrn0ovxfdtgw/state/v1/config" - } - ] -} diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/services.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/services.json deleted file mode 100644 index 1a434afafbb..00000000000 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/services.json +++ /dev/null @@ -1,18 +0,0 @@ -{ - "clusters": [ - { - "name": "cluster1", - "type": "content", - "url": "http://localhost:8080/application/v4/tenant/tenant1/application/application1/environment/prod/region/us-central-1/instance/instance1/service/container-clustercontroller-6s8slgtps7ry8uh6lx21ejjiv/cluster/v2/cluster1", - "services": [ - { - "url": "http://localhost:8080/application/v4/tenant/tenant1/application/application1/environment/prod/region/us-central-1/instance/instance1/service/storagenode-awe3slno6mmq2fye191y324jl/state/v1/", - "serviceType": "storagenode", - "serviceName": "storagenode", - "configId": "cluster1/storage/0", - "host": "host1" - } - ] - } - ] -} diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/deployment/BadgeApiTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/deployment/BadgeApiTest.java index cf6453235d3..c195b623c11 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/deployment/BadgeApiTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/deployment/BadgeApiTest.java @@ -44,11 +44,15 @@ public class BadgeApiTest extends ControllerContainerTest { .runJob(JobType.productionApSoutheast1) .failDeployment(JobType.testApSoutheast1); application.submit(applicationPackage) - .runJob(JobType.systemTest) + .failTests(JobType.systemTest, true) .runJob(JobType.stagingTest); for (int i = 0; i < 31; i++) - application.failDeployment(JobType.productionUsWest1); + if ((i & 1) == 0) + application.failDeployment(JobType.productionUsWest1); + else + application.triggerJobs().abortJob(JobType.productionUsWest1); application.triggerJobs(); + tester.controller().applications().deploymentTrigger().reTrigger(application.instanceId(), JobType.systemTest, "reason"); tester.controller().applications().deploymentTrigger().reTrigger(application.instanceId(), JobType.testEuWest1, "reason"); tester.assertResponse(authenticatedRequest("http://localhost:8080/badge/v1/tenant/application/default"), diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/deployment/responses/history.svg b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/deployment/responses/history.svg index 0e30796bae2..c0566ade33a 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/deployment/responses/history.svg +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/deployment/responses/history.svg @@ -33,9 +33,15 @@ <animate attributeName='x1' values='-110%;150%;20%;-110%' dur='6s' repeatCount='indefinite' /> <animate attributeName='x2' values='-10%;250%;120%;-10%' dur='6s' repeatCount='indefinite' /> </linearGradient> + <linearGradient id='run-on-warning' x1='40%' x2='80%' y2='0%'> + <stop offset='0' stop-color='#ab83ff' /> + <stop offset='1' stop-color='#bd890b' /> + <animate attributeName='x1' values='-110%;150%;20%;-110%' dur='6s' repeatCount='indefinite' /> + <animate attributeName='x2' values='-10%;250%;120%;-10%' dur='6s' repeatCount='indefinite' /> + </linearGradient> <linearGradient id='run-on-success' x1='40%' x2='80%' y2='0%'> <stop offset='0' stop-color='#ab83ff' /> - <stop offset='1' stop-color='#00f244' /> + <stop offset='1' stop-color='#00f844' /> <animate attributeName='x1' values='-110%;150%;20%;-110%' dur='6s' repeatCount='indefinite' /> <animate attributeName='x2' values='-10%;250%;120%;-10%' dur='6s' repeatCount='indefinite' /> </linearGradient> @@ -44,100 +50,100 @@ </clipPath> <g clip-path='url(#rounded)'> <rect x='653.26809109179' rx='3' width='8' height='20' fill='url(#shadow)'/> - <rect x='646.1879570885093' rx='3' width='13.080134003280707' height='20' fill='#00f244'/> + <rect x='646.1879570885093' rx='3' width='13.080134003280707' height='20' fill='#00f844'/> <rect x='646.1879570885093' rx='3' width='13.080134003280707' height='20' fill='url(#shade)'/> <rect x='646.4043039211865' rx='3' width='8' height='20' fill='url(#shadow)'/> - <rect x='639.1078230852286' rx='3' width='13.296480835957981' height='20' fill='#00f244'/> + <rect x='639.1078230852286' rx='3' width='13.296480835957981' height='20' fill='#00f844'/> <rect x='639.1078230852286' rx='3' width='13.296480835957981' height='20' fill='url(#shade)'/> <rect x='639.3307808029524' rx='3' width='8' height='20' fill='url(#shadow)'/> <rect x='631.8113422492706' rx='3' width='13.519438553681752' height='20' fill='#bf103c'/> <rect x='631.8113422492706' rx='3' width='13.519438553681752' height='20' fill='url(#shade)'/> <rect x='632.0411128600877' rx='3' width='8' height='20' fill='url(#shadow)'/> - <rect x='624.2919036955889' rx='3' width='13.749209164498811' height='20' fill='#bf103c'/> + <rect x='624.2919036955889' rx='3' width='13.749209164498811' height='20' fill='#bd890b'/> <rect x='624.2919036955889' rx='3' width='13.749209164498811' height='20' fill='url(#shade)'/> <rect x='624.5286953802824' rx='3' width='8' height='20' fill='url(#shadow)'/> <rect x='616.5426945310901' rx='3' width='13.986000849192376' height='20' fill='#bf103c'/> <rect x='616.5426945310901' rx='3' width='13.986000849192376' height='20' fill='url(#shade)'/> <rect x='616.7867218317995' rx='3' width='8' height='20' fill='url(#shadow)'/> - <rect x='608.5566936818977' rx='3' width='14.230028149901685' height='20' fill='#bf103c'/> + <rect x='608.5566936818977' rx='3' width='14.230028149901685' height='20' fill='#bd890b'/> <rect x='608.5566936818977' rx='3' width='14.230028149901685' height='20' fill='url(#shade)'/> <rect x='608.8081776965013' rx='3' width='8' height='20' fill='url(#shadow)'/> <rect x='600.326665531996' rx='3' width='14.48151216450522' height='20' fill='#bf103c'/> <rect x='600.326665531996' rx='3' width='14.48151216450522' height='20' fill='url(#shade)'/> <rect x='600.5858341144344' rx='3' width='8' height='20' fill='url(#shadow)'/> - <rect x='591.8451533674908' rx='3' width='14.740680746943664' height='20' fill='#bf103c'/> + <rect x='591.8451533674908' rx='3' width='14.740680746943664' height='20' fill='#bd890b'/> <rect x='591.8451533674908' rx='3' width='14.740680746943664' height='20' fill='url(#shade)'/> <rect x='592.1122413342111' rx='3' width='8' height='20' fill='url(#shadow)'/> <rect x='583.1044726205471' rx='3' width='15.007768713664104' height='20' fill='#bf103c'/> <rect x='583.1044726205471' rx='3' width='15.007768713664104' height='20' fill='url(#shade)'/> <rect x='583.3797219632555' rx='3' width='8' height='20' fill='url(#shadow)'/> - <rect x='574.096703906883' rx='3' width='15.283018056372542' height='20' fill='#bf103c'/> + <rect x='574.096703906883' rx='3' width='15.283018056372542' height='20' fill='#bd890b'/> <rect x='574.096703906883' rx='3' width='15.283018056372542' height='20' fill='url(#shade)'/> <rect x='574.380364011798' rx='3' width='8' height='20' fill='url(#shadow)'/> <rect x='564.8136858505105' rx='3' width='15.566678161287442' height='20' fill='#bf103c'/> <rect x='564.8136858505105' rx='3' width='15.566678161287442' height='20' fill='url(#shade)'/> <rect x='565.1060137243161' rx='3' width='8' height='20' fill='url(#shadow)'/> - <rect x='555.2470076892231' rx='3' width='15.859006035092985' height='20' fill='#bf103c'/> + <rect x='555.2470076892231' rx='3' width='15.859006035092985' height='20' fill='#bd890b'/> <rect x='555.2470076892231' rx='3' width='15.859006035092985' height='20' fill='url(#shade)'/> <rect x='555.5482681919269' rx='3' width='8' height='20' fill='url(#shadow)'/> <rect x='545.3880016541301' rx='3' width='16.16026653779677' height='20' fill='#bf103c'/> <rect x='545.3880016541301' rx='3' width='16.16026653779677' height='20' fill='url(#shade)'/> <rect x='545.6984677390362' rx='3' width='8' height='20' fill='url(#shadow)'/> - <rect x='535.2277351163333' rx='3' width='16.470732622702883' height='20' fill='#bf103c'/> + <rect x='535.2277351163333' rx='3' width='16.470732622702883' height='20' fill='#bd890b'/> <rect x='535.2277351163333' rx='3' width='16.470732622702883' height='20' fill='url(#shade)'/> <rect x='535.5476880773482' rx='3' width='8' height='20' fill='url(#shadow)'/> <rect x='524.7570024936304' rx='3' width='16.790685583717845' height='20' fill='#bf103c'/> <rect x='524.7570024936304' rx='3' width='16.790685583717845' height='20' fill='url(#shade)'/> <rect x='525.0867322201259' rx='3' width='8' height='20' fill='url(#shadow)'/> - <rect x='513.9663169099125' rx='3' width='17.12041531021341' height='20' fill='#bf103c'/> + <rect x='513.9663169099125' rx='3' width='17.12041531021341' height='20' fill='#bd890b'/> <rect x='513.9663169099125' rx='3' width='17.12041531021341' height='20' fill='url(#shade)'/> <rect x='514.3061221493763' rx='3' width='8' height='20' fill='url(#shadow)'/> <rect x='502.84590159969906' rx='3' width='17.460220549677203' height='20' fill='#bf103c'/> <rect x='502.84590159969906' rx='3' width='17.460220549677203' height='20' fill='url(#shade)'/> <rect x='503.196090228411' rx='3' width='8' height='20' fill='url(#shadow)'/> - <rect x='491.38568105002184' rx='3' width='17.810409178389147' height='20' fill='#bf103c'/> + <rect x='491.38568105002184' rx='3' width='17.810409178389147' height='20' fill='#bd890b'/> <rect x='491.38568105002184' rx='3' width='17.810409178389147' height='20' fill='url(#shade)'/> <rect x='491.7465703520016' rx='3' width='8' height='20' fill='url(#shadow)'/> <rect x='479.5752718716327' rx='3' width='18.171298480368904' height='20' fill='#bf103c'/> <rect x='479.5752718716327' rx='3' width='18.171298480368904' height='20' fill='url(#shade)'/> <rect x='479.9471888261109' rx='3' width='8' height='20' fill='url(#shadow)'/> - <rect x='467.40397339126383' rx='3' width='18.543215434847077' height='20' fill='#bf103c'/> + <rect x='467.40397339126383' rx='3' width='18.543215434847077' height='20' fill='#bd890b'/> <rect x='467.40397339126383' rx='3' width='18.543215434847077' height='20' fill='url(#shade)'/> <rect x='467.7872549689374' rx='3' width='8' height='20' fill='url(#shadow)'/> <rect x='454.86075795641676' rx='3' width='18.92649701252067' height='20' fill='#bf103c'/> <rect x='454.86075795641676' rx='3' width='18.92649701252067' height='20' fill='url(#shade)'/> <rect x='455.25575142475725' rx='3' width='8' height='20' fill='url(#shadow)'/> - <rect x='441.9342609438961' rx='3' width='19.32149048086113' height='20' fill='#bf103c'/> + <rect x='441.9342609438961' rx='3' width='19.32149048086113' height='20' fill='#bd890b'/> <rect x='441.9342609438961' rx='3' width='19.32149048086113' height='20' fill='url(#shade)'/> <rect x='442.3413241817867' rx='3' width='8' height='20' fill='url(#shadow)'/> <rect x='428.61277046303496' rx='3' width='19.728553718751726' height='20' fill='#bf103c'/> <rect x='428.61277046303496' rx='3' width='19.728553718751726' height='20' fill='url(#shade)'/> <rect x='429.03227228502243' rx='3' width='8' height='20' fill='url(#shadow)'/> - <rect x='414.8842167442832' rx='3' width='20.148055540739207' height='20' fill='#bf103c'/> + <rect x='414.8842167442832' rx='3' width='20.148055540739207' height='20' fill='#bd890b'/> <rect x='414.8842167442832' rx='3' width='20.148055540739207' height='20' fill='url(#shade)'/> <rect x='415.31653723473755' rx='3' width='8' height='20' fill='url(#shadow)'/> <rect x='400.736161203544' rx='3' width='20.58037603119359' height='20' fill='#bf103c'/> <rect x='400.736161203544' rx='3' width='20.58037603119359' height='20' fill='url(#shade)'/> <rect x='401.1816920610293' rx='3' width='8' height='20' fill='url(#shadow)'/> - <rect x='386.1557851723504' rx='3' width='21.025906888678875' height='20' fill='#bf103c'/> + <rect x='386.1557851723504' rx='3' width='21.025906888678875' height='20' fill='#bd890b'/> <rect x='386.1557851723504' rx='3' width='21.025906888678875' height='20' fill='url(#shade)'/> <rect x='386.61493006451815' rx='3' width='8' height='20' fill='url(#shadow)'/> <rect x='371.12987828367153' rx='3' width='21.485051780846597' height='20' fill='#bf103c'/> <rect x='371.12987828367153' rx='3' width='21.485051780846597' height='20' fill='url(#shade)'/> <rect x='371.60305321299876' rx='3' width='8' height='20' fill='url(#shadow)'/> - <rect x='355.6448265028249' rx='3' width='21.958226710173836' height='20' fill='#bf103c'/> + <rect x='355.6448265028249' rx='3' width='21.958226710173836' height='20' fill='#bd890b'/> <rect x='355.6448265028249' rx='3' width='21.958226710173836' height='20' fill='url(#shade)'/> <rect x='356.13246018352817' rx='3' width='8' height='20' fill='url(#shadow)'/> <rect x='339.68659979265107' rx='3' width='22.445860390877083' height='20' fill='#bf103c'/> <rect x='339.68659979265107' rx='3' width='22.445860390877083' height='20' fill='url(#shade)'/> <rect x='340.18913403911733' rx='3' width='8' height='20' fill='url(#shadow)'/> - <rect x='323.24073940177396' rx='3' width='22.948394637343355' height='20' fill='#bf103c'/> + <rect x='323.24073940177396' rx='3' width='22.948394637343355' height='20' fill='#bd890b'/> <rect x='323.24073940177396' rx='3' width='22.948394637343355' height='20' fill='url(#shade)'/> <rect x='323.7586295288612' rx='3' width='8' height='20' fill='url(#shadow)'/> <rect x='306.2923447644306' rx='3' width='23.466284764430597' height='20' fill='#bf103c'/> <rect x='306.2923447644306' rx='3' width='23.466284764430597' height='20' fill='url(#shade)'/> <rect x='306.82606' rx='3' width='8' height='20' fill='url(#shadow)'/> - <rect x='288.82606' rx='3' width='24.0' height='20' fill='#bf103c'/> + <rect x='288.82606' rx='3' width='24.0' height='20' fill='#bd890b'/> <rect x='288.82606' rx='3' width='24.0' height='20' fill='url(#shade)'/> <rect x='288.82606' rx='3' width='8' height='20' fill='url(#shadow)'/> <rect x='163.18729000000002' rx='3' width='131.63876999999997' height='20' fill='url(#run-on-failure)'/> diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/deployment/responses/history2.svg b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/deployment/responses/history2.svg index 73d65b08b69..e527fa8d80f 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/deployment/responses/history2.svg +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/deployment/responses/history2.svg @@ -33,9 +33,15 @@ <animate attributeName='x1' values='-110%;150%;20%;-110%' dur='6s' repeatCount='indefinite' /> <animate attributeName='x2' values='-10%;250%;120%;-10%' dur='6s' repeatCount='indefinite' /> </linearGradient> + <linearGradient id='run-on-warning' x1='40%' x2='80%' y2='0%'> + <stop offset='0' stop-color='#ab83ff' /> + <stop offset='1' stop-color='#bd890b' /> + <animate attributeName='x1' values='-110%;150%;20%;-110%' dur='6s' repeatCount='indefinite' /> + <animate attributeName='x2' values='-10%;250%;120%;-10%' dur='6s' repeatCount='indefinite' /> + </linearGradient> <linearGradient id='run-on-success' x1='40%' x2='80%' y2='0%'> <stop offset='0' stop-color='#ab83ff' /> - <stop offset='1' stop-color='#00f244' /> + <stop offset='1' stop-color='#00f844' /> <animate attributeName='x1' values='-110%;150%;20%;-110%' dur='6s' repeatCount='indefinite' /> <animate attributeName='x2' values='-10%;250%;120%;-10%' dur='6s' repeatCount='indefinite' /> </linearGradient> @@ -44,103 +50,103 @@ </clipPath> <g clip-path='url(#rounded)'> <rect x='653.26809109179' rx='3' width='8' height='20' fill='url(#shadow)'/> - <rect x='646.1879570885093' rx='3' width='13.080134003280707' height='20' fill='#00f244'/> + <rect x='646.1879570885093' rx='3' width='13.080134003280707' height='20' fill='#00f844'/> <rect x='646.1879570885093' rx='3' width='13.080134003280707' height='20' fill='url(#shade)'/> <rect x='646.4043039211865' rx='3' width='8' height='20' fill='url(#shadow)'/> <rect x='639.1078230852286' rx='3' width='13.296480835957981' height='20' fill='#bf103c'/> <rect x='639.1078230852286' rx='3' width='13.296480835957981' height='20' fill='url(#shade)'/> <rect x='639.3307808029524' rx='3' width='8' height='20' fill='url(#shadow)'/> - <rect x='631.8113422492706' rx='3' width='13.519438553681752' height='20' fill='#bf103c'/> + <rect x='631.8113422492706' rx='3' width='13.519438553681752' height='20' fill='#bd890b'/> <rect x='631.8113422492706' rx='3' width='13.519438553681752' height='20' fill='url(#shade)'/> <rect x='632.0411128600877' rx='3' width='8' height='20' fill='url(#shadow)'/> <rect x='624.2919036955889' rx='3' width='13.749209164498811' height='20' fill='#bf103c'/> <rect x='624.2919036955889' rx='3' width='13.749209164498811' height='20' fill='url(#shade)'/> <rect x='624.5286953802824' rx='3' width='8' height='20' fill='url(#shadow)'/> - <rect x='616.5426945310901' rx='3' width='13.986000849192376' height='20' fill='#bf103c'/> + <rect x='616.5426945310901' rx='3' width='13.986000849192376' height='20' fill='#bd890b'/> <rect x='616.5426945310901' rx='3' width='13.986000849192376' height='20' fill='url(#shade)'/> <rect x='616.7867218317995' rx='3' width='8' height='20' fill='url(#shadow)'/> <rect x='608.5566936818977' rx='3' width='14.230028149901685' height='20' fill='#bf103c'/> <rect x='608.5566936818977' rx='3' width='14.230028149901685' height='20' fill='url(#shade)'/> <rect x='608.8081776965013' rx='3' width='8' height='20' fill='url(#shadow)'/> - <rect x='600.326665531996' rx='3' width='14.48151216450522' height='20' fill='#bf103c'/> + <rect x='600.326665531996' rx='3' width='14.48151216450522' height='20' fill='#bd890b'/> <rect x='600.326665531996' rx='3' width='14.48151216450522' height='20' fill='url(#shade)'/> <rect x='600.5858341144344' rx='3' width='8' height='20' fill='url(#shadow)'/> <rect x='591.8451533674908' rx='3' width='14.740680746943664' height='20' fill='#bf103c'/> <rect x='591.8451533674908' rx='3' width='14.740680746943664' height='20' fill='url(#shade)'/> <rect x='592.1122413342111' rx='3' width='8' height='20' fill='url(#shadow)'/> - <rect x='583.1044726205471' rx='3' width='15.007768713664104' height='20' fill='#bf103c'/> + <rect x='583.1044726205471' rx='3' width='15.007768713664104' height='20' fill='#bd890b'/> <rect x='583.1044726205471' rx='3' width='15.007768713664104' height='20' fill='url(#shade)'/> <rect x='583.3797219632555' rx='3' width='8' height='20' fill='url(#shadow)'/> <rect x='574.096703906883' rx='3' width='15.283018056372542' height='20' fill='#bf103c'/> <rect x='574.096703906883' rx='3' width='15.283018056372542' height='20' fill='url(#shade)'/> <rect x='574.380364011798' rx='3' width='8' height='20' fill='url(#shadow)'/> - <rect x='564.8136858505105' rx='3' width='15.566678161287442' height='20' fill='#bf103c'/> + <rect x='564.8136858505105' rx='3' width='15.566678161287442' height='20' fill='#bd890b'/> <rect x='564.8136858505105' rx='3' width='15.566678161287442' height='20' fill='url(#shade)'/> <rect x='565.1060137243161' rx='3' width='8' height='20' fill='url(#shadow)'/> <rect x='555.2470076892231' rx='3' width='15.859006035092985' height='20' fill='#bf103c'/> <rect x='555.2470076892231' rx='3' width='15.859006035092985' height='20' fill='url(#shade)'/> <rect x='555.5482681919269' rx='3' width='8' height='20' fill='url(#shadow)'/> - <rect x='545.3880016541301' rx='3' width='16.16026653779677' height='20' fill='#bf103c'/> + <rect x='545.3880016541301' rx='3' width='16.16026653779677' height='20' fill='#bd890b'/> <rect x='545.3880016541301' rx='3' width='16.16026653779677' height='20' fill='url(#shade)'/> <rect x='545.6984677390362' rx='3' width='8' height='20' fill='url(#shadow)'/> <rect x='535.2277351163333' rx='3' width='16.470732622702883' height='20' fill='#bf103c'/> <rect x='535.2277351163333' rx='3' width='16.470732622702883' height='20' fill='url(#shade)'/> <rect x='535.5476880773482' rx='3' width='8' height='20' fill='url(#shadow)'/> - <rect x='524.7570024936304' rx='3' width='16.790685583717845' height='20' fill='#bf103c'/> + <rect x='524.7570024936304' rx='3' width='16.790685583717845' height='20' fill='#bd890b'/> <rect x='524.7570024936304' rx='3' width='16.790685583717845' height='20' fill='url(#shade)'/> <rect x='525.0867322201259' rx='3' width='8' height='20' fill='url(#shadow)'/> <rect x='513.9663169099125' rx='3' width='17.12041531021341' height='20' fill='#bf103c'/> <rect x='513.9663169099125' rx='3' width='17.12041531021341' height='20' fill='url(#shade)'/> <rect x='514.3061221493763' rx='3' width='8' height='20' fill='url(#shadow)'/> - <rect x='502.84590159969906' rx='3' width='17.460220549677203' height='20' fill='#bf103c'/> + <rect x='502.84590159969906' rx='3' width='17.460220549677203' height='20' fill='#bd890b'/> <rect x='502.84590159969906' rx='3' width='17.460220549677203' height='20' fill='url(#shade)'/> <rect x='503.196090228411' rx='3' width='8' height='20' fill='url(#shadow)'/> <rect x='491.38568105002184' rx='3' width='17.810409178389147' height='20' fill='#bf103c'/> <rect x='491.38568105002184' rx='3' width='17.810409178389147' height='20' fill='url(#shade)'/> <rect x='491.7465703520016' rx='3' width='8' height='20' fill='url(#shadow)'/> - <rect x='479.5752718716327' rx='3' width='18.171298480368904' height='20' fill='#bf103c'/> + <rect x='479.5752718716327' rx='3' width='18.171298480368904' height='20' fill='#bd890b'/> <rect x='479.5752718716327' rx='3' width='18.171298480368904' height='20' fill='url(#shade)'/> <rect x='479.9471888261109' rx='3' width='8' height='20' fill='url(#shadow)'/> <rect x='467.40397339126383' rx='3' width='18.543215434847077' height='20' fill='#bf103c'/> <rect x='467.40397339126383' rx='3' width='18.543215434847077' height='20' fill='url(#shade)'/> <rect x='467.7872549689374' rx='3' width='8' height='20' fill='url(#shadow)'/> - <rect x='454.86075795641676' rx='3' width='18.92649701252067' height='20' fill='#bf103c'/> + <rect x='454.86075795641676' rx='3' width='18.92649701252067' height='20' fill='#bd890b'/> <rect x='454.86075795641676' rx='3' width='18.92649701252067' height='20' fill='url(#shade)'/> <rect x='455.25575142475725' rx='3' width='8' height='20' fill='url(#shadow)'/> <rect x='441.9342609438961' rx='3' width='19.32149048086113' height='20' fill='#bf103c'/> <rect x='441.9342609438961' rx='3' width='19.32149048086113' height='20' fill='url(#shade)'/> <rect x='442.3413241817867' rx='3' width='8' height='20' fill='url(#shadow)'/> - <rect x='428.61277046303496' rx='3' width='19.728553718751726' height='20' fill='#bf103c'/> + <rect x='428.61277046303496' rx='3' width='19.728553718751726' height='20' fill='#bd890b'/> <rect x='428.61277046303496' rx='3' width='19.728553718751726' height='20' fill='url(#shade)'/> <rect x='429.03227228502243' rx='3' width='8' height='20' fill='url(#shadow)'/> <rect x='414.8842167442832' rx='3' width='20.148055540739207' height='20' fill='#bf103c'/> <rect x='414.8842167442832' rx='3' width='20.148055540739207' height='20' fill='url(#shade)'/> <rect x='415.31653723473755' rx='3' width='8' height='20' fill='url(#shadow)'/> - <rect x='400.736161203544' rx='3' width='20.58037603119359' height='20' fill='#bf103c'/> + <rect x='400.736161203544' rx='3' width='20.58037603119359' height='20' fill='#bd890b'/> <rect x='400.736161203544' rx='3' width='20.58037603119359' height='20' fill='url(#shade)'/> <rect x='401.1816920610293' rx='3' width='8' height='20' fill='url(#shadow)'/> <rect x='386.1557851723504' rx='3' width='21.025906888678875' height='20' fill='#bf103c'/> <rect x='386.1557851723504' rx='3' width='21.025906888678875' height='20' fill='url(#shade)'/> <rect x='386.61493006451815' rx='3' width='8' height='20' fill='url(#shadow)'/> - <rect x='371.12987828367153' rx='3' width='21.485051780846597' height='20' fill='#bf103c'/> + <rect x='371.12987828367153' rx='3' width='21.485051780846597' height='20' fill='#bd890b'/> <rect x='371.12987828367153' rx='3' width='21.485051780846597' height='20' fill='url(#shade)'/> <rect x='371.60305321299876' rx='3' width='8' height='20' fill='url(#shadow)'/> <rect x='355.6448265028249' rx='3' width='21.958226710173836' height='20' fill='#bf103c'/> <rect x='355.6448265028249' rx='3' width='21.958226710173836' height='20' fill='url(#shade)'/> <rect x='356.13246018352817' rx='3' width='8' height='20' fill='url(#shadow)'/> - <rect x='339.68659979265107' rx='3' width='22.445860390877083' height='20' fill='#bf103c'/> + <rect x='339.68659979265107' rx='3' width='22.445860390877083' height='20' fill='#bd890b'/> <rect x='339.68659979265107' rx='3' width='22.445860390877083' height='20' fill='url(#shade)'/> <rect x='340.18913403911733' rx='3' width='8' height='20' fill='url(#shadow)'/> <rect x='323.24073940177396' rx='3' width='22.948394637343355' height='20' fill='#bf103c'/> <rect x='323.24073940177396' rx='3' width='22.948394637343355' height='20' fill='url(#shade)'/> <rect x='323.7586295288612' rx='3' width='8' height='20' fill='url(#shadow)'/> - <rect x='306.2923447644306' rx='3' width='23.466284764430597' height='20' fill='#bf103c'/> + <rect x='306.2923447644306' rx='3' width='23.466284764430597' height='20' fill='#bd890b'/> <rect x='306.2923447644306' rx='3' width='23.466284764430597' height='20' fill='url(#shade)'/> <rect x='306.82606' rx='3' width='8' height='20' fill='url(#shadow)'/> <rect x='288.82606' rx='3' width='24.0' height='20' fill='#bf103c'/> <rect x='288.82606' rx='3' width='24.0' height='20' fill='url(#shade)'/> <rect x='288.82606' rx='3' width='8' height='20' fill='url(#shadow)'/> - <rect x='163.18729000000002' rx='3' width='131.63876999999997' height='20' fill='#00f244'/> + <rect x='163.18729000000002' rx='3' width='131.63876999999997' height='20' fill='#00f844'/> <rect x='163.18729000000002' rx='3' width='131.63876999999997' height='20' fill='url(#shade)'/> <rect width='169.18729000000002' height='20' fill='#404040'/> <rect x='-6.0' rx='3' width='175.18729000000002' height='20' fill='url(#shade)'/> diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/deployment/responses/overview.svg b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/deployment/responses/overview.svg index dde2b740e37..a0005ed6d76 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/deployment/responses/overview.svg +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/deployment/responses/overview.svg @@ -33,9 +33,15 @@ <animate attributeName='x1' values='-110%;150%;20%;-110%' dur='6s' repeatCount='indefinite' /> <animate attributeName='x2' values='-10%;250%;120%;-10%' dur='6s' repeatCount='indefinite' /> </linearGradient> + <linearGradient id='run-on-warning' x1='40%' x2='80%' y2='0%'> + <stop offset='0' stop-color='#ab83ff' /> + <stop offset='1' stop-color='#bd890b' /> + <animate attributeName='x1' values='-110%;150%;20%;-110%' dur='6s' repeatCount='indefinite' /> + <animate attributeName='x2' values='-10%;250%;120%;-10%' dur='6s' repeatCount='indefinite' /> + </linearGradient> <linearGradient id='run-on-success' x1='40%' x2='80%' y2='0%'> <stop offset='0' stop-color='#ab83ff' /> - <stop offset='1' stop-color='#00f244' /> + <stop offset='1' stop-color='#00f844' /> <animate attributeName='x1' values='-110%;150%;20%;-110%' dur='6s' repeatCount='indefinite' /> <animate attributeName='x2' values='-10%;250%;120%;-10%' dur='6s' repeatCount='indefinite' /> </linearGradient> @@ -45,21 +51,21 @@ <g clip-path='url(#rounded)'> <rect x='757.7809900000001' rx='3' width='8' height='20' fill='url(#shadow)'/> <rect x='725.59036' rx='3' width='38.19063' height='20' fill='url(#run-on-success)'/> - <polygon points='635.8470950000001 0 635.8470950000001 20 734.59036 20 742.59036 0' fill='#00f244'/> + <polygon points='635.8470950000001 0 635.8470950000001 20 734.59036 20 742.59036 0' fill='#00f844'/> <rect x='635.8470950000001' rx='3' width='131.74345499999998' height='20' fill='url(#shade)'/> <rect x='635.8470950000001' rx='3' width='8' height='20' fill='url(#shadow)'/> <rect x='603.656465' rx='3' width='38.19063' height='20' fill='#bf103c'/> - <polygon points='486.981225 0 486.981225 20 612.656465 20 620.656465 0' fill='#00f244'/> + <polygon points='486.981225 0 486.981225 20 612.656465 20 620.656465 0' fill='#00f844'/> <rect x='486.981225' rx='3' width='158.67543' height='20' fill='url(#shade)'/> <rect x='486.981225' rx='3' width='8' height='20' fill='url(#shadow)'/> <rect x='348.865175' rx='3' width='144.11604999999997' height='20' fill='url(#run-on-success)'/> <rect x='358.865175' rx='3' width='134.11604999999997' height='20' fill='url(#shade)'/> <rect x='358.865175' rx='3' width='8' height='20' fill='url(#shadow)'/> - <rect x='326.674545' rx='3' width='38.19063' height='20' fill='#00f244'/> + <rect x='326.674545' rx='3' width='38.19063' height='20' fill='#00f844'/> <polygon points='237.71563000000003 0 237.71563000000003 20 335.674545 20 343.674545 0' fill='url(#run-on-failure)'/> <rect x='237.71563000000003' rx='3' width='130.959105' height='20' fill='url(#shade)'/> <rect x='237.71563000000003' rx='3' width='8' height='20' fill='url(#shadow)'/> - <rect x='153.18729000000002' rx='3' width='90.52834000000001' height='20' fill='#00f244'/> + <rect x='153.18729000000002' rx='3' width='90.52834000000001' height='20' fill='url(#run-on-warning)'/> <rect x='163.18729000000002' rx='3' width='80.52834000000001' height='20' fill='url(#shade)'/> <rect width='169.18729000000002' height='20' fill='#404040'/> <rect x='-6.0' rx='3' width='175.18729000000002' height='20' fill='url(#shade)'/> diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/deployment/responses/root.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/deployment/responses/root.json index f09b60efacc..9c34f9410ee 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/deployment/responses/root.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/deployment/responses/root.json @@ -353,25 +353,21 @@ "jobs": [ "system-test", "staging-test", - "production-us-east-3", - "test-us-east-3", - "production-us-west-1", - "test-us-west-1", - "production-us-central-1", - "test-us-central-1", + "production-aws-us-east-1a", + "test-aws-us-east-1a", "production-ap-northeast-1", "test-ap-northeast-1", "production-ap-northeast-2", "test-ap-northeast-2", "production-ap-southeast-1", "test-ap-southeast-1", + "production-us-east-3", + "test-us-east-3", + "production-us-west-1", + "test-us-west-1", + "production-us-central-1", + "test-us-central-1", "production-eu-west-1", - "test-eu-west-1", - "production-aws-us-east-1a", - "test-aws-us-east-1a", - "production-aws-us-west-2a", - "test-aws-us-west-2a", - "production-aws-us-east-1b", - "test-aws-us-east-1b" + "test-eu-west-1" ] } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/deployment/responses/single-done.svg b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/deployment/responses/single-done.svg index 3bcbed97499..0bdaa3f30ad 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/deployment/responses/single-done.svg +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/deployment/responses/single-done.svg @@ -33,9 +33,15 @@ <animate attributeName='x1' values='-110%;150%;20%;-110%' dur='6s' repeatCount='indefinite' /> <animate attributeName='x2' values='-10%;250%;120%;-10%' dur='6s' repeatCount='indefinite' /> </linearGradient> + <linearGradient id='run-on-warning' x1='40%' x2='80%' y2='0%'> + <stop offset='0' stop-color='#ab83ff' /> + <stop offset='1' stop-color='#bd890b' /> + <animate attributeName='x1' values='-110%;150%;20%;-110%' dur='6s' repeatCount='indefinite' /> + <animate attributeName='x2' values='-10%;250%;120%;-10%' dur='6s' repeatCount='indefinite' /> + </linearGradient> <linearGradient id='run-on-success' x1='40%' x2='80%' y2='0%'> <stop offset='0' stop-color='#ab83ff' /> - <stop offset='1' stop-color='#00f244' /> + <stop offset='1' stop-color='#00f844' /> <animate attributeName='x1' values='-110%;150%;20%;-110%' dur='6s' repeatCount='indefinite' /> <animate attributeName='x2' values='-10%;250%;120%;-10%' dur='6s' repeatCount='indefinite' /> </linearGradient> @@ -44,7 +50,7 @@ </clipPath> <g clip-path='url(#rounded)'> <rect x='288.82606' rx='3' width='8' height='20' fill='url(#shadow)'/> - <rect x='163.18729000000002' rx='3' width='131.63876999999997' height='20' fill='#00f244'/> + <rect x='163.18729000000002' rx='3' width='131.63876999999997' height='20' fill='#00f844'/> <rect x='163.18729000000002' rx='3' width='131.63876999999997' height='20' fill='url(#shade)'/> <rect width='169.18729000000002' height='20' fill='#404040'/> <rect x='-6.0' rx='3' width='175.18729000000002' height='20' fill='url(#shade)'/> diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/deployment/responses/single-running.svg b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/deployment/responses/single-running.svg index 27e967f8e46..1a38228e75d 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/deployment/responses/single-running.svg +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/deployment/responses/single-running.svg @@ -33,9 +33,15 @@ <animate attributeName='x1' values='-110%;150%;20%;-110%' dur='6s' repeatCount='indefinite' /> <animate attributeName='x2' values='-10%;250%;120%;-10%' dur='6s' repeatCount='indefinite' /> </linearGradient> + <linearGradient id='run-on-warning' x1='40%' x2='80%' y2='0%'> + <stop offset='0' stop-color='#ab83ff' /> + <stop offset='1' stop-color='#bd890b' /> + <animate attributeName='x1' values='-110%;150%;20%;-110%' dur='6s' repeatCount='indefinite' /> + <animate attributeName='x2' values='-10%;250%;120%;-10%' dur='6s' repeatCount='indefinite' /> + </linearGradient> <linearGradient id='run-on-success' x1='40%' x2='80%' y2='0%'> <stop offset='0' stop-color='#ab83ff' /> - <stop offset='1' stop-color='#00f244' /> + <stop offset='1' stop-color='#00f844' /> <animate attributeName='x1' values='-110%;150%;20%;-110%' dur='6s' repeatCount='indefinite' /> <animate attributeName='x2' values='-10%;250%;120%;-10%' dur='6s' repeatCount='indefinite' /> </linearGradient> diff --git a/document/src/tests/fieldpathupdatetestcase.cpp b/document/src/tests/fieldpathupdatetestcase.cpp index 1d72fafa607..89a95c1d62f 100644 --- a/document/src/tests/fieldpathupdatetestcase.cpp +++ b/document/src/tests/fieldpathupdatetestcase.cpp @@ -52,100 +52,6 @@ protected: namespace { -DocumenttypesConfig getRepoConfig() { - const int struct2_id = 64; - DocumenttypesConfigBuilderHelper builder; - builder.document( - 42, "test", - Struct("test.header") - .addField("primitive1", DataType::T_INT) - .addField("l1s1", Struct("struct3") - .addField("primitive1", DataType::T_INT) - .addField("ss", Struct("struct2") - .setId(struct2_id) - .addField("primitive1", DataType::T_INT) - .addField("primitive2", DataType::T_INT) - .addField("iarray", Array(DataType::T_INT)) - .addField("sarray", Array( - Struct("struct1") - .addField("primitive1", - DataType::T_INT) - .addField("primitive2", - DataType::T_INT))) - .addField("smap", Map(DataType::T_STRING, - DataType::T_STRING))) - .addField("structmap", - Map(DataType::T_STRING, struct2_id)) - .addField("wset", Wset(DataType::T_STRING)) - .addField("structwset", Wset(struct2_id))), - Struct("test.body")); - return builder.config(); -} - -Document::UP -createTestDocument(const DocumentTypeRepo &repo) -{ - const DocumentType* type(repo.getDocumentType("test")); - const DataType* struct3(repo.getDataType(*type, "struct3")); - const DataType* struct2(repo.getDataType(*type, "struct2")); - const DataType* iarr(repo.getDataType(*type, "Array<Int>")); - const DataType* sarr(repo.getDataType(*type, "Array<struct1>")); - const DataType* struct1(repo.getDataType(*type, "struct1")); - const DataType* smap(repo.getDataType(*type, "Map<String,String>")); - const DataType* structmap(repo.getDataType(*type, "Map<String,struct2>")); - const DataType* wset(repo.getDataType(*type, "WeightedSet<String>")); - const DataType* structwset(repo.getDataType(*type, "WeightedSet<struct2>")); - Document::UP doc(new Document(*type, DocumentId("id:ns:test::1"))); - doc->setRepo(repo); - doc->setValue("primitive1", IntFieldValue(1)); - StructFieldValue l1s1(*struct3); - l1s1.setValue("primitive1", IntFieldValue(2)); - - StructFieldValue l2s1(*struct2); - l2s1.setValue("primitive1", IntFieldValue(3)); - l2s1.setValue("primitive2", IntFieldValue(4)); - StructFieldValue l2s2(*struct2); - l2s2.setValue("primitive1", IntFieldValue(5)); - l2s2.setValue("primitive2", IntFieldValue(6)); - ArrayFieldValue iarr1(*iarr); - iarr1.add(IntFieldValue(11)); - iarr1.add(IntFieldValue(12)); - iarr1.add(IntFieldValue(13)); - ArrayFieldValue sarr1(*sarr); - StructFieldValue l3s1(*struct1); - l3s1.setValue("primitive1", IntFieldValue(1)); - l3s1.setValue("primitive2", IntFieldValue(2)); - sarr1.add(l3s1); - sarr1.add(l3s1); - MapFieldValue smap1(*smap); - smap1.put(StringFieldValue("leonardo"), StringFieldValue("dicaprio")); - smap1.put(StringFieldValue("ellen"), StringFieldValue("page")); - smap1.put(StringFieldValue("joseph"), StringFieldValue("gordon-levitt")); - l2s1.setValue("smap", smap1); - l2s1.setValue("iarray", iarr1); - l2s1.setValue("sarray", sarr1); - - l1s1.setValue("ss", l2s1); - MapFieldValue structmap1(*structmap); - structmap1.put(StringFieldValue("test"), l2s1); - l1s1.setValue("structmap", structmap1); - - WeightedSetFieldValue wwset1(*wset); - WSetHelper wset1(wwset1); - wset1.add("foo"); - wset1.add("bar"); - wset1.add("zoo"); - l1s1.setValue("wset", wwset1); - - WeightedSetFieldValue wset2(*structwset); - wset2.add(l2s1); - wset2.add(l2s2); - l1s1.setValue("structwset", wset2); - - doc->setValue("l1s1", l1s1); - return doc; -} - nbostream serializeHEAD(const DocumentUpdate & update) { @@ -191,61 +97,6 @@ void testSerialize(const DocumentTypeRepo& repo, const DocumentUpdate& a) { } // anon ns -struct TestFieldPathUpdate : FieldPathUpdate -{ - struct TestIteratorHandler : fieldvalue::IteratorHandler - { - TestIteratorHandler(std::string& str) - : _str(str) {} - - ModificationStatus doModify(FieldValue& value) override - { - std::ostringstream ss; - value.print(ss, false, ""); - if (!_str.empty()) { - _str += ';'; - } - _str += ss.str(); - return ModificationStatus::NOT_MODIFIED; - } - - bool onComplex(const Content&) override { return false; } - - std::string& _str; - }; - - mutable std::string _str; - - ~TestFieldPathUpdate(); - TestFieldPathUpdate(const std::string& fieldPath, const std::string& whereClause); - - TestFieldPathUpdate(const TestFieldPathUpdate& other); - - std::unique_ptr<IteratorHandler> getIteratorHandler(Document&, const DocumentTypeRepo &) const override { - return std::unique_ptr<IteratorHandler>(new TestIteratorHandler(_str)); - } - - TestFieldPathUpdate* clone() const override { return new TestFieldPathUpdate(*this); } - - void print(std::ostream& out, bool, const std::string&) const override { - out << "TestFieldPathUpdate()"; - } - - void accept(UpdateVisitor & visitor) const override { (void) visitor; } - uint8_t getSerializedType() const override { assert(false); return 7; } -}; - -TestFieldPathUpdate::~TestFieldPathUpdate() { } -TestFieldPathUpdate::TestFieldPathUpdate(const std::string& fieldPath, const std::string& whereClause) - : FieldPathUpdate(fieldPath, whereClause) -{ -} - -TestFieldPathUpdate::TestFieldPathUpdate(const TestFieldPathUpdate& other) - : FieldPathUpdate(other) -{ -} - FieldPathUpdateTestCase::FieldPathUpdateTestCase() : _foobar_type(nullptr) {} @@ -280,42 +131,12 @@ FieldPathUpdateTestCase::TearDown() { } -TEST_F(FieldPathUpdateTestCase, testWhereClause) -{ - DocumentTypeRepo repo(getRepoConfig()); - Document::UP doc(createTestDocument(repo)); - std::string where = "test.l1s1.structmap.value.smap{$x} == \"dicaprio\""; - TestFieldPathUpdate update("l1s1.structmap.value.smap{$x}", where); - update.applyTo(*doc); - EXPECT_EQ(std::string("dicaprio"), update._str); -} - -TEST_F(FieldPathUpdateTestCase, testBrokenWhereClause) -{ - DocumentTypeRepo repo(getRepoConfig()); - Document::UP doc(createTestDocument(repo)); - std::string where = "l1s1.structmap.value.smap{$x} == \"dicaprio\""; - TestFieldPathUpdate update("l1s1.structmap.value.smap{$x}", where); - update.applyTo(*doc); - EXPECT_EQ(std::string(""), update._str); -} - -TEST_F(FieldPathUpdateTestCase, testNoIterateMapValues) -{ - DocumentTypeRepo repo(getRepoConfig()); - Document::UP doc(createTestDocument(repo)); - TestFieldPathUpdate update("l1s1.structwset.primitive1", "true"); - update.applyTo(*doc); - EXPECT_EQ(std::string("3;5"), update._str); -} - TEST_F(FieldPathUpdateTestCase, testRemoveField) { auto doc = std::make_unique<Document>(*_foobar_type, DocumentId("id:ns:foobar::things:thangs")); EXPECT_TRUE(doc->hasValue("strfoo") == false); doc->setValue("strfoo", StringFieldValue("cocacola")); EXPECT_EQ(vespalib::string("cocacola"), doc->getValue("strfoo")->getAsString()); - //doc->print(std::cerr, true, ""); DocumentUpdate docUp(*_repo, *_foobar_type, DocumentId("id:ns:foobar::barbar:foofoo")); docUp.addFieldPathUpdate(std::make_unique<RemoveFieldPathUpdate>("strfoo")); docUp.applyTo(*doc); @@ -417,13 +238,13 @@ TEST_F(FieldPathUpdateTestCase, testApplyAssignSingle) EXPECT_TRUE(doc->hasValue("strfoo") == false); // Test assignment of non-existing DocumentUpdate docUp(*_repo, *_foobar_type, DocumentId("id:ns:foobar::barbar:foofoo")); - docUp.addFieldPathUpdate(std::make_unique<AssignFieldPathUpdate>(*doc->getDataType(), "strfoo", std::string(), StringFieldValue("himert"))); + docUp.addFieldPathUpdate(std::make_unique<AssignFieldPathUpdate>(*doc->getDataType(), "strfoo", std::string(), StringFieldValue::make("himert"))); docUp.applyTo(*doc); EXPECT_TRUE(doc->hasValue("strfoo")); EXPECT_EQ(vespalib::string("himert"), doc->getValue("strfoo")->getAsString()); // Test overwriting existing DocumentUpdate docUp2(*_repo, *_foobar_type, DocumentId("id:ns:foobar::barbar:foofoo")); - docUp2.addFieldPathUpdate(std::make_unique<AssignFieldPathUpdate>(*doc->getDataType(), "strfoo", std::string(), StringFieldValue("wunderbaum"))); + docUp2.addFieldPathUpdate(std::make_unique<AssignFieldPathUpdate>(*doc->getDataType(), "strfoo", std::string(), StringFieldValue::make("wunderbaum"))); docUp2.applyTo(*doc); EXPECT_EQ(vespalib::string("wunderbaum"), doc->getValue("strfoo")->getAsString()); } @@ -538,7 +359,7 @@ TEST_F(FieldPathUpdateTestCase, testAssignSimpleMapValueWithVariable) DocumentUpdate docUp(*_repo, *_foobar_type, DocumentId("id:ns:foobar::barbar:foofoo")); // Select on value, not key docUp.addFieldPathUpdate(std::make_unique<AssignFieldPathUpdate>(*doc->getDataType(), - "strmap{$x}", "foobar.strmap{$x} == \"bar\"", StringFieldValue("shinyvalue"))); + "strmap{$x}", "foobar.strmap{$x} == \"bar\"", StringFieldValue::make("shinyvalue"))); docUp.applyTo(*doc); std::unique_ptr<MapFieldValue> valueNow(doc->getAs<MapFieldValue>(doc->getField("strmap"))); @@ -581,16 +402,15 @@ TEST_F(FieldPathUpdateTestCase, testApplyAssignMultiList) EXPECT_TRUE(doc->hasValue("strarray")); } - ArrayFieldValue updateArray(doc->getType().getField("strarray").getDataType()); - updateArray.add(StringFieldValue("assigned val 0")); - updateArray.add(StringFieldValue("assigned val 1")); + auto updateArray = std::make_unique<ArrayFieldValue>(doc->getType().getField("strarray").getDataType()); + updateArray->add(StringFieldValue("assigned val 0")); + updateArray->add(StringFieldValue("assigned val 1")); DocumentUpdate docUp(*_repo, *_foobar_type, DocumentId("id:ns:foobar::barbar:foofoo")); - docUp.addFieldPathUpdate(std::make_unique<AssignFieldPathUpdate>(*doc->getDataType(), "strarray", std::string(), updateArray)); + docUp.addFieldPathUpdate(std::make_unique<AssignFieldPathUpdate>(*doc->getDataType(), "strarray", std::string(), std::move(updateArray))); docUp.applyTo(*doc); { - std::unique_ptr<ArrayFieldValue> strArray = - doc->getAs<ArrayFieldValue>(doc->getField("strarray")); + std::unique_ptr<ArrayFieldValue> strArray = doc->getAs<ArrayFieldValue>(doc->getField("strarray")); ASSERT_EQ(std::size_t(2), strArray->size()); EXPECT_EQ(vespalib::string("assigned val 0"), (*strArray)[0].getAsString()); EXPECT_EQ(vespalib::string("assigned val 1"), (*strArray)[1].getAsString()); @@ -611,12 +431,12 @@ TEST_F(FieldPathUpdateTestCase, testApplyAssignMultiWset) EXPECT_TRUE(doc->hasValue("strwset")); } - WeightedSetFieldValue assignWset(doc->getType().getField("strwset").getDataType()); - assignWset.add(StringFieldValue("assigned val 0"), 5); - assignWset.add(StringFieldValue("assigned val 1"), 10); + auto assignWset = std::make_unique<WeightedSetFieldValue>(doc->getType().getField("strwset").getDataType()); + assignWset->add(StringFieldValue("assigned val 0"), 5); + assignWset->add(StringFieldValue("assigned val 1"), 10); DocumentUpdate docUp(*_repo, *_foobar_type, DocumentId("id:ns:foobar::barbar:foofoo")); - docUp.addFieldPathUpdate(std::make_unique<AssignFieldPathUpdate>(*doc->getDataType(), "strwset", std::string(), assignWset)); + docUp.addFieldPathUpdate(std::make_unique<AssignFieldPathUpdate>(*doc->getDataType(), "strwset", std::string(), std::move(assignWset))); //doc->print(std::cerr, true, ""); docUp.applyTo(*doc); //doc->print(std::cerr, true, ""); @@ -643,8 +463,7 @@ TEST_F(FieldPathUpdateTestCase, testAssignWsetRemoveIfZero) { DocumentUpdate docUp(*_repo, *_foobar_type, DocumentId("id:ns:foobar::barbar:foofoo")); - IntFieldValue zeroWeight(0); - auto assignUpdate = std::make_unique<AssignFieldPathUpdate>(*doc->getDataType(), "strwset{you say goodbye}", std::string(), zeroWeight); + auto assignUpdate = std::make_unique<AssignFieldPathUpdate>(*doc->getDataType(), "strwset{you say goodbye}", std::string(), IntFieldValue::make(0)); static_cast<AssignFieldPathUpdate&>(*assignUpdate).setRemoveIfZero(true); docUp.addFieldPathUpdate(std::move(assignUpdate)); //doc->print(std::cerr, true, ""); @@ -663,13 +482,13 @@ TEST_F(FieldPathUpdateTestCase, testApplyAddMultiList) auto doc = std::make_unique<Document>(*_foobar_type, DocumentId("id:ns:foobar::george:costanza")); EXPECT_TRUE(doc->hasValue("strarray") == false); - ArrayFieldValue adds(doc->getType().getField("strarray").getDataType()); - adds.add(StringFieldValue("serenity now")); - adds.add(StringFieldValue("a festivus for the rest of us")); - adds.add(StringFieldValue("george is getting upset!")); + auto adds = std::make_unique<ArrayFieldValue>(doc->getType().getField("strarray").getDataType()); + adds->add(StringFieldValue("serenity now")); + adds->add(StringFieldValue("a festivus for the rest of us")); + adds->add(StringFieldValue("george is getting upset!")); DocumentUpdate docUp(*_repo, *_foobar_type, DocumentId("id:ns:foobar::barbar:foofoo")); - docUp.addFieldPathUpdate(std::make_unique<AddFieldPathUpdate>(*doc->getDataType(), "strarray", std::string(), adds)); + docUp.addFieldPathUpdate(std::make_unique<AddFieldPathUpdate>(*doc->getDataType(), "strarray", std::string(), std::move(adds))); //doc->print(std::cerr, true, ""); docUp.applyTo(*doc); //doc->print(std::cerr, true, ""); @@ -691,12 +510,12 @@ TEST_F(FieldPathUpdateTestCase, testAddAndAssignList) DocumentUpdate docUp(*_repo, *_foobar_type, DocumentId("id:ns:foobar::barbar:foofoo")); docUp.addFieldPathUpdate(std::make_unique<AssignFieldPathUpdate>(*doc->getDataType(), - "strarray[1]", std::string(), StringFieldValue("assigned val 1"))); + "strarray[1]", std::string(), StringFieldValue::make("assigned val 1"))); - ArrayFieldValue adds(doc->getType().getField("strarray").getDataType()); - adds.add(StringFieldValue("new value")); + auto adds = std::make_unique<ArrayFieldValue>(doc->getType().getField("strarray").getDataType()); + adds->add(StringFieldValue("new value")); - docUp.addFieldPathUpdate(std::make_unique<AddFieldPathUpdate>(*doc->getDataType(), "strarray", std::string(), adds)); + docUp.addFieldPathUpdate(std::make_unique<AddFieldPathUpdate>(*doc->getDataType(), "strarray", std::string(), std::move(adds))); //doc->print(std::cerr, true, ""); docUp.applyTo(*doc); //doc->print(std::cerr, true, ""); @@ -722,26 +541,34 @@ Keys::Keys() : key1("foo"), key2("bar"), key3("zoo") {} Keys::~Keys() {} struct Fixture { + const DocumentType * _doc_type; Document::UP doc; MapFieldValue mfv; - StructFieldValue fv1, fv2, fv3, fv4; + StructFieldValue fv1, fv2, fv3; - const MapDataType &getMapType(const DocumentType &doc_type) { + static const MapDataType & + getMapType(const DocumentType &doc_type) { return static_cast<const MapDataType &>(doc_type.getField("structmap").getDataType()); } + std::unique_ptr<FieldValue> fv4() const { + auto sval = std::make_unique<StructFieldValue>(getMapType(*_doc_type).getValueType()); + sval->setValue("title", StringFieldValue("farnsworth")); + sval->setValue("rating", IntFieldValue(48)); + return sval; + } ~Fixture(); Fixture(const DocumentType &doc_type, const Keys &k); }; Fixture::~Fixture() = default; Fixture::Fixture(const DocumentType &doc_type, const Keys &k) - : doc(new Document(doc_type, DocumentId("id:ns:" + doc_type.getName() + "::planet:express"))), + : _doc_type(&doc_type), + doc(new Document(doc_type, DocumentId("id:ns:" + doc_type.getName() + "::planet:express"))), mfv(getMapType(doc_type)), fv1(getMapType(doc_type).getValueType()), fv2(getMapType(doc_type).getValueType()), - fv3(getMapType(doc_type).getValueType()), - fv4(getMapType(doc_type).getValueType()) + fv3(getMapType(doc_type).getValueType()) { fv1.setValue("title", StringFieldValue("fry")); fv1.setValue("rating", IntFieldValue(30)); @@ -756,9 +583,6 @@ Fixture::Fixture(const DocumentType &doc_type, const Keys &k) mfv.put(StringFieldValue(k.key3), fv3); doc->setValue("structmap", mfv); - - fv4.setValue("title", StringFieldValue("farnsworth")); - fv4.setValue("rating", IntFieldValue(48)); } } // namespace @@ -769,17 +593,14 @@ TEST_F(FieldPathUpdateTestCase, testAssignMap) Fixture f(*_foobar_type, k); DocumentUpdate docUp(*_repo, *_foobar_type, DocumentId("id:ns:foobar::barbar:foofoo")); - docUp.addFieldPathUpdate(std::make_unique<AssignFieldPathUpdate>(*f.doc->getDataType(), "structmap{" + k.key2 + "}", std::string(), f.fv4)); + docUp.addFieldPathUpdate(std::make_unique<AssignFieldPathUpdate>(*f.doc->getDataType(), "structmap{" + k.key2 + "}", std::string(), f.fv4())); docUp.applyTo(*f.doc); std::unique_ptr<MapFieldValue> valueNow = f.doc->getAs<MapFieldValue>(f.doc->getField("structmap")); ASSERT_EQ(std::size_t(3), valueNow->size()); - EXPECT_EQ(static_cast<FieldValue&>(f.fv1), - *valueNow->get(StringFieldValue(k.key1))); - EXPECT_EQ(static_cast<FieldValue&>(f.fv4), - *valueNow->get(StringFieldValue(k.key2))); - EXPECT_EQ(static_cast<FieldValue&>(f.fv3), - *valueNow->get(StringFieldValue(k.key3))); + EXPECT_EQ(static_cast<FieldValue&>(f.fv1), *valueNow->get(StringFieldValue(k.key1))); + EXPECT_EQ(*f.fv4(), *valueNow->get(StringFieldValue(k.key2))); + EXPECT_EQ(static_cast<FieldValue&>(f.fv3), *valueNow->get(StringFieldValue(k.key3))); } TEST_F(FieldPathUpdateTestCase, testAssignMapStruct) @@ -789,17 +610,14 @@ TEST_F(FieldPathUpdateTestCase, testAssignMapStruct) DocumentUpdate docUp(*_repo, *_foobar_type, DocumentId("id:ns:foobar::barbar:foofoo")); docUp.addFieldPathUpdate(std::make_unique<AssignFieldPathUpdate>(*f.doc->getDataType(), "structmap{" + k.key2 + "}.rating", - std::string(), IntFieldValue(48))); + std::string(), IntFieldValue::make(48))); docUp.applyTo(*f.doc); std::unique_ptr<MapFieldValue> valueNow = f.doc->getAs<MapFieldValue>(f.doc->getField("structmap")); ASSERT_EQ(std::size_t(3), valueNow->size()); - EXPECT_EQ(static_cast<FieldValue&>(f.fv1), - *valueNow->get(StringFieldValue(k.key1))); - EXPECT_EQ(static_cast<FieldValue&>(f.fv4), - *valueNow->get(StringFieldValue(k.key2))); - EXPECT_EQ(static_cast<FieldValue&>(f.fv3), - *valueNow->get(StringFieldValue(k.key3))); + EXPECT_EQ(static_cast<FieldValue&>(f.fv1), *valueNow->get(StringFieldValue(k.key1))); + EXPECT_EQ(*f.fv4(), *valueNow->get(StringFieldValue(k.key2))); + EXPECT_EQ(static_cast<FieldValue&>(f.fv3), *valueNow->get(StringFieldValue(k.key3))); } TEST_F(FieldPathUpdateTestCase, testAssignMapStructVariable) @@ -809,38 +627,39 @@ TEST_F(FieldPathUpdateTestCase, testAssignMapStructVariable) DocumentUpdate docUp(*_repo, *_foobar_type, DocumentId("id:ns:foobar::barbar:foofoo")); docUp.addFieldPathUpdate(std::make_unique<AssignFieldPathUpdate>(*f.doc->getDataType(), "structmap{$x}.rating", - "foobar.structmap{$x}.title == \"farnsworth\"", IntFieldValue(48))); + "foobar.structmap{$x}.title == \"farnsworth\"", IntFieldValue::make(48))); f.doc->setRepo(*_repo); docUp.applyTo(*f.doc); std::unique_ptr<MapFieldValue> valueNow = f.doc->getAs<MapFieldValue>(f.doc->getField("structmap")); ASSERT_EQ(std::size_t(3), valueNow->size()); - EXPECT_EQ(static_cast<FieldValue&>(f.fv1), - *valueNow->get(StringFieldValue(k.key1))); - EXPECT_EQ(static_cast<FieldValue&>(f.fv4), - *valueNow->get(StringFieldValue(k.key2))); - EXPECT_EQ(static_cast<FieldValue&>(f.fv3), - *valueNow->get(StringFieldValue(k.key3))); + EXPECT_EQ(static_cast<FieldValue&>(f.fv1), *valueNow->get(StringFieldValue(k.key1))); + EXPECT_EQ(*f.fv4(), *valueNow->get(StringFieldValue(k.key2))); + EXPECT_EQ(static_cast<FieldValue&>(f.fv3), *valueNow->get(StringFieldValue(k.key3))); } +std::unique_ptr<FieldValue> +createFry(const DataType & type) { + auto fv = std::make_unique<StructFieldValue>(type); + fv->setValue("title", StringFieldValue("fry")); + fv->setValue("rating", IntFieldValue(30)); + return fv; +} TEST_F(FieldPathUpdateTestCase, testAssignMapNoExist) { auto doc = std::make_unique<Document>(*_foobar_type, DocumentId("id:ns:foobar::planet:express")); MapFieldValue mfv(doc->getType().getField("structmap").getDataType()); - StructFieldValue fv1(dynamic_cast<const MapDataType&>(*mfv.getDataType()).getValueType()); - fv1.setValue("title", StringFieldValue("fry")); - fv1.setValue("rating", IntFieldValue(30)); - DocumentUpdate docUp(*_repo, *_foobar_type, DocumentId("id:ns:foobar::barbar:foofoo")); - docUp.addFieldPathUpdate(std::make_unique<AssignFieldPathUpdate>(*doc->getDataType(), "structmap{foo}", std::string(), fv1)); + docUp.addFieldPathUpdate(std::make_unique<AssignFieldPathUpdate>(*doc->getDataType(), "structmap{foo}", std::string(), + createFry(dynamic_cast<const MapDataType&>(*mfv.getDataType()).getValueType()))); //doc->print(std::cerr, true, ""); docUp.applyTo(*doc); //doc->print(std::cerr, true, ""); std::unique_ptr<MapFieldValue> valueNow = doc->getAs<MapFieldValue>(doc->getField("structmap")); ASSERT_EQ(std::size_t(1), valueNow->size()); - EXPECT_EQ(static_cast<FieldValue&>(fv1), *valueNow->get(StringFieldValue("foo"))); + EXPECT_EQ(*createFry(dynamic_cast<const MapDataType&>(*mfv.getDataType()).getValueType()), *valueNow->get(StringFieldValue("foo"))); } TEST_F(FieldPathUpdateTestCase, testAssignMapNoExistNoCreate) @@ -848,12 +667,9 @@ TEST_F(FieldPathUpdateTestCase, testAssignMapNoExistNoCreate) auto doc = std::make_unique<Document>(*_foobar_type, DocumentId("id:ns:foobar::planet:express")); MapFieldValue mfv(doc->getType().getField("structmap").getDataType()); - StructFieldValue fv1(dynamic_cast<const MapDataType&>(*mfv.getDataType()).getValueType()); - fv1.setValue("title", StringFieldValue("fry")); - fv1.setValue("rating", IntFieldValue(30)); - DocumentUpdate docUp(*_repo, *_foobar_type, DocumentId("id:ns:foobar::barbar:foofoo")); - auto assignUpdate = std::make_unique<AssignFieldPathUpdate>(*doc->getDataType(), "structmap{foo}", std::string(), fv1); + auto assignUpdate = std::make_unique<AssignFieldPathUpdate>(*doc->getDataType(), "structmap{foo}", std::string(), + createFry(dynamic_cast<const MapDataType&>(*mfv.getDataType()).getValueType())); static_cast<AssignFieldPathUpdate&>(*assignUpdate).setCreateMissingPath(false); docUp.addFieldPathUpdate(std::move(assignUpdate)); @@ -873,44 +689,50 @@ TEST_F(FieldPathUpdateTestCase, testQuotedStringKey) Fixture f(*_foobar_type, k); DocumentUpdate docUp(*_repo, *_foobar_type, DocumentId("id:ns:foobar::barbar:foofoo")); - docUp.addFieldPathUpdate(std::make_unique<AssignFieldPathUpdate>(*f.doc->getDataType(), field_path, std::string(), f.fv4)); + docUp.addFieldPathUpdate(std::make_unique<AssignFieldPathUpdate>(*f.doc->getDataType(), field_path, std::string(), f.fv4())); docUp.applyTo(*f.doc); std::unique_ptr<MapFieldValue> valueNow = f.doc->getAs<MapFieldValue>(f.doc->getField("structmap")); ASSERT_EQ(std::size_t(3), valueNow->size()); - EXPECT_EQ(static_cast<FieldValue&>(f.fv1), - *valueNow->get(StringFieldValue(k.key1))); - EXPECT_EQ(static_cast<FieldValue&>(f.fv4), - *valueNow->get(StringFieldValue(k.key2))); - EXPECT_EQ(static_cast<FieldValue&>(f.fv3), - *valueNow->get(StringFieldValue(k.key3))); + EXPECT_EQ(static_cast<FieldValue&>(f.fv1), *valueNow->get(StringFieldValue(k.key1))); + EXPECT_EQ(*f.fv4(), *valueNow->get(StringFieldValue(k.key2))); + EXPECT_EQ(static_cast<FieldValue&>(f.fv3), *valueNow->get(StringFieldValue(k.key3))); } +namespace { +std::unique_ptr<FieldValue> +createTastyCake(const DataType &type) { + auto fv = std::make_unique<StructFieldValue>(type); + fv->setValue("title", StringFieldValue("tasty cake")); + fv->setValue("rating", IntFieldValue(95)); + return fv; +} +} TEST_F(FieldPathUpdateTestCase, testEqualityComparison) { auto doc = std::make_unique<Document>(*_foobar_type, DocumentId("id:ns:foobar::foo:zoo")); MapFieldValue mfv(doc->getType().getField("structmap").getDataType()); - StructFieldValue fv4(dynamic_cast<const MapDataType&>(*mfv.getDataType()).getValueType()); - fv4.setValue("title", StringFieldValue("tasty cake")); - fv4.setValue("rating", IntFieldValue(95)); - { DocumentUpdate docUp1(*_repo, *_foobar_type, DocumentId("id:ns:foobar::barbar:foofoo")); DocumentUpdate docUp2(*_repo, *_foobar_type, DocumentId("id:ns:foobar::barbar:foofoo")); EXPECT_TRUE(docUp1 == docUp2); - docUp1.addFieldPathUpdate(std::make_unique<AssignFieldPathUpdate>(*doc->getDataType(), "structmap{here be dragons}", std::string(), fv4)); + docUp1.addFieldPathUpdate(std::make_unique<AssignFieldPathUpdate>(*doc->getDataType(), "structmap{here be dragons}", std::string(), + createTastyCake(dynamic_cast<const MapDataType&>(*mfv.getDataType()).getValueType()))); EXPECT_TRUE(docUp1 != docUp2); - docUp2.addFieldPathUpdate(std::make_unique<AssignFieldPathUpdate>(*doc->getDataType(), "structmap{here be dragons}", std::string(), fv4)); + docUp2.addFieldPathUpdate(std::make_unique<AssignFieldPathUpdate>(*doc->getDataType(), "structmap{here be dragons}", std::string(), + createTastyCake(dynamic_cast<const MapDataType&>(*mfv.getDataType()).getValueType()))); EXPECT_TRUE(docUp1 == docUp2); } { DocumentUpdate docUp1(*_repo, *_foobar_type, DocumentId("id:ns:foobar::barbar:foofoo")); DocumentUpdate docUp2(*_repo, *_foobar_type, DocumentId("id:ns:foobar::barbar:foofoo")); // where-clause diff - docUp1.addFieldPathUpdate(std::make_unique<AssignFieldPathUpdate>(*doc->getDataType(), "structmap{here be dragons}", std::string(), fv4)); - docUp2.addFieldPathUpdate(std::make_unique<AssignFieldPathUpdate>(*doc->getDataType(), "structmap{here be dragons}", "false", fv4)); + docUp1.addFieldPathUpdate(std::make_unique<AssignFieldPathUpdate>(*doc->getDataType(), "structmap{here be dragons}", std::string(), + createTastyCake(dynamic_cast<const MapDataType&>(*mfv.getDataType()).getValueType()))); + docUp2.addFieldPathUpdate(std::make_unique<AssignFieldPathUpdate>(*doc->getDataType(), "structmap{here be dragons}", "false", + createTastyCake(dynamic_cast<const MapDataType&>(*mfv.getDataType()).getValueType()))); EXPECT_TRUE(docUp1 != docUp2); } { @@ -918,8 +740,10 @@ TEST_F(FieldPathUpdateTestCase, testEqualityComparison) DocumentUpdate docUp2(*_repo, *_foobar_type, DocumentId("id:ns:foobar::barbar:foofoo")); // fieldpath diff - docUp1.addFieldPathUpdate(std::make_unique<AssignFieldPathUpdate>(*doc->getDataType(),"structmap{here be dragons}", std::string(), fv4)); - docUp2.addFieldPathUpdate(std::make_unique<AssignFieldPathUpdate>(*doc->getDataType(), "structmap{here be kittens}", std::string(), fv4)); + docUp1.addFieldPathUpdate(std::make_unique<AssignFieldPathUpdate>(*doc->getDataType(),"structmap{here be dragons}", std::string(), + createTastyCake(dynamic_cast<const MapDataType&>(*mfv.getDataType()).getValueType()))); + docUp2.addFieldPathUpdate(std::make_unique<AssignFieldPathUpdate>(*doc->getDataType(), "structmap{here be kittens}", std::string(), + createTastyCake(dynamic_cast<const MapDataType&>(*mfv.getDataType()).getValueType()))); EXPECT_TRUE(docUp1 != docUp2); } @@ -930,15 +754,15 @@ TEST_F(FieldPathUpdateTestCase, testAffectsDocumentBody) auto doc = std::make_unique<Document>(*_foobar_type, DocumentId("id:ns:foobar::things:stuff")); MapFieldValue mfv(doc->getType().getField("structmap").getDataType()); - StructFieldValue fv4(dynamic_cast<const MapDataType&>(*mfv.getDataType()).getValueType()); - fv4.setValue("title", StringFieldValue("scruffy")); - fv4.setValue("rating", IntFieldValue(90)); + auto fv4 = std::make_unique<StructFieldValue>(dynamic_cast<const MapDataType&>(*mfv.getDataType()).getValueType()); + fv4->setValue("title", StringFieldValue("scruffy")); + fv4->setValue("rating", IntFieldValue(90)); // structmap is body field { DocumentUpdate docUp(*_repo, *_foobar_type, DocumentId("id:ns:foobar::barbar:foofoo")); - auto update1 = std::make_unique<AssignFieldPathUpdate>(*doc->getDataType(), "structmap{janitor}", std::string(), fv4); + auto update1 = std::make_unique<AssignFieldPathUpdate>(*doc->getDataType(), "structmap{janitor}", std::string(), std::move(fv4)); static_cast<AssignFieldPathUpdate&>(*update1).setCreateMissingPath(true); docUp.addFieldPathUpdate(std::move(update1)); } @@ -946,7 +770,7 @@ TEST_F(FieldPathUpdateTestCase, testAffectsDocumentBody) // strfoo is header field { DocumentUpdate docUp(*_repo, *_foobar_type, DocumentId("id:ns:foobar::barbar:foofoo")); - auto update1 = std::make_unique<AssignFieldPathUpdate>(*doc->getDataType(), "strfoo", std::string(), StringFieldValue("helloworld")); + auto update1 = std::make_unique<AssignFieldPathUpdate>(*doc->getDataType(), "strfoo", std::string(), StringFieldValue::make("helloworld")); static_cast<AssignFieldPathUpdate&>(*update1).setCreateMissingPath(true); docUp.addFieldPathUpdate(std::move(update1)); } @@ -961,7 +785,7 @@ TEST_F(FieldPathUpdateTestCase, testIncompatibleDataTypeFails) DocumentUpdate docUp(*_repo, *_foobar_type, DocumentId("id:ns:foobar::barbar:foofoo")); try { - auto update1 = std::make_unique<AssignFieldPathUpdate>(*doc->getDataType(), "structmap{foo}", std::string(), StringFieldValue("bad things")); + auto update1 = std::make_unique<AssignFieldPathUpdate>(*doc->getDataType(), "structmap{foo}", std::string(), StringFieldValue::make("bad things")); EXPECT_TRUE(false); } catch (const vespalib::IllegalArgumentException& e) { // OK @@ -973,13 +797,13 @@ TEST_F(FieldPathUpdateTestCase, testSerializeAssign) auto doc = std::make_unique<Document>(*_foobar_type, DocumentId("id:ns:foobar::weloveto:serializestuff")); MapFieldValue mfv(doc->getType().getField("structmap").getDataType()); - StructFieldValue val(dynamic_cast<const MapDataType&>(*mfv.getDataType()).getValueType()); - val.setValue("title", StringFieldValue("cool frog")); - val.setValue("rating", IntFieldValue(100)); + auto val = std::make_unique<StructFieldValue>(dynamic_cast<const MapDataType&>(*mfv.getDataType()).getValueType()); + val->setValue("title", StringFieldValue("cool frog")); + val->setValue("rating", IntFieldValue(100)); DocumentUpdate docUp(*_repo, *_foobar_type, DocumentId("id:ns:foobar::barbar:foofoo")); - auto update1 = std::make_unique<AssignFieldPathUpdate>(*doc->getDataType(), "structmap{ribbit}", "true", val); + auto update1 = std::make_unique<AssignFieldPathUpdate>(*doc->getDataType(), "structmap{ribbit}", "true", std::move(val)); static_cast<AssignFieldPathUpdate&>(*update1).setCreateMissingPath(true); docUp.addFieldPathUpdate(std::move(update1)); @@ -991,14 +815,14 @@ TEST_F(FieldPathUpdateTestCase, testSerializeAdd) auto doc = std::make_unique<Document>(*_foobar_type, DocumentId("id:ns:foobar::george:costanza")); EXPECT_TRUE(doc->hasValue("strarray") == false); - ArrayFieldValue adds(doc->getType().getField("strarray").getDataType()); - adds.add(StringFieldValue("serenity now")); - adds.add(StringFieldValue("a festivus for the rest of us")); - adds.add(StringFieldValue("george is getting upset!")); + auto adds = std::make_unique<ArrayFieldValue>(doc->getType().getField("strarray").getDataType()); + adds->add(StringFieldValue("serenity now")); + adds->add(StringFieldValue("a festivus for the rest of us")); + adds->add(StringFieldValue("george is getting upset!")); DocumentUpdate docUp(*_repo, *_foobar_type, DocumentId("id:ns:foobar::barbar:foofoo")); - docUp.addFieldPathUpdate(std::make_unique<AddFieldPathUpdate>(*doc->getDataType(), "strarray", std::string(), adds)); + docUp.addFieldPathUpdate(std::make_unique<AddFieldPathUpdate>(*doc->getDataType(), "strarray", std::string(), std::move(adds))); testSerialize(*_repo, docUp); } @@ -1037,11 +861,11 @@ FieldPathUpdateTestCase::createDocumentUpdateForSerialization(const DocumentType static_cast<AssignFieldPathUpdate&>(*assign).setCreateMissingPath(false); docUp->addFieldPathUpdate(std::move(assign)); - ArrayFieldValue fArray(docType->getField("arrayoffloatfield").getDataType()); - fArray.add(FloatFieldValue(12.0)); - fArray.add(FloatFieldValue(5.0)); + auto fArray = std::make_unique<ArrayFieldValue>(docType->getField("arrayoffloatfield").getDataType()); + fArray->add(FloatFieldValue(12.0)); + fArray->add(FloatFieldValue(5.0)); - docUp->addFieldPathUpdate(std::make_unique<AddFieldPathUpdate>(*docType, "arrayoffloatfield", "", fArray)); + docUp->addFieldPathUpdate(std::make_unique<AddFieldPathUpdate>(*docType, "arrayoffloatfield", "", std::move(fArray))); docUp->addFieldPathUpdate(std::make_unique<RemoveFieldPathUpdate>("intfield", "serializetest.intfield > 0")); return docUp; @@ -1098,7 +922,7 @@ TEST_F(FieldPathUpdateTestCase, array_element_update_for_invalid_index_is_ignore doc->setValue("strarray", str_array); DocumentUpdate docUp(*_repo, *_foobar_type, DocumentId("id::foobar::1")); - docUp.addFieldPathUpdate(std::make_unique<AssignFieldPathUpdate>(*doc->getDataType(), "strarray[1]", "", StringFieldValue("george"))); + docUp.addFieldPathUpdate(std::make_unique<AssignFieldPathUpdate>(*doc->getDataType(), "strarray[1]", "", StringFieldValue::make("george"))); docUp.applyTo(*doc); // Doc is unmodified. diff --git a/document/src/vespa/document/update/addfieldpathupdate.cpp b/document/src/vespa/document/update/addfieldpathupdate.cpp index bd06451759e..f41110fd4c1 100644 --- a/document/src/vespa/document/update/addfieldpathupdate.cpp +++ b/document/src/vespa/document/update/addfieldpathupdate.cpp @@ -17,27 +17,21 @@ namespace document { using namespace fieldvalue; using vespalib::make_string; -IMPLEMENT_IDENTIFIABLE(AddFieldPathUpdate, FieldPathUpdate); - AddFieldPathUpdate::AddFieldPathUpdate(const DataType& type, stringref fieldPath, - stringref whereClause, const ArrayFieldValue& values) - : FieldPathUpdate(fieldPath, whereClause), - _values(vespalib::CloneablePtr<ArrayFieldValue>(values.clone())) + stringref whereClause, std::unique_ptr<ArrayFieldValue> values) + : FieldPathUpdate(Add, fieldPath, whereClause), + _values(std::move(values)) { checkCompatibility(*_values, type); } AddFieldPathUpdate::AddFieldPathUpdate() - : FieldPathUpdate(), _values() + : FieldPathUpdate(Add), + _values() { } AddFieldPathUpdate::~AddFieldPathUpdate() = default; -FieldPathUpdate* -AddFieldPathUpdate::clone() const { - return new AddFieldPathUpdate(*this); -} - namespace { class AddIteratorHandler : public fieldvalue::IteratorHandler { @@ -54,7 +48,7 @@ private: ModificationStatus AddIteratorHandler::doModify(FieldValue &fv) { if (fv.isCollection()) { - CollectionFieldValue &cf = static_cast<CollectionFieldValue &>(fv); + auto &cf = static_cast<CollectionFieldValue &>(fv); for (std::size_t i = 0; i < _values.size(); ++i) { cf.add(_values[i]); } @@ -70,10 +64,8 @@ AddIteratorHandler::doModify(FieldValue &fv) { bool AddFieldPathUpdate::operator==(const FieldPathUpdate& other) const { - if (other.getClass().id() != AddFieldPathUpdate::classId) return false; if (!FieldPathUpdate::operator==(other)) return false; - const AddFieldPathUpdate& addOther - = static_cast<const AddFieldPathUpdate&>(other); + const auto & addOther = static_cast<const AddFieldPathUpdate&>(other); return *addOther._values == *_values; } diff --git a/document/src/vespa/document/update/addfieldpathupdate.h b/document/src/vespa/document/update/addfieldpathupdate.h index 2d1a459e0e5..a226e95c789 100644 --- a/document/src/vespa/document/update/addfieldpathupdate.h +++ b/document/src/vespa/document/update/addfieldpathupdate.h @@ -7,30 +7,30 @@ namespace document { class ArrayFieldValue; -class AddFieldPathUpdate : public FieldPathUpdate +class AddFieldPathUpdate final : public FieldPathUpdate { public: /** For deserialization */ AddFieldPathUpdate(); AddFieldPathUpdate(const DataType& type, stringref fieldPath, - stringref whereClause, const ArrayFieldValue& values); + stringref whereClause, std::unique_ptr<ArrayFieldValue> values); + AddFieldPathUpdate(AddFieldPathUpdate &&) noexcept = default; + AddFieldPathUpdate & operator =(AddFieldPathUpdate &&) noexcept = default; + AddFieldPathUpdate(const AddFieldPathUpdate &) = delete; + AddFieldPathUpdate & operator =( const AddFieldPathUpdate &) = delete; ~AddFieldPathUpdate(); - FieldPathUpdate* clone() const override; bool operator==(const FieldPathUpdate& other) const override; void print(std::ostream& out, bool verbose, const std::string& indent) const override; const ArrayFieldValue & getValues() const { return *_values; } - DECLARE_IDENTIFIABLE(AddFieldPathUpdate); ACCEPT_UPDATE_VISITOR; - private: uint8_t getSerializedType() const override { return AddMagic; } void deserialize(const DocumentTypeRepo& repo, const DataType& type, nbostream & stream) override; - std::unique_ptr<fieldvalue::IteratorHandler> getIteratorHandler(Document &, const DocumentTypeRepo &) const override; - vespalib::CloneablePtr<ArrayFieldValue> _values; + std::unique_ptr<ArrayFieldValue> _values; }; } diff --git a/document/src/vespa/document/update/assignfieldpathupdate.cpp b/document/src/vespa/document/update/assignfieldpathupdate.cpp index d4cbff2aae9..649b1c1b6b8 100644 --- a/document/src/vespa/document/update/assignfieldpathupdate.cpp +++ b/document/src/vespa/document/update/assignfieldpathupdate.cpp @@ -19,10 +19,8 @@ namespace document { using namespace fieldvalue; -IMPLEMENT_IDENTIFIABLE(AssignFieldPathUpdate, FieldPathUpdate); - AssignFieldPathUpdate::AssignFieldPathUpdate() - : FieldPathUpdate(), + : FieldPathUpdate(Assign), _newValue(), _expression(), _removeIfZero(false), @@ -34,9 +32,9 @@ AssignFieldPathUpdate::AssignFieldPathUpdate( const DataType& type, stringref fieldPath, stringref whereClause, - const FieldValue& newValue) - : FieldPathUpdate(fieldPath, whereClause), - _newValue(newValue.clone()), + std::unique_ptr<FieldValue> newValue) + : FieldPathUpdate(Assign, fieldPath, whereClause), + _newValue(std::move(newValue)), _expression(), _removeIfZero(false), _createMissingPath(true) @@ -45,7 +43,7 @@ AssignFieldPathUpdate::AssignFieldPathUpdate( } AssignFieldPathUpdate::AssignFieldPathUpdate(stringref fieldPath, stringref whereClause, stringref expression) - : FieldPathUpdate(fieldPath, whereClause), + : FieldPathUpdate(Assign, fieldPath, whereClause), _newValue(), _expression(expression), _removeIfZero(false), @@ -57,12 +55,8 @@ AssignFieldPathUpdate::AssignFieldPathUpdate(stringref fieldPath, stringref wher } } -AssignFieldPathUpdate::~AssignFieldPathUpdate() { } +AssignFieldPathUpdate::~AssignFieldPathUpdate() = default; -FieldPathUpdate* -AssignFieldPathUpdate::clone() const { - return new AssignFieldPathUpdate(*this); -} namespace { class AssignValueIteratorHandler : public IteratorHandler @@ -186,10 +180,8 @@ AssignFieldPathUpdate::getIteratorHandler(Document& doc, const DocumentTypeRepo bool AssignFieldPathUpdate::operator==(const FieldPathUpdate& other) const { - if (other.getClass().id() != AssignFieldPathUpdate::classId) return false; if (!FieldPathUpdate::operator==(other)) return false; - const AssignFieldPathUpdate& assignOther - = static_cast<const AssignFieldPathUpdate&>(other); + const auto & assignOther = static_cast<const AssignFieldPathUpdate&>(other); if (assignOther._newValue.get() && _newValue.get()) { if (*assignOther._newValue != *_newValue) return false; } diff --git a/document/src/vespa/document/update/assignfieldpathupdate.h b/document/src/vespa/document/update/assignfieldpathupdate.h index 9cbd4087e0f..8463c56d775 100644 --- a/document/src/vespa/document/update/assignfieldpathupdate.h +++ b/document/src/vespa/document/update/assignfieldpathupdate.h @@ -6,7 +6,7 @@ namespace document { -class AssignFieldPathUpdate : public FieldPathUpdate +class AssignFieldPathUpdate final : public FieldPathUpdate { public: enum SerializationFlag @@ -18,8 +18,11 @@ public: /** For deserialization */ AssignFieldPathUpdate(); - - AssignFieldPathUpdate(const DataType& type, stringref fieldPath, stringref whereClause, const FieldValue& newValue); + AssignFieldPathUpdate(AssignFieldPathUpdate &&) noexcept = default; + AssignFieldPathUpdate & operator =(AssignFieldPathUpdate &&) noexcept = default; + AssignFieldPathUpdate(const AssignFieldPathUpdate &) = delete; + AssignFieldPathUpdate & operator =(const AssignFieldPathUpdate &) = delete; + AssignFieldPathUpdate(const DataType& type, stringref fieldPath, stringref whereClause, std::unique_ptr<FieldValue> newValue); AssignFieldPathUpdate(stringref fieldPath, stringref whereClause, stringref expression); ~AssignFieldPathUpdate(); @@ -32,26 +35,23 @@ public: } bool getCreateMissingPath() const { return _createMissingPath; } const vespalib::string& getExpression() const { return _expression; } - bool hasValue() const { return _newValue.get() != nullptr; } + bool hasValue() const { return bool(_newValue); } const FieldValue & getValue() const { return *_newValue; } - FieldPathUpdate* clone() const override; bool operator==(const FieldPathUpdate& other) const override; void print(std::ostream& out, bool verbose, const std::string& indent) const override; - DECLARE_IDENTIFIABLE(AssignFieldPathUpdate); ACCEPT_UPDATE_VISITOR; - private: uint8_t getSerializedType() const override { return AssignMagic; } void deserialize(const DocumentTypeRepo& repo, const DataType& type, nbostream & stream) override; std::unique_ptr<fieldvalue::IteratorHandler> getIteratorHandler(Document& doc, const DocumentTypeRepo & repo) const override; - vespalib::CloneablePtr<FieldValue> _newValue; - vespalib::string _expression; - bool _removeIfZero; - bool _createMissingPath; + std::unique_ptr<FieldValue> _newValue; + vespalib::string _expression; + bool _removeIfZero; + bool _createMissingPath; }; } diff --git a/document/src/vespa/document/update/fieldpathupdate.cpp b/document/src/vespa/document/update/fieldpathupdate.cpp index 5b47b48dd57..a8a42f69215 100644 --- a/document/src/vespa/document/update/fieldpathupdate.cpp +++ b/document/src/vespa/document/update/fieldpathupdate.cpp @@ -21,8 +21,6 @@ namespace document { using namespace fieldvalue; -IMPLEMENT_IDENTIFIABLE_ABSTRACT(FieldPathUpdate, Identifiable); - namespace { std::unique_ptr<select::Node> @@ -40,17 +38,19 @@ parseDocumentSelection(vespalib::stringref query, const DocumentTypeRepo& repo) } // namespace -FieldPathUpdate::FieldPathUpdate() : - _originalFieldPath(), - _originalWhereClause() +FieldPathUpdate::FieldPathUpdate(FieldPathUpdateType type) + : _type(type), + _originalFieldPath(), + _originalWhereClause() { } FieldPathUpdate::FieldPathUpdate(const FieldPathUpdate &) = default; FieldPathUpdate & FieldPathUpdate::operator =(const FieldPathUpdate &) = default; -FieldPathUpdate::FieldPathUpdate(stringref fieldPath, stringref whereClause) : - _originalFieldPath(fieldPath), - _originalWhereClause(whereClause) +FieldPathUpdate::FieldPathUpdate(FieldPathUpdateType type, stringref fieldPath, stringref whereClause) + : _type(type), + _originalFieldPath(fieldPath), + _originalWhereClause(whereClause) { } FieldPathUpdate::~FieldPathUpdate() = default; @@ -58,7 +58,8 @@ FieldPathUpdate::~FieldPathUpdate() = default; bool FieldPathUpdate::operator==(const FieldPathUpdate& other) const { - return (other._originalFieldPath == _originalFieldPath) + return (_type == other._type) + && (other._originalFieldPath == _originalFieldPath) && (other._originalWhereClause == _originalWhereClause); } diff --git a/document/src/vespa/document/update/fieldpathupdate.h b/document/src/vespa/document/update/fieldpathupdate.h index 327493407a1..fb205b67a60 100644 --- a/document/src/vespa/document/update/fieldpathupdate.h +++ b/document/src/vespa/document/update/fieldpathupdate.h @@ -17,19 +17,11 @@ class DataType; namespace select { class Node; } namespace fieldvalue { class IteratorHandler; } -class FieldPathUpdate : public vespalib::Identifiable +class FieldPathUpdate { -protected: +public: using nbostream = vespalib::nbostream; using stringref = vespalib::stringref; - /** To be used for deserialization */ - FieldPathUpdate(); - FieldPathUpdate(const FieldPathUpdate &); - FieldPathUpdate & operator =(const FieldPathUpdate &); - - static stringref getString(nbostream & stream); -public: - ~FieldPathUpdate() override; enum FieldPathUpdateType { Add = IDENTIFIABLE_CLASSID(AddFieldPathUpdate), @@ -37,9 +29,10 @@ public: Remove = IDENTIFIABLE_CLASSID(RemoveFieldPathUpdate) }; - void applyTo(Document& doc) const; + virtual ~FieldPathUpdate(); - virtual FieldPathUpdate* clone() const = 0; + FieldPathUpdateType type() const noexcept { return _type; } + void applyTo(Document& doc) const; virtual bool operator==(const FieldPathUpdate& other) const; bool operator!=(const FieldPathUpdate& other) const { @@ -58,8 +51,6 @@ public: virtual void print(std::ostream& out, bool verbose, const std::string& indent) const = 0; - DECLARE_IDENTIFIABLE_ABSTRACT(FieldPathUpdate); - virtual void accept(UpdateVisitor &visitor) const = 0; virtual uint8_t getSerializedType() const = 0; @@ -69,7 +60,13 @@ public: static std::unique_ptr<FieldPathUpdate> createInstance(const DocumentTypeRepo& repo, const DataType &type, nbostream & stream); protected: - FieldPathUpdate(stringref fieldPath, stringref whereClause = stringref()); + /** To be used for deserialization */ + FieldPathUpdate(FieldPathUpdateType type); + FieldPathUpdate(const FieldPathUpdate &); + FieldPathUpdate & operator =(const FieldPathUpdate &); + + static stringref getString(nbostream & stream); + FieldPathUpdate(FieldPathUpdateType type, stringref fieldPath, stringref whereClause = stringref()); virtual void deserialize(const DocumentTypeRepo& repo, const DataType& type, nbostream & stream); @@ -80,8 +77,9 @@ private: // TODO: rename to createIteratorHandler? virtual std::unique_ptr<fieldvalue::IteratorHandler> getIteratorHandler(Document& doc, const DocumentTypeRepo & repo) const = 0; - vespalib::string _originalFieldPath; - vespalib::string _originalWhereClause; + FieldPathUpdateType _type; + vespalib::string _originalFieldPath; + vespalib::string _originalWhereClause; }; } diff --git a/document/src/vespa/document/update/removefieldpathupdate.cpp b/document/src/vespa/document/update/removefieldpathupdate.cpp index 325202589cd..1b94039240b 100644 --- a/document/src/vespa/document/update/removefieldpathupdate.cpp +++ b/document/src/vespa/document/update/removefieldpathupdate.cpp @@ -8,23 +8,14 @@ namespace document { using namespace fieldvalue; -IMPLEMENT_IDENTIFIABLE(RemoveFieldPathUpdate, FieldPathUpdate); - RemoveFieldPathUpdate::RemoveFieldPathUpdate() - : FieldPathUpdate() + : FieldPathUpdate(Remove) { } RemoveFieldPathUpdate::RemoveFieldPathUpdate(stringref fieldPath, stringref whereClause) - : FieldPathUpdate(fieldPath, whereClause) -{ -} - -bool -RemoveFieldPathUpdate::operator==(const FieldPathUpdate& other) const + : FieldPathUpdate(Remove, fieldPath, whereClause) { - if (other.getClass().id() != RemoveFieldPathUpdate::classId) return false; - return FieldPathUpdate::operator==(other); } void diff --git a/document/src/vespa/document/update/removefieldpathupdate.h b/document/src/vespa/document/update/removefieldpathupdate.h index 05e0c8d88ca..df71fcd717c 100644 --- a/document/src/vespa/document/update/removefieldpathupdate.h +++ b/document/src/vespa/document/update/removefieldpathupdate.h @@ -5,22 +5,20 @@ namespace document { -class RemoveFieldPathUpdate : public FieldPathUpdate +class RemoveFieldPathUpdate final : public FieldPathUpdate { public: /** For deserialization */ RemoveFieldPathUpdate(); - + RemoveFieldPathUpdate(RemoveFieldPathUpdate &&) noexcept = default; + RemoveFieldPathUpdate & operator =(RemoveFieldPathUpdate &&) noexcept = default; + RemoveFieldPathUpdate(const RemoveFieldPathUpdate &) = delete; + RemoveFieldPathUpdate & operator =(const RemoveFieldPathUpdate &) = delete; RemoveFieldPathUpdate(stringref fieldPath, stringref whereClause = stringref()); - FieldPathUpdate* clone() const override { return new RemoveFieldPathUpdate(*this); } - - bool operator==(const FieldPathUpdate& other) const override; void print(std::ostream& out, bool verbose, const std::string& indent) const override; - DECLARE_IDENTIFIABLE(RemoveFieldPathUpdate); ACCEPT_UPDATE_VISITOR; - private: uint8_t getSerializedType() const override { return RemoveMagic; } void deserialize(const DocumentTypeRepo& repo, const DataType& type, nbostream & buffer) override; diff --git a/documentapi/abi-spec.json b/documentapi/abi-spec.json index 20043450501..49bdf32bfb1 100644 --- a/documentapi/abi-spec.json +++ b/documentapi/abi-spec.json @@ -484,12 +484,16 @@ "public" ], "methods": [ + "public void <init>()", "public void <init>(long)", "public void <init>(com.yahoo.documentapi.Result$ResultType, java.lang.Error)", + "public void <init>(com.yahoo.documentapi.Result$ResultType, com.yahoo.messagebus.Error)", "public boolean isSuccess()", "public java.lang.Error getError()", + "public com.yahoo.messagebus.Error error()", "public long getRequestId()", - "public com.yahoo.documentapi.Result$ResultType type()" + "public com.yahoo.documentapi.Result$ResultType type()", + "public static com.yahoo.messagebus.Error toError(com.yahoo.documentapi.Result$ResultType)" ], "fields": [] }, diff --git a/documentapi/src/main/java/com/yahoo/documentapi/Result.java b/documentapi/src/main/java/com/yahoo/documentapi/Result.java index 9509a485654..d4426ed8c9d 100644 --- a/documentapi/src/main/java/com/yahoo/documentapi/Result.java +++ b/documentapi/src/main/java/com/yahoo/documentapi/Result.java @@ -1,6 +1,9 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.documentapi; +import com.yahoo.messagebus.Error; +import com.yahoo.messagebus.ErrorCode; + /** * The <i>synchronous</i> result of submitting an asynchronous operation. * A result is either a success or not. If it is not a success, it will contain an explanation of why. @@ -11,25 +14,43 @@ package com.yahoo.documentapi; public class Result { /** Null if this is a success, set to the error occurring if this is a failure */ - private Error error = null; + private final Error error; /** The id of this operation */ - private long requestId; + private final long requestId; + + private final ResultType type; - private ResultType type = ResultType.SUCCESS; + /** Creates a successful result with requestId zero */ + public Result() { + this(0); + } /** * Creates a successful result * * @param requestId the ID of the request */ public Result(long requestId) { + this.error = null; this.requestId = requestId; + type = ResultType.SUCCESS; } /** * Creates a unsuccessful result * + * @deprecated Will be removed on Vespa 8 due to incorrect java.lang.Error + */ + @Deprecated(forRemoval = true, since="7") + public Result(ResultType type, java.lang.Error error) { + this.type = type; + this.error = new Error(0, error.getMessage()); + this.requestId = 0; + } + /** + * Creates a unsuccessful result + * * @param type the type of failure * @param error the error to encapsulate in this Result * @see com.yahoo.documentapi.Result.ResultType @@ -37,6 +58,7 @@ public class Result { public Result(ResultType type, Error error) { this.type = type; this.error = error; + this.requestId = 0; } /** @@ -54,8 +76,12 @@ public class Result { * If this was a success, this method returns null. * * @return the Error, or null + * @deprecated Will be removed on Vespa 8 */ - public Error getError() { return error; } + @Deprecated(forRemoval = true, since="7") + public java.lang.Error getError() { return new java.lang.Error(error.getMessage()); } + + public Error error() { return error; } /** * Returns the id of this operation. The asynchronous response to this operation @@ -85,5 +111,16 @@ public class Result { @Deprecated(since = "7", forRemoval = true) // TODO: Remove on Vespa 8 — this is a Response outcome, not a Result outcome. CONDITION_NOT_MET_ERROR } + public static Error toError(ResultType result) { + switch (result) { + case TRANSIENT_ERROR: + return new Error(ErrorCode.TRANSIENT_ERROR, ResultType.TRANSIENT_ERROR.name()); + case CONDITION_NOT_MET_ERROR: + return new Error(ErrorCode.FATAL_ERROR, ResultType.CONDITION_NOT_MET_ERROR.name()); + case FATAL_ERROR: + return new Error(ErrorCode.FATAL_ERROR, ResultType.FATAL_ERROR.name()); + } + return new Error(ErrorCode.NONE, "SUCCESS"); + } } diff --git a/documentapi/src/main/java/com/yahoo/documentapi/local/LocalAsyncSession.java b/documentapi/src/main/java/com/yahoo/documentapi/local/LocalAsyncSession.java index 102fa73f2ec..591b4c80436 100644 --- a/documentapi/src/main/java/com/yahoo/documentapi/local/LocalAsyncSession.java +++ b/documentapi/src/main/java/com/yahoo/documentapi/local/LocalAsyncSession.java @@ -45,8 +45,8 @@ public class LocalAsyncSession implements AsyncSession { private final Executor executor = Executors.newCachedThreadPool(); private final AtomicReference<Phaser> phaser; - private AtomicLong requestId = new AtomicLong(0); - private AtomicReference<Result.ResultType> result = new AtomicReference<>(SUCCESS); + private final AtomicLong requestId = new AtomicLong(0); + private final AtomicReference<Result.ResultType> result = new AtomicReference<>(SUCCESS); public LocalAsyncSession(AsyncParameters params, LocalDocumentAccess access) { this.handler = params.getResponseHandler(); @@ -163,7 +163,7 @@ public class LocalAsyncSession implements AsyncSession { private Result send(Function<Long, Response> responses, DocumentOperationParameters parameters) { Result.ResultType resultType = result.get(); if (resultType != SUCCESS) - return new Result(resultType, new Error()); + return new Result(resultType, Result.toError(resultType)); ResponseHandler responseHandler = parameters.responseHandler().orElse(this::addResponse); long req = requestId.incrementAndGet(); diff --git a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/MessageBusAsyncSession.java b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/MessageBusAsyncSession.java index 279e04c43b4..8809e05caf3 100644 --- a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/MessageBusAsyncSession.java +++ b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/MessageBusAsyncSession.java @@ -26,6 +26,7 @@ import com.yahoo.documentapi.messagebus.protocol.RemoveDocumentMessage; import com.yahoo.documentapi.messagebus.protocol.RemoveDocumentReply; import com.yahoo.documentapi.messagebus.protocol.UpdateDocumentMessage; import com.yahoo.documentapi.messagebus.protocol.UpdateDocumentReply; +import com.yahoo.messagebus.Error; import com.yahoo.messagebus.ErrorCode; import com.yahoo.messagebus.Message; import com.yahoo.messagebus.MessageBus; @@ -187,7 +188,7 @@ public class MessageBusAsyncSession implements MessageBusSession, AsyncSession { return toResult(reqId, session.send(msg)); } } catch (Exception e) { - return new Result(Result.ResultType.FATAL_ERROR, new Error(e.getMessage(), e)); + return new Result(Result.ResultType.FATAL_ERROR, new Error(ErrorCode.FATAL_ERROR, e.toString())); } } @@ -285,8 +286,7 @@ public class MessageBusAsyncSession implements MessageBusSession, AsyncSession { return new Result(reqId); } return new Result( - messageBusErrorToResultType(mbusResult.getError().getCode()), - new Error(mbusResult.getError().getMessage() + " (" + mbusResult.getError().getCode() + ")")); + messageBusErrorToResultType(mbusResult.getError().getCode()), mbusResult.getError()); } private static Response.Outcome toOutcome(Reply reply) { diff --git a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/MessageBusSyncSession.java b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/MessageBusSyncSession.java index 318b518f44e..1b0a6db3d53 100755 --- a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/MessageBusSyncSession.java +++ b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/MessageBusSyncSession.java @@ -112,7 +112,7 @@ public class MessageBusSyncSession implements MessageBusSession, SyncSession, Re Thread.sleep(100); } if (!result.isSuccess()) { - throw new DocumentAccessException(result.getError().toString()); + throw new DocumentAccessException(result.error().toString()); } return monitor.waitForReply(); } catch (InterruptedException e) { diff --git a/documentapi/src/test/java/com/yahoo/documentapi/local/LocalDocumentApiTestCase.java b/documentapi/src/test/java/com/yahoo/documentapi/local/LocalDocumentApiTestCase.java index 3a748ca173a..69fa32986f2 100644 --- a/documentapi/src/test/java/com/yahoo/documentapi/local/LocalDocumentApiTestCase.java +++ b/documentapi/src/test/java/com/yahoo/documentapi/local/LocalDocumentApiTestCase.java @@ -111,7 +111,7 @@ public class LocalDocumentApiTestCase extends AbstractDocumentApiTestCase { for (DocumentId id : ids) { Result result = session.get(id); if ( ! result.isSuccess()) - throw new IllegalStateException("Failed requesting document " + id, result.getError().getCause()); + throw new IllegalStateException("Failed requesting document " + id + ": " + result.error().toString()); outstandingRequests.add(result.getRequestId()); } diff --git a/documentapi/src/test/java/com/yahoo/documentapi/messagebus/Destination.java b/documentapi/src/test/java/com/yahoo/documentapi/messagebus/Destination.java index e08d9b14ee5..0b2664a75ee 100644 --- a/documentapi/src/test/java/com/yahoo/documentapi/messagebus/Destination.java +++ b/documentapi/src/test/java/com/yahoo/documentapi/messagebus/Destination.java @@ -4,9 +4,11 @@ package com.yahoo.documentapi.messagebus; import com.yahoo.document.DocumentRemove; import com.yahoo.documentapi.DocumentAccess; import com.yahoo.documentapi.DocumentAccessParams; +import com.yahoo.documentapi.ProgressToken; import com.yahoo.documentapi.SyncParameters; import com.yahoo.documentapi.SyncSession; import com.yahoo.documentapi.local.LocalDocumentAccess; +import com.yahoo.documentapi.messagebus.protocol.CreateVisitorReply; import com.yahoo.documentapi.messagebus.protocol.DocumentMessage; import com.yahoo.documentapi.messagebus.protocol.DocumentProtocol; import com.yahoo.documentapi.messagebus.protocol.GetDocumentMessage; @@ -20,21 +22,23 @@ import com.yahoo.messagebus.Error; import com.yahoo.messagebus.ErrorCode; import com.yahoo.messagebus.Message; import com.yahoo.messagebus.MessageHandler; -import com.yahoo.messagebus.Protocol; import com.yahoo.messagebus.RPCMessageBus; import com.yahoo.messagebus.Reply; import com.yahoo.messagebus.network.Identity; import com.yahoo.messagebus.network.rpc.RPCNetworkParams; -import java.util.Arrays; +import java.util.List; +import java.util.concurrent.atomic.AtomicBoolean; /** * Mock-up destination used for testing. * - * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a> + * @author Einar M R Rosenvinge */ public class Destination implements MessageHandler { + final AtomicBoolean discard = new AtomicBoolean(); + private final DestinationSession session; private final DocumentAccess access; private final SyncSession local; @@ -46,7 +50,7 @@ public class Destination implements MessageHandler { params.setDocumentManagerConfigId(documentManagerConfigId); access = new LocalDocumentAccess(params); local = access.createSyncSession(new SyncParameters.Builder().build()); - bus = new RPCMessageBus(Arrays.asList((Protocol)new DocumentProtocol(access.getDocumentTypeManager())), + bus = new RPCMessageBus(List.of(new DocumentProtocol(access.getDocumentTypeManager())), new RPCNetworkParams().setNumNetworkThreads(1) .setIdentity(new Identity("test/destination")) .setSlobrokConfigId(slobrokConfigId), @@ -54,33 +58,38 @@ public class Destination implements MessageHandler { session = bus.getMessageBus().createDestinationSession("session", true, this); } - protected void sendReply(Reply reply) { - session.reply(reply); - } - public void handleMessage(Message msg) { + if (discard.get()) { + msg.discard(); + return; + } + Reply reply = ((DocumentMessage)msg).createReply(); try { switch (msg.getType()) { - case DocumentProtocol.MESSAGE_GETDOCUMENT: - reply = new GetDocumentReply(local.get(((GetDocumentMessage)msg).getDocumentId())); - break; + case DocumentProtocol.MESSAGE_GETDOCUMENT: + reply = new GetDocumentReply(local.get(((GetDocumentMessage)msg).getDocumentId())); + break; + + case DocumentProtocol.MESSAGE_PUTDOCUMENT: + local.put(((PutDocumentMessage)msg).getDocumentPut()); + break; - case DocumentProtocol.MESSAGE_PUTDOCUMENT: - local.put(((PutDocumentMessage)msg).getDocumentPut()); - break; + case DocumentProtocol.MESSAGE_REMOVEDOCUMENT: + local.remove(new DocumentRemove(((RemoveDocumentMessage)msg).getDocumentId())); + break; - case DocumentProtocol.MESSAGE_REMOVEDOCUMENT: - local.remove(new DocumentRemove(((RemoveDocumentMessage)msg).getDocumentId())); - break; + case DocumentProtocol.MESSAGE_UPDATEDOCUMENT: + local.update(((UpdateDocumentMessage)msg).getDocumentUpdate()); + break; - case DocumentProtocol.MESSAGE_UPDATEDOCUMENT: - local.update(((UpdateDocumentMessage)msg).getDocumentUpdate()); - break; + case DocumentProtocol.MESSAGE_CREATEVISITOR: + ((CreateVisitorReply) reply).setLastBucket(ProgressToken.FINISHED_BUCKET); + break; - default: - throw new UnsupportedOperationException("Unsupported message type '" + msg.getType() + "'."); + default: + throw new UnsupportedOperationException("Unsupported message type '" + msg.getType() + "'."); } } catch (Exception e) { reply = new EmptyReply(); diff --git a/documentapi/src/test/java/com/yahoo/documentapi/messagebus/MessageBusDocumentApiTestCase.java b/documentapi/src/test/java/com/yahoo/documentapi/messagebus/MessageBusDocumentApiTestCase.java index db7ab0ea238..d1fbbd74795 100644 --- a/documentapi/src/test/java/com/yahoo/documentapi/messagebus/MessageBusDocumentApiTestCase.java +++ b/documentapi/src/test/java/com/yahoo/documentapi/messagebus/MessageBusDocumentApiTestCase.java @@ -10,20 +10,15 @@ import com.yahoo.documentapi.AsyncParameters; import com.yahoo.documentapi.AsyncSession; import com.yahoo.documentapi.DocumentAccess; import com.yahoo.documentapi.DocumentOperationParameters; -import com.yahoo.documentapi.ProgressToken; import com.yahoo.documentapi.Response; import com.yahoo.documentapi.VisitorParameters; import com.yahoo.documentapi.VisitorSession; -import com.yahoo.documentapi.messagebus.protocol.CreateVisitorReply; -import com.yahoo.documentapi.messagebus.protocol.DocumentMessage; -import com.yahoo.documentapi.messagebus.protocol.DocumentProtocol; + import com.yahoo.documentapi.test.AbstractDocumentApiTestCase; import com.yahoo.jrt.ListenFailedException; import com.yahoo.jrt.slobrok.server.Slobrok; import com.yahoo.messagebus.AllPassThrottlePolicy; import com.yahoo.messagebus.DynamicThrottlePolicy; -import com.yahoo.messagebus.Message; -import com.yahoo.messagebus.Reply; import com.yahoo.messagebus.ThrottlePolicy; import com.yahoo.messagebus.network.Identity; import org.junit.After; @@ -36,7 +31,6 @@ import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicReference; import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertSame; import static org.junit.Assert.assertTrue; @@ -70,7 +64,7 @@ public class MessageBusDocumentApiTestCase extends AbstractDocumentApiTestCase { params.setTraceLevel(9); access = new MessageBusDocumentAccess(params); - destination = new VisitableDestination(slobrokConfigId, params.getDocumentManagerConfigId()); + destination = new Destination(slobrokConfigId, params.getDocumentManagerConfigId()); } @After @@ -80,25 +74,6 @@ public class MessageBusDocumentApiTestCase extends AbstractDocumentApiTestCase { slobrok.stop(); } - private static class VisitableDestination extends Destination { - private VisitableDestination(String slobrokConfigId, String documentManagerConfigId) { - super(slobrokConfigId, documentManagerConfigId); - } - - public void handleMessage(Message msg) { - if (msg.getType() == DocumentProtocol.MESSAGE_CREATEVISITOR) { - Reply reply = ((DocumentMessage)msg).createReply(); - msg.swapState(reply); - CreateVisitorReply visitorReply = (CreateVisitorReply)reply; - visitorReply.setLastBucket(ProgressToken.FINISHED_BUCKET); - sendReply(reply); - } else { - super.handleMessage(msg); - } - } - } - - @Test public void requireThatVisitorSessionWorksWithMessageBus() throws ParseException, InterruptedException { VisitorParameters parameters = new VisitorParameters("id.user==1234"); @@ -119,16 +94,17 @@ public class MessageBusDocumentApiTestCase extends AbstractDocumentApiTestCase { AsyncSession session = access().createAsyncSession(new AsyncParameters()); DocumentType type = access().getDocumentTypeManager().getDocumentType("music"); Document doc1 = new Document(type, new DocumentId("id:ns:music::1")); + + destination.discard.set(true); assertTrue(session.put(new DocumentPut(doc1), DocumentOperationParameters.parameters() .withResponseHandler(result -> { response.set(result); latch.countDown(); }) - .withDeadline(Instant.now().minusSeconds(1))) + .withDeadline(Instant.now().plusMillis(100))) .isSuccess()); assertTrue(latch.await(60, TimeUnit.SECONDS)); - assertNotNull(response.get()); assertEquals(Response.Outcome.TIMEOUT, response.get().outcome()); session.destroy(); } diff --git a/documentapi/src/test/java/com/yahoo/documentapi/test/AbstractDocumentApiTestCase.java b/documentapi/src/test/java/com/yahoo/documentapi/test/AbstractDocumentApiTestCase.java index a004ad368e0..001a5d284e2 100644 --- a/documentapi/src/test/java/com/yahoo/documentapi/test/AbstractDocumentApiTestCase.java +++ b/documentapi/src/test/java/com/yahoo/documentapi/test/AbstractDocumentApiTestCase.java @@ -173,4 +173,5 @@ public abstract class AbstractDocumentApiTestCase { latch.countDown(); } } + } diff --git a/flags/src/main/java/com/yahoo/vespa/flags/Flags.java b/flags/src/main/java/com/yahoo/vespa/flags/Flags.java index 524310d017a..e6e34b84439 100644 --- a/flags/src/main/java/com/yahoo/vespa/flags/Flags.java +++ b/flags/src/main/java/com/yahoo/vespa/flags/Flags.java @@ -376,13 +376,6 @@ public class Flags { "Takes effect immediately", ZONE_ID); - public static final UnboundBooleanFlag ALLOW_NO_TESTS = defineFeatureFlag( - "allow-no-tests", false, - List.of("jonmv"), "2022-02-28", "2022-06-25", - "Whether test jobs without any tests run are acceptable", - "Takes effect immediately", - TENANT_ID); - public static final UnboundBooleanFlag MERGE_GROUPING_RESULT_IN_SEARCH_INVOKER = defineFeatureFlag( "merge-grouping-result-in-search-invoker", false, List.of("bjorncs", "baldersheim"), "2022-02-23", "2022-08-01", @@ -398,7 +391,7 @@ public class Flags { ZONE_ID, APPLICATION_ID); public static final UnboundStringFlag CONTROLLER_LOCK_SCHEME = defineStringFlag( - "controller-lock-scheme", "OLD", + "controller-lock-scheme", "NEW", List.of("hmusum"), "2022-04-07", "2022-05-07", "Lock scheme to application-related controller locks (valid values: OLD, BOTH, NEW)", "Takes effect immediately", diff --git a/hosted-api/src/main/java/ai/vespa/hosted/api/ControllerHttpClient.java b/hosted-api/src/main/java/ai/vespa/hosted/api/ControllerHttpClient.java index 2860b8878b7..67cd7e1477e 100644 --- a/hosted-api/src/main/java/ai/vespa/hosted/api/ControllerHttpClient.java +++ b/hosted-api/src/main/java/ai/vespa/hosted/api/ControllerHttpClient.java @@ -418,6 +418,8 @@ public abstract class ControllerHttpClient { submission.sourceUrl().ifPresent(url -> rootObject.setString("sourceUrl", url)); submission.authorEmail().ifPresent(email -> rootObject.setString("authorEmail", email)); submission.projectId().ifPresent(projectId -> rootObject.setLong("projectId", projectId)); + submission.risk().ifPresent(risk -> rootObject.setLong("risk", risk)); + submission.description().ifPresent(description -> rootObject.setString("description", description)); return toJson(slime); } diff --git a/hosted-api/src/main/java/ai/vespa/hosted/api/Submission.java b/hosted-api/src/main/java/ai/vespa/hosted/api/Submission.java index d3ebd715da8..173b4946d5a 100644 --- a/hosted-api/src/main/java/ai/vespa/hosted/api/Submission.java +++ b/hosted-api/src/main/java/ai/vespa/hosted/api/Submission.java @@ -20,10 +20,13 @@ public class Submission { private final Path applicationZip; private final Path applicationTestZip; private final Optional<Long> projectId; + private final Optional<Integer> risk; + private final Optional<String> description; public Submission(Optional<String> repository, Optional<String> branch, Optional<String> commit, Optional<String> sourceUrl, Optional<String> authorEmail, - Path applicationZip, Path applicationTestZip, Optional<Long> projectId) { + Path applicationZip, Path applicationTestZip, Optional<Long> projectId, + Optional<Integer> risk, Optional<String> description) { this.repository = repository; this.branch = branch; this.commit = commit; @@ -32,6 +35,8 @@ public class Submission { this.applicationZip = applicationZip; this.applicationTestZip = applicationTestZip; this.projectId = projectId; + this.risk = risk; + this.description = description; } public Optional<String> repository() { return repository; } @@ -42,5 +47,7 @@ public class Submission { public Path applicationZip() { return applicationZip; } public Path applicationTestZip() { return applicationTestZip; } public Optional<Long> projectId() { return projectId; } + public Optional<Integer> risk() { return risk; } + public Optional<String> description() { return description; } } diff --git a/jdisc_core/src/main/java/com/yahoo/jdisc/core/StandaloneMain.java b/jdisc_core/src/main/java/com/yahoo/jdisc/core/StandaloneMain.java index dc5a9e7984f..f3671b5f36d 100644 --- a/jdisc_core/src/main/java/com/yahoo/jdisc/core/StandaloneMain.java +++ b/jdisc_core/src/main/java/com/yahoo/jdisc/core/StandaloneMain.java @@ -36,23 +36,18 @@ public class StandaloneMain { void run(String bundleLocation) { try { - // We're not logging at this point since the application is responsible - // for setting up logging. - System.out.println("debug\tInitializing application without privileges."); + log.log(Level.FINE, "Initializing application without privileges."); setupSignalHandlers(); loader.init(bundleLocation, false); loader.start(); waitForShutdown(); - System.out.println("debug\tTrying to shutdown in a controlled manner."); log.log(Level.INFO, "JDisc shutting down"); loader.stop(); - System.out.println("debug\tTrying to clean up in a controlled manner."); + log.log(Level.FINE, "Trying to clean up in a controlled manner."); loader.destroy(); - System.out.println("debug\tStopped ok."); + log.log(Level.FINE, "Stopped ok."); System.exit(0); } catch (Throwable e) { - System.out.print("debug\tUnexpected: "); - e.printStackTrace(); log.log(Level.SEVERE, "JDisc exiting: Throwable caught: ", e); System.exit(6); } diff --git a/jdisc_core_test/test_bundles/cert-k-pkgs/src/main/java/com/yahoo/jdisc/bundle/k/CertificateK.java b/jdisc_core_test/test_bundles/cert-k-pkgs/src/main/java/com/yahoo/jdisc/bundle/k/CertificateK.java index ad93a7f322c..f8d85233602 100644 --- a/jdisc_core_test/test_bundles/cert-k-pkgs/src/main/java/com/yahoo/jdisc/bundle/k/CertificateK.java +++ b/jdisc_core_test/test_bundles/cert-k-pkgs/src/main/java/com/yahoo/jdisc/bundle/k/CertificateK.java @@ -4,7 +4,7 @@ package com.yahoo.jdisc.bundle.k; /** * @author Simon Thoresen Hult */ -@SuppressWarnings({ "UnusedDeclaration", "deprecation" }) +@SuppressWarnings({ "UnusedDeclaration", "deprecation", "removal" }) public class CertificateK { private final com.google.common.annotations.Beta beta = null; @@ -37,6 +37,7 @@ public class CertificateK { private final com.yahoo.yolean.trace.TraceNode traceNode = null; private final com.sun.security.auth.LdapPrincipal principal = null; private final com.sun.security.auth.module.JndiLoginModule jndiLoginModule = null; + private final java.security.cert.Certificate certificate = null; private final javax.accessibility.Accessible accessible = null; private final javax.annotation.PostConstruct postConstruct = null; private final javax.annotation.processing.FilerException filerException = null; @@ -82,7 +83,7 @@ public class CertificateK { private final javax.security.auth.login.AccountException accountException = null; private final javax.security.auth.spi.LoginModule loginModule = null; private final javax.security.auth.x500.X500Principal x500Principal = null; - private final javax.security.cert.Certificate certificate = null; + private final javax.security.cert.Certificate deprecatedCertificate = null; private final javax.security.sasl.AuthorizeCallback authorizeCallback = null; private final javax.sound.midi.ControllerEventListener controllerEventListener = null; private final javax.sound.midi.spi.MidiDeviceProvider midiDeviceProvider = null; diff --git a/logserver/src/test/java/ai/vespa/logserver/protocol/ArchiveLogMessagesMethodTest.java b/logserver/src/test/java/ai/vespa/logserver/protocol/ArchiveLogMessagesMethodTest.java index 367744e53bf..d60587d1642 100644 --- a/logserver/src/test/java/ai/vespa/logserver/protocol/ArchiveLogMessagesMethodTest.java +++ b/logserver/src/test/java/ai/vespa/logserver/protocol/ArchiveLogMessagesMethodTest.java @@ -10,8 +10,6 @@ import com.yahoo.jrt.Supervisor; import com.yahoo.jrt.Target; import com.yahoo.jrt.Transport; import com.yahoo.jrt.Values; -import java.util.logging.Level; - import com.yahoo.log.LogLevel; import com.yahoo.log.LogMessage; import com.yahoo.logserver.LogDispatcher; @@ -21,6 +19,7 @@ import java.time.Instant; import java.time.temporal.ChronoUnit; import java.util.ArrayList; import java.util.List; +import java.util.logging.Level; import static ai.vespa.logserver.protocol.ProtobufSerialization.fromLogResponse; import static org.junit.Assert.assertEquals; @@ -69,7 +68,7 @@ public class ArchiveLogMessagesMethodTest { request.parameters().add(new Int8Value((byte)0)); request.parameters().add(new Int32Value(requestPayload.length)); request.parameters().add(new DataValue(requestPayload)); - target.invokeSync(request, 10/*seconds*/); + target.invokeSync(request, 30/*seconds*/); Values returnValues = request.returnValues(); assertEquals(3, returnValues.size()); assertEquals(0, returnValues.get(0).asInt8()); diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integration/ContainerTester.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integration/ContainerTester.java index 415a8b6f438..a2312a23925 100644 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integration/ContainerTester.java +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integration/ContainerTester.java @@ -36,6 +36,7 @@ import java.util.Optional; import java.util.concurrent.Phaser; import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeoutException; +import java.util.concurrent.atomic.AtomicBoolean; import java.util.logging.Logger; import static org.mockito.ArgumentMatchers.any; @@ -98,6 +99,10 @@ public class ContainerTester implements AutoCloseable { } @Override public void stopForHostSuspension(NodeAgentContext context) { super.stopForHostSuspension(context); + phaser.arriveAndAwaitAdvance(); + } + @Override public void stopForRemoval(NodeAgentContext context) { + super.stopForRemoval(context); phaser.arriveAndDeregister(); } }; @@ -109,7 +114,7 @@ public class ContainerTester implements AutoCloseable { loopThread = new Thread(() -> { nodeAdminStateUpdater.start(); - while ( !phaser.isTerminated()) { + while ( ! phaser.isTerminated()) { try { nodeAdminStateUpdater.converge(wantedState); } catch (RuntimeException e) { diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integration/RebootTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integration/RebootTest.java index f3635be1b4b..277f9906dde 100644 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integration/RebootTest.java +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integration/RebootTest.java @@ -35,8 +35,7 @@ public class RebootTest { tester.setWantedState(NodeAdminStateUpdater.State.SUSPENDED); - tester.inOrder(tester.orchestrator).suspend( - eq(HOST_HOSTNAME.value()), eq(List.of(hostname, HOST_HOSTNAME.value()))); + tester.inOrder(tester.orchestrator).suspend(eq(HOST_HOSTNAME.value()), eq(List.of(hostname, HOST_HOSTNAME.value()))); tester.inOrder(tester.containerOperations).stopServices(containerMatcher(host1)); assertTrue(tester.nodeAdmin.setFrozen(true)); } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/CapacityChecker.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/CapacityChecker.java index 26a0315809b..e7e1e371047 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/CapacityChecker.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/CapacityChecker.java @@ -187,6 +187,8 @@ public class CapacityChecker { var resourceMap = new HashMap<>(availableResources); List<Node> validAllocationTargets = allHosts.stream() .filter(h -> !hostsToRemove.contains(h)) + .filter(host -> !host.status().wantToRetire() && + !host.status().wantToFail()) .collect(Collectors.toList()); if (validAllocationTargets.size() == 0) return Optional.of(HostRemovalFailure.none()); diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/Nodes.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/Nodes.java index d544ea76983..3aa09b1b667 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/Nodes.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/Nodes.java @@ -773,12 +773,15 @@ public class Nodes { public Mutex lockUnallocated() { return db.lockInactive(); } /** Returns the unallocated/application lock, and the node acquired under that lock. */ - public Optional<NodeMutex> lockAndGet(Node node) { + public Optional<NodeMutex> lockAndGet(Node node) { return lockAndGet(node, Optional.empty()); } + + /** Returns the unallocated/application lock, and the node acquired under that lock. */ + public Optional<NodeMutex> lockAndGet(Node node, Optional<Duration> timeout) { Node staleNode = node; final int maxRetries = 4; for (int i = 0; i < maxRetries; ++i) { - Mutex lockToClose = lock(staleNode); + Mutex lockToClose = timeout.isPresent() ? lock(staleNode, timeout.get()) : lock(staleNode); try { // As an optimization we first try finding the node in the same state Optional<Node> freshNode = node(staleNode.hostname(), staleNode.state()); @@ -813,6 +816,11 @@ public class Nodes { } /** Returns the unallocated/application lock, and the node acquired under that lock. */ + public Optional<NodeMutex> lockAndGet(String hostname, Duration timeout) { + return node(hostname).flatMap(node -> lockAndGet(node, Optional.of(timeout))); + } + + /** Returns the unallocated/application lock, and the node acquired under that lock. */ public NodeMutex lockAndGetRequired(Node node) { return lockAndGet(node).orElseThrow(() -> new NoSuchNodeException("No node with hostname '" + node.hostname() + "'")); } @@ -822,10 +830,19 @@ public class Nodes { return lockAndGet(hostname).orElseThrow(() -> new NoSuchNodeException("No node with hostname '" + hostname + "'")); } + /** Returns the unallocated/application lock, and the node acquired under that lock. */ + public NodeMutex lockAndGetRequired(String hostname, Duration timeout) { + return lockAndGet(hostname, timeout).orElseThrow(() -> new NoSuchNodeException("No node with hostname '" + hostname + "'")); + } + private Mutex lock(Node node) { return node.allocation().isPresent() ? lock(node.allocation().get().owner()) : lockUnallocated(); } + private Mutex lock(Node node, Duration timeout) { + return node.allocation().isPresent() ? lock(node.allocation().get().owner(), timeout) : lockUnallocated(); + } + private Node requireNode(String hostname) { return node(hostname).orElseThrow(() -> new NoSuchNodeException("No node with hostname '" + hostname + "'")); } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/CuratorDatabaseClient.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/CuratorDatabaseClient.java index a9abc352d8c..13dd458c041 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/CuratorDatabaseClient.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/CuratorDatabaseClient.java @@ -69,6 +69,7 @@ public class CuratorDatabaseClient { private static final Path firmwareCheckPath = root.append("firmwareCheck"); private static final Path archiveUrisPath = root.append("archiveUris"); + // TODO: Explain reasoning behind timeout value (why its it as high as 10 minutes?) private static final Duration defaultLockTimeout = Duration.ofMinutes(10); private final NodeSerializer nodeSerializer; @@ -319,7 +320,12 @@ public class CuratorDatabaseClient { /** Acquires the single cluster-global, reentrant lock for all non-active nodes */ public Lock lockInactive() { - return db.lock(lockPath.append("unallocatedLock"), defaultLockTimeout); + return lockInactive(defaultLockTimeout); + } + + /** Acquires the single cluster-global, reentrant lock for all non-active nodes */ + public Lock lockInactive(Duration timeout) { + return db.lock(lockPath.append("unallocatedLock"), timeout); } /** Acquires the single cluster-global, reentrant lock for active nodes of this application */ diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/NodePatcher.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/NodePatcher.java index fa6c44e6851..d8e1828b10c 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/NodePatcher.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/NodePatcher.java @@ -29,6 +29,7 @@ import com.yahoo.yolean.Exceptions; import java.io.InputStream; import java.time.Clock; +import java.time.Duration; import java.time.Instant; import java.util.ArrayList; import java.util.HashMap; @@ -86,7 +87,7 @@ public class NodePatcher { Map<String, Inspector> recursiveFields = Maps.filterKeys(fields, RECURSIVE_FIELDS::contains); // Patch - NodeMutex nodeMutex = nodeRepository.nodes().lockAndGetRequired(hostname); + NodeMutex nodeMutex = nodeRepository.nodes().lockAndGetRequired(hostname, Duration.ofSeconds(10)); // timeout should match the one used by clients patch(nodeMutex, regularFields, root, false); patchIpConfig(hostname, ipConfigFields); if (nodeMutex.node().type().isHost()) { diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/NodesResponse.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/NodesResponse.java index aa429522147..729ea97d627 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/NodesResponse.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/NodesResponse.java @@ -172,6 +172,7 @@ class NodesResponse extends SlimeJsonResponse { object.setBool("preferToRetire", node.status().preferToRetire()); object.setBool("wantToDeprovision", node.status().wantToDeprovision()); object.setBool("wantToRebuild", node.status().wantToRebuild()); + object.setBool("down", node.isDown()); toSlime(node.history().events(), object.setArray("history")); toSlime(node.history().log(), object.setArray("log")); ipAddressesToSlime(node.ipConfig().primary(), object.setArray("ipAddresses")); diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/cfg1.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/cfg1.json index 7541e7df42d..6e2a3b87fe3 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/cfg1.json +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/cfg1.json @@ -16,6 +16,7 @@ "preferToRetire": false, "wantToDeprovision": false, "wantToRebuild": false, + "down": false, "history": [ { "event": "provisioned", diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/cfg2.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/cfg2.json index e4abedfa926..20809fe51c3 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/cfg2.json +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/cfg2.json @@ -16,6 +16,7 @@ "preferToRetire": false, "wantToDeprovision": false, "wantToRebuild": false, + "down": false, "history": [ { "event": "provisioned", diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/controller1.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/controller1.json index 97df1ba9761..11f183a13d0 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/controller1.json +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/controller1.json @@ -16,6 +16,7 @@ "preferToRetire": false, "wantToDeprovision": false, "wantToRebuild": false, + "down": false, "history": [ { "event": "provisioned", diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/docker-container1.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/docker-container1.json index 56d9502af61..77da20104fe 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/docker-container1.json +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/docker-container1.json @@ -33,6 +33,7 @@ "preferToRetire": false, "wantToDeprovision": false, "wantToRebuild": false, + "down": false, "history": [ { "event": "reserved", diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/docker-node1-os-upgrade-complete.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/docker-node1-os-upgrade-complete.json index 50dc4f3d236..b885f7bd7fc 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/docker-node1-os-upgrade-complete.json +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/docker-node1-os-upgrade-complete.json @@ -35,6 +35,7 @@ "preferToRetire": false, "wantToDeprovision": false, "wantToRebuild": false, + "down": false, "history": [ { "event": "provisioned", diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/docker-node1-reports-2.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/docker-node1-reports-2.json index 1f269636a94..9c413cf3d82 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/docker-node1-reports-2.json +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/docker-node1-reports-2.json @@ -41,6 +41,7 @@ "preferToRetire": false, "wantToDeprovision": true, "wantToRebuild": false, + "down": false, "history": [ { "event": "provisioned", diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/docker-node1-reports-3.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/docker-node1-reports-3.json index 4bdc81cf0cd..77d701904a6 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/docker-node1-reports-3.json +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/docker-node1-reports-3.json @@ -41,6 +41,7 @@ "preferToRetire": false, "wantToDeprovision": false, "wantToRebuild": false, + "down": false, "history": [ { "event": "provisioned", diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/docker-node1-reports-4.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/docker-node1-reports-4.json index 46830c0b9f1..8d00444bb1b 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/docker-node1-reports-4.json +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/docker-node1-reports-4.json @@ -41,6 +41,7 @@ "preferToRetire": false, "wantToDeprovision": false, "wantToRebuild": false, + "down": false, "history": [ { "event": "provisioned", diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/docker-node1-reports.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/docker-node1-reports.json index dc11457c218..051b1799324 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/docker-node1-reports.json +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/docker-node1-reports.json @@ -41,6 +41,7 @@ "preferToRetire": false, "wantToDeprovision": true, "wantToRebuild": false, + "down": false, "history": [ { "event": "provisioned", diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/docker-node1.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/docker-node1.json index 64c309d8e51..14a0662a470 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/docker-node1.json +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/docker-node1.json @@ -33,6 +33,7 @@ "preferToRetire": false, "wantToDeprovision": false, "wantToRebuild": false, + "down": false, "history": [ { "event": "provisioned", diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/docker-node2.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/docker-node2.json index 7d7528ecacf..6bde9d15fb2 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/docker-node2.json +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/docker-node2.json @@ -33,6 +33,7 @@ "preferToRetire": false, "wantToDeprovision": false, "wantToRebuild": false, + "down": false, "history": [ { "event": "provisioned", diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/docker-node3.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/docker-node3.json index 49ebe7fdab7..d1dc2375e1e 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/docker-node3.json +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/docker-node3.json @@ -33,6 +33,7 @@ "preferToRetire": false, "wantToDeprovision": false, "wantToRebuild": false, + "down": false, "history": [ { "event": "provisioned", diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/docker-node4.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/docker-node4.json index 6e084d0a882..c4163f32d7a 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/docker-node4.json +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/docker-node4.json @@ -33,6 +33,7 @@ "preferToRetire": false, "wantToDeprovision": false, "wantToRebuild": false, + "down": false, "history": [ { "event": "provisioned", diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/docker-node5.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/docker-node5.json index 168a32432d5..1964a2bac07 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/docker-node5.json +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/docker-node5.json @@ -33,6 +33,7 @@ "preferToRetire": false, "wantToDeprovision": false, "wantToRebuild": false, + "down": false, "history": [ { "event": "provisioned", diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/dockerhost1-with-firmware-data.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/dockerhost1-with-firmware-data.json index 91a544da298..d2e1c88236b 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/dockerhost1-with-firmware-data.json +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/dockerhost1-with-firmware-data.json @@ -35,6 +35,7 @@ "preferToRetire": false, "wantToDeprovision": false, "wantToRebuild": false, + "down": false, "history": [ { "event": "provisioned", diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/dockerhost6.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/dockerhost6.json index 566989c4508..54f29ee26a4 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/dockerhost6.json +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/dockerhost6.json @@ -16,6 +16,7 @@ "preferToRetire": false, "wantToDeprovision": false, "wantToRebuild": false, + "down": false, "history": [ { "event": "provisioned", diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/node1.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/node1.json index ae8ca77ddda..9a0817a441d 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/node1.json +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/node1.json @@ -32,6 +32,7 @@ "preferToRetire": false, "wantToDeprovision": false, "wantToRebuild": false, + "down": false, "history": [ { "event": "provisioned", diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/node10.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/node10.json index 1f68b970007..7e875f4013c 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/node10.json +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/node10.json @@ -35,6 +35,7 @@ "preferToRetire": false, "wantToDeprovision": false, "wantToRebuild": false, + "down": false, "history": [ { "event": "provisioned", diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/node11.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/node11.json index d825e105291..c7d7d17b0f4 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/node11.json +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/node11.json @@ -16,6 +16,7 @@ "preferToRetire": false, "wantToDeprovision": false, "wantToRebuild": false, + "down": false, "history": [ { "event": "provisioned", diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/node13.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/node13.json index 0545c600c19..f1fb2def612 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/node13.json +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/node13.json @@ -32,6 +32,7 @@ "preferToRetire": false, "wantToDeprovision": false, "wantToRebuild": false, + "down": false, "history": [ { "event": "readied", diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/node14.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/node14.json index 05668bf83c6..c19ca13066d 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/node14.json +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/node14.json @@ -32,6 +32,7 @@ "preferToRetire": false, "wantToDeprovision": false, "wantToRebuild": false, + "down": false, "history": [ { "event": "readied", diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/node2.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/node2.json index b6d53e2c509..c49a5bcf76d 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/node2.json +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/node2.json @@ -32,6 +32,7 @@ "preferToRetire": false, "wantToDeprovision": false, "wantToRebuild": false, + "down": false, "history": [ { "event": "provisioned", diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/node3.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/node3.json index e560027668a..2e3181053ad 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/node3.json +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/node3.json @@ -15,6 +15,7 @@ "preferToRetire": false, "wantToDeprovision": false, "wantToRebuild": false, + "down": false, "history": [ { "event": "provisioned", diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/node4-after-changes.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/node4-after-changes.json index f59d9ff561f..1775477116e 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/node4-after-changes.json +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/node4-after-changes.json @@ -38,6 +38,7 @@ "preferToRetire": false, "wantToDeprovision": false, "wantToRebuild": false, + "down": false, "history": [ { "event": "provisioned", diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/node4-with-hostnames.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/node4-with-hostnames.json index 78c6d1d7243..c80790e2ad3 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/node4-with-hostnames.json +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/node4-with-hostnames.json @@ -35,6 +35,7 @@ "preferToRetire": false, "wantToDeprovision": false, "wantToRebuild": false, + "down": false, "history": [ { "event": "provisioned", diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/node4.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/node4.json index 86e904d7970..7bd02db118a 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/node4.json +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/node4.json @@ -35,6 +35,7 @@ "preferToRetire": false, "wantToDeprovision": false, "wantToRebuild": false, + "down": false, "history": [ { "event": "provisioned", diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/node5-after-changes.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/node5-after-changes.json index b70675f5ab2..e0c9a63e522 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/node5-after-changes.json +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/node5-after-changes.json @@ -16,6 +16,7 @@ "preferToRetire": false, "wantToDeprovision": false, "wantToRebuild": false, + "down": false, "history": [ { "event": "provisioned", diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/node5.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/node5.json index 80dfbbaa982..62cd07d462f 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/node5.json +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/node5.json @@ -18,6 +18,7 @@ "preferToRetire": false, "wantToDeprovision": false, "wantToRebuild": false, + "down": false, "history": [ { "event": "provisioned", diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/node55.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/node55.json index 015c5a5558f..94c03e54b9b 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/node55.json +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/node55.json @@ -15,6 +15,7 @@ "preferToRetire": false, "wantToDeprovision": true, "wantToRebuild": false, + "down": false, "history": [ { "event": "provisioned", diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/node6.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/node6.json index f9e9053dec2..9fa2b9ac703 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/node6.json +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/node6.json @@ -32,6 +32,7 @@ "preferToRetire": false, "wantToDeprovision": false, "wantToRebuild": false, + "down": false, "history": [ { "event": "provisioned", diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/node7.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/node7.json index 62c7faf30ae..d049d831973 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/node7.json +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/node7.json @@ -15,6 +15,7 @@ "preferToRetire": false, "wantToDeprovision": false, "wantToRebuild": false, + "down": false, "history": [ { "event": "provisioned", diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/node8.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/node8.json index 7ab57e42757..7df4cb042a1 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/node8.json +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/node8.json @@ -16,6 +16,7 @@ "preferToRetire": false, "wantToDeprovision": false, "wantToRebuild": false, + "down": false, "history": [ { "event": "provisioned", diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/node9.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/node9.json index e94ae56f00f..249cd77f780 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/node9.json +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/node9.json @@ -16,6 +16,7 @@ "preferToRetire": false, "wantToDeprovision": false, "wantToRebuild": false, + "down": false, "history": [ { "event": "provisioned", diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/parent2.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/parent2.json index 741f4bc1274..e06ef551e74 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/parent2.json +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/parent2.json @@ -18,6 +18,7 @@ "preferToRetire": false, "wantToDeprovision": false, "wantToRebuild": false, + "down": false, "history": [ { "event": "provisioned", diff --git a/parent/pom.xml b/parent/pom.xml index ea7b2aa41cc..ecf257727d1 100644 --- a/parent/pom.xml +++ b/parent/pom.xml @@ -729,12 +729,12 @@ <dependency> <groupId>org.eclipse.collections</groupId> <artifactId>eclipse-collections</artifactId> - <version>11.0.0</version> + <version>${eclipse-collections.version}</version> </dependency> <dependency> <groupId>org.eclipse.collections</groupId> <artifactId>eclipse-collections-api</artifactId> - <version>11.0.0</version> + <version>${eclipse-collections.version}</version> </dependency> <dependency> <groupId>org.eclipse.jetty.alpn</groupId> @@ -935,6 +935,7 @@ <curator.version>5.2.0</curator.version> <commons.codec.version>1.15</commons.codec.version> <commons.math3.version>3.6.1</commons.math3.version> + <eclipse-collections.version>11.0.0</eclipse-collections.version> <gson.version>2.8.9</gson.version> <hdrhistogram.version>2.1.12</hdrhistogram.version> <jna.version>5.9.0</jna.version> @@ -950,7 +951,7 @@ <maven-javadoc-plugin.version>3.3.1</maven-javadoc-plugin.version> <maven-plugin-tools.version>3.6.1</maven-plugin-tools.version> <maven-resources-plugin.version>3.2.0</maven-resources-plugin.version> - <maven-shade-plugin.version>3.2.4</maven-shade-plugin.version> + <maven-shade-plugin.version>3.3.0</maven-shade-plugin.version> <maven-site-plugin.version>3.9.1</maven-site-plugin.version> <maven-source-plugin.version>3.2.1</maven-source-plugin.version> <mockito.version>4.0.0</mockito.version> @@ -117,7 +117,6 @@ <module>searchsummary</module> <module>security-tools</module> <module>security-utils</module> - <module>serviceview</module> <module>service-monitor</module> <module>socket_test</module> <module>standalone-container</module> diff --git a/searchcommon/src/vespa/searchcommon/attribute/i_multi_value_attribute.h b/searchcommon/src/vespa/searchcommon/attribute/i_multi_value_attribute.h index 3ed86a076b3..641e602e522 100644 --- a/searchcommon/src/vespa/searchcommon/attribute/i_multi_value_attribute.h +++ b/searchcommon/src/vespa/searchcommon/attribute/i_multi_value_attribute.h @@ -38,6 +38,7 @@ public: virtual const IArrayReadView<int64_t>* make_read_view(ArrayTag<int64_t>, vespalib::Stash&) const { return nullptr; } virtual const IArrayReadView<float>* make_read_view(ArrayTag<float>, vespalib::Stash&) const { return nullptr; } virtual const IArrayReadView<double>* make_read_view(ArrayTag<double>, vespalib::Stash&) const { return nullptr; } + virtual const IArrayReadView<const char*>* make_read_view(ArrayTag<const char*>, vespalib::Stash&) const { return nullptr; } virtual const IWeightedSetReadView<int8_t>* make_read_view(WeightedSetTag<int8_t>, vespalib::Stash&) const { return nullptr; } virtual const IWeightedSetReadView<int16_t>* make_read_view(WeightedSetTag<int16_t>, vespalib::Stash&) const { return nullptr; } @@ -45,6 +46,7 @@ public: virtual const IWeightedSetReadView<int64_t>* make_read_view(WeightedSetTag<int64_t>, vespalib::Stash&) const { return nullptr; } virtual const IWeightedSetReadView<float>* make_read_view(WeightedSetTag<float>, vespalib::Stash&) const { return nullptr; } virtual const IWeightedSetReadView<double>* make_read_view(WeightedSetTag<double>, vespalib::Stash&) const { return nullptr; } + virtual const IWeightedSetReadView<const char*>* make_read_view(WeightedSetTag<const char*>, vespalib::Stash&) const { return nullptr; } virtual const IArrayEnumReadView* make_read_view(ArrayEnumTag, vespalib::Stash&) const { return nullptr; } virtual const IWeightedSetEnumReadView* make_read_view(WeightedSetEnumTag, vespalib::Stash&) const { return nullptr; } diff --git a/searchcore/src/vespa/searchcore/config/proton.def b/searchcore/src/vespa/searchcore/config/proton.def index 808535924f1..51536299d51 100644 --- a/searchcore/src/vespa/searchcore/config/proton.def +++ b/searchcore/src/vespa/searchcore/config/proton.def @@ -138,7 +138,7 @@ indexing.optimize enum {LATENCY, THROUGHPUT, ADAPTIVE} default=THROUGHPUT restar ## Maximum number of pending operations for each of the internal ## indexing threads. Only used when visibility delay is zero. -indexing.tasklimit int default=1000 +indexing.tasklimit int default=-1000 ## Deprecated and ignored, will soon go away indexing.semiunboundtasklimit int default = 1000 @@ -514,14 +514,14 @@ feeding.concurrency double default = 0.2 restart ## DOCUMENT_DB: Use a shared executor for index field inverter, index field writer, and attribute field writer among all document dbs. ## ## TODO: Remove this when a shared executor is the default. -feeding.shared_field_writer_executor enum {NONE, INDEX, INDEX_AND_ATTRIBUTE, DOCUMENT_DB} default = NONE restart +feeding.shared_field_writer_executor enum {NONE, INDEX, INDEX_AND_ATTRIBUTE, DOCUMENT_DB} default = DOCUMENT_DB restart ## Maximum number of pending tasks for the master thread in each document db. ## ## This limit is only considered when executing tasks for handling external feed operations. ## In that case the calling thread (persistence thread) is blocked until the master thread has capacity to handle more tasks. ## When this limit is set to 0 it is ignored. -feeding.master_task_limit int default = 1000 +feeding.master_task_limit int default = 0 ## Adjustment to resource limit when determining if maintenance jobs can run. ## @@ -550,7 +550,7 @@ forward_issues bool default = true ## Chooses the throttling policy used to control the window size ## of the SharedOperationThrottler component used by the transaction log replay feed state. -replay_throttling_policy.type enum { UNLIMITED, DYNAMIC } default=UNLIMITED +replay_throttling_policy.type enum { UNLIMITED, DYNAMIC } default=DYNAMIC ## Only used if replay_throttling_policy.type == DYNAMIC: replay_throttling_policy.min_window_size int default=100 replay_throttling_policy.max_window_size int default=10000 diff --git a/searchcore/src/vespa/searchcore/proton/server/documentdb.cpp b/searchcore/src/vespa/searchcore/proton/server/documentdb.cpp index 33a0001cc75..e8c3ac885ea 100644 --- a/searchcore/src/vespa/searchcore/proton/server/documentdb.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/documentdb.cpp @@ -763,7 +763,6 @@ DocumentDB::getNewestFlushedSerial() std::unique_ptr<SearchReply> DocumentDB::match(const SearchRequest &req, vespalib::ThreadBundle &threadBundle) const { - // Ignore input searchhandler. Use readysubdb's searchhandler instead. ISearchHandler::SP view(_subDBs.getReadySubDB()->getSearchView()); return view->match(req, threadBundle); } diff --git a/searchcore/src/vespa/searchcore/proton/server/matchers.cpp b/searchcore/src/vespa/searchcore/proton/server/matchers.cpp index 9ed9b45c1bd..e87f9928ce4 100644 --- a/searchcore/src/vespa/searchcore/proton/server/matchers.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/matchers.cpp @@ -4,26 +4,31 @@ #include <vespa/searchcore/proton/matching/matcher.h> #include <vespa/searchcore/proton/matching/ranking_expressions.h> #include <vespa/searchcore/proton/matching/onnx_models.h> +#include <vespa/vespalib/util/issue.h> +#include <vespa/vespalib/util/stringfmt.h> #include <vespa/vespalib/stllike/hash_map.hpp> namespace proton { using matching::RankingExpressions; using matching::OnnxModels; +using matching::Matcher; +using matching::MatchingStats; +using namespace vespalib::make_string_short; Matchers::Matchers(const vespalib::Clock &clock, matching::QueryLimiter &queryLimiter, const matching::IConstantValueRepo &constantValueRepo) : _rpmap(), - _fallback(new matching::Matcher(search::index::Schema(), search::fef::Properties(), - clock, queryLimiter, constantValueRepo, RankingExpressions(), OnnxModels(), -1)), + _fallback(std::make_shared<Matcher>(search::index::Schema(), search::fef::Properties(), clock, queryLimiter, + constantValueRepo, RankingExpressions(), OnnxModels(), -1)), _default() { } Matchers::~Matchers() = default; void -Matchers::add(const vespalib::string &name, std::shared_ptr<matching::Matcher> matcher) +Matchers::add(const vespalib::string &name, std::shared_ptr<Matcher> matcher) { if ((name == "default") || ! _default) { _default = matcher; @@ -31,30 +36,48 @@ Matchers::add(const vespalib::string &name, std::shared_ptr<matching::Matcher> m _rpmap[name] = std::move(matcher); } -matching::MatchingStats +MatchingStats Matchers::getStats() const { - matching::MatchingStats stats; + MatchingStats stats; for (const auto & entry : _rpmap) { stats.add(entry.second->getStats()); } return stats; } -matching::MatchingStats +MatchingStats Matchers::getStats(const vespalib::string &name) const { auto it = _rpmap.find(name); - return it != _rpmap.end() ? it->second->getStats() : - matching::MatchingStats(); + return it != _rpmap.end() ? it->second->getStats() : MatchingStats(); } -matching::Matcher::SP +std::shared_ptr<Matcher> Matchers::lookup(const vespalib::string &name) const { Map::const_iterator found(_rpmap.find(name)); - return (found != _rpmap.end()) ? found->second : _default; - //TODO add warning log message when not found, may want to use "_fallback" in most cases here + if (found == _rpmap.end()) { + if (_default) { + // TODO Report as issue on Vespa 8 + // vespalib::Issue::report(fmt("Failed to find rank-profile '%s'. Falling back to 'default'", name.c_str())); + return _default; + } else { + vespalib::Issue::report(fmt("Failed to find rank-profile '%s'. Most likely a configuration issue.", name.c_str())); + return _fallback; + } + } + return found->second; +} + +vespalib::string +Matchers::listMatchers() const { + vespalib::string matchers; + for (const auto & entry : _rpmap) { + matchers += entry.first; + matchers += ' '; + } + return matchers; } } // namespace proton diff --git a/searchcore/src/vespa/searchcore/proton/server/matchers.h b/searchcore/src/vespa/searchcore/proton/server/matchers.h index b9c47da4c3e..fe467c1f565 100644 --- a/searchcore/src/vespa/searchcore/proton/server/matchers.h +++ b/searchcore/src/vespa/searchcore/proton/server/matchers.h @@ -17,7 +17,7 @@ namespace matching { class Matchers { private: - typedef vespalib::hash_map<vespalib::string, std::shared_ptr<matching::Matcher>> Map; + using Map = vespalib::hash_map<vespalib::string, std::shared_ptr<matching::Matcher>>; Map _rpmap; std::shared_ptr<matching::Matcher> _fallback; std::shared_ptr<matching::Matcher> _default; @@ -27,11 +27,15 @@ public: Matchers(const vespalib::Clock &clock, matching::QueryLimiter &queryLimiter, const matching::IConstantValueRepo &constantValueRepo); + Matchers(const Matchers &) = delete; + Matchers & operator =(const Matchers &) = delete; ~Matchers(); void add(const vespalib::string &name, std::shared_ptr<matching::Matcher> matcher); matching::MatchingStats getStats() const; matching::MatchingStats getStats(const vespalib::string &name) const; std::shared_ptr<matching::Matcher> lookup(const vespalib::string &name) const; + vespalib::string listMatchers() const; + uint32_t numMatchers() const { return _rpmap.size(); } }; } // namespace proton diff --git a/searchcore/src/vespa/searchcore/proton/server/matchview.cpp b/searchcore/src/vespa/searchcore/proton/server/matchview.cpp index a265281b4ce..768e2c80b62 100644 --- a/searchcore/src/vespa/searchcore/proton/server/matchview.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/matchview.cpp @@ -6,6 +6,7 @@ #include <vespa/searchlib/engine/searchrequest.h> #include <vespa/searchlib/engine/searchreply.h> #include <vespa/vespalib/util/stringfmt.h> +#include <vespa/vespalib/util/exceptions.h> #include <vespa/log/log.h> LOG_SETUP(".proton.server.matchview"); @@ -22,8 +23,9 @@ using search::queryeval::FieldSpec; using search::queryeval::FieldSpecList; using search::queryeval::Searchable; using searchcorespi::IndexSearchable; -using vespalib::make_string; using vespalib::ThreadBundle; +using vespalib::IllegalArgumentException; +using namespace vespalib::make_string_short; namespace proton { @@ -50,12 +52,7 @@ MatchView::~MatchView() = default; Matcher::SP MatchView::getMatcher(const vespalib::string & rankProfile) const { - Matcher::SP retval = _matchers->lookup(rankProfile); - if ( ! retval) { - throw std::runtime_error(make_string("Failed locating Matcher for rank profile '%s'", rankProfile.c_str())); - } - LOG(debug, "Rankprofile = %s has termwise_limit=%f", rankProfile.c_str(), retval->get_termwise_limit()); - return retval; + return _matchers->lookup(rankProfile); } MatchContext::UP diff --git a/searchcore/src/vespa/searchcore/proton/test/mock_shared_threading_service.cpp b/searchcore/src/vespa/searchcore/proton/test/mock_shared_threading_service.cpp index e5d42b34370..e25c1459300 100644 --- a/searchcore/src/vespa/searchcore/proton/test/mock_shared_threading_service.cpp +++ b/searchcore/src/vespa/searchcore/proton/test/mock_shared_threading_service.cpp @@ -1,6 +1,9 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "mock_shared_threading_service.h" +#include <vespa/vespalib/util/sequencedtaskexecutor.h> + +VESPA_THREAD_STACK_TAG(mock_field_writer_executor) namespace proton { @@ -9,6 +12,7 @@ MockSharedThreadingService::MockSharedThreadingService(ThreadExecutor& warmup_in size_t num_bucket_executors) : _warmup(warmup_in), _shared(shared_in), + _field_writer(vespalib::SequencedTaskExecutor::create(mock_field_writer_executor, 1)), _invokeService(10ms), _transport(), _bucket_executor(num_bucket_executors), diff --git a/searchcore/src/vespa/searchcore/proton/test/mock_shared_threading_service.h b/searchcore/src/vespa/searchcore/proton/test/mock_shared_threading_service.h index 167d15d70eb..00ffdc92020 100644 --- a/searchcore/src/vespa/searchcore/proton/test/mock_shared_threading_service.h +++ b/searchcore/src/vespa/searchcore/proton/test/mock_shared_threading_service.h @@ -14,6 +14,7 @@ private: using ThreadExecutor = vespalib::ThreadExecutor; ThreadExecutor & _warmup; ThreadExecutor & _shared; + std::unique_ptr<vespalib::ISequencedTaskExecutor> _field_writer; vespalib::InvokeServiceImpl _invokeService; Transport _transport; storage::spi::dummy::DummyBucketExecutor _bucket_executor; @@ -25,7 +26,7 @@ public: ~MockSharedThreadingService() override; ThreadExecutor& warmup() override { return _warmup; } ThreadExecutor& shared() override { return _shared; } - vespalib::ISequencedTaskExecutor* field_writer() override { return nullptr; } + vespalib::ISequencedTaskExecutor* field_writer() override { return _field_writer.get(); } vespalib::InvokeService & invokeService() override { return _invokeService; } FNET_Transport & transport() override { return _transport.transport(); } storage::spi::BucketExecutor& bucket_executor() override { return _bucket_executor; } diff --git a/searchlib/CMakeLists.txt b/searchlib/CMakeLists.txt index a7886cc3c61..300900dbb77 100644 --- a/searchlib/CMakeLists.txt +++ b/searchlib/CMakeLists.txt @@ -88,6 +88,7 @@ vespa_define_module( src/tests/attribute/imported_attribute_vector src/tests/attribute/imported_search_context src/tests/attribute/multi_value_mapping + src/tests/attribute/multi_value_read_view src/tests/attribute/posting_list_merger src/tests/attribute/postinglist src/tests/attribute/postinglistattribute diff --git a/searchlib/abi-spec.json b/searchlib/abi-spec.json index 5a534562d32..3081c88ec99 100644 --- a/searchlib/abi-spec.json +++ b/searchlib/abi-spec.json @@ -353,12 +353,9 @@ "public" ], "methods": [ - "public static com.yahoo.searchlib.rankingexpression.Reference fromIdentifier(java.lang.String)", "public void <init>(java.lang.String, com.yahoo.searchlib.rankingexpression.rule.Arguments, java.lang.String)", "public com.yahoo.searchlib.rankingexpression.rule.Arguments arguments()", "public java.lang.String output()", - "public static com.yahoo.searchlib.rankingexpression.Reference simple(java.lang.String, java.lang.String)", - "public static java.util.Optional simple(java.lang.String)", "public boolean isIdentifier()", "public boolean isSimple()", "public java.util.Optional simpleArgument()", @@ -369,6 +366,9 @@ "public java.lang.String toString()", "public java.lang.StringBuilder toString(java.lang.StringBuilder, com.yahoo.searchlib.rankingexpression.rule.SerializationContext, java.util.Deque, com.yahoo.searchlib.rankingexpression.rule.CompositeNode)", "public int compareTo(com.yahoo.searchlib.rankingexpression.Reference)", + "public static com.yahoo.searchlib.rankingexpression.Reference fromIdentifier(java.lang.String)", + "public static com.yahoo.searchlib.rankingexpression.Reference simple(java.lang.String, java.lang.String)", + "public static java.util.Optional simple(java.lang.String)", "public bridge synthetic int compareTo(java.lang.Object)" ], "fields": [] @@ -1598,7 +1598,6 @@ "public void <init>(java.lang.String, java.util.List, java.lang.String)", "public void <init>(com.yahoo.searchlib.rankingexpression.Reference)", "public java.lang.String getName()", - "public int hashCode()", "public com.yahoo.searchlib.rankingexpression.rule.Arguments getArguments()", "public com.yahoo.searchlib.rankingexpression.rule.ReferenceNode setArguments(java.util.List)", "public java.lang.String getOutput()", @@ -1608,7 +1607,8 @@ "public com.yahoo.searchlib.rankingexpression.Reference reference()", "public com.yahoo.tensor.TensorType type(com.yahoo.tensor.evaluation.TypeContext)", "public com.yahoo.searchlib.rankingexpression.evaluation.Value evaluate(com.yahoo.searchlib.rankingexpression.evaluation.Context)", - "public com.yahoo.searchlib.rankingexpression.rule.CompositeNode setChildren(java.util.List)" + "public com.yahoo.searchlib.rankingexpression.rule.CompositeNode setChildren(java.util.List)", + "public int hashCode()" ], "fields": [] }, diff --git a/searchlib/src/main/java/com/yahoo/searchlib/aggregation/Group.java b/searchlib/src/main/java/com/yahoo/searchlib/aggregation/Group.java index 0b4a1eb8bbc..4ff3b342a20 100644 --- a/searchlib/src/main/java/com/yahoo/searchlib/aggregation/Group.java +++ b/searchlib/src/main/java/com/yahoo/searchlib/aggregation/Group.java @@ -229,11 +229,16 @@ public class Group extends Identifiable { return this; } - /** Returns the list of child groups to this. */ + /** Returns immutable list of child groups to this. */ public List<Group> getChildren() { return List.copyOf(children); } + /** Returns number of children groups */ + public int getNumChildren() { + return children.size(); + } + /** * Returns the tag of this group. This value is set per-level in the grouping request, and then becomes assigned * to each group of that level in the grouping result as they are copied from the prototype. diff --git a/searchlib/src/main/java/com/yahoo/searchlib/expression/IntegerBucketResultNode.java b/searchlib/src/main/java/com/yahoo/searchlib/expression/IntegerBucketResultNode.java index 4bf33b15ffe..c5d66406fd5 100644 --- a/searchlib/src/main/java/com/yahoo/searchlib/expression/IntegerBucketResultNode.java +++ b/searchlib/src/main/java/com/yahoo/searchlib/expression/IntegerBucketResultNode.java @@ -81,11 +81,11 @@ public class IntegerBucketResultNode extends BucketResultNode { return (classId - rhs.getClassId()); } IntegerBucketResultNode b = (IntegerBucketResultNode)rhs; - long diff = from - b.from; - if (diff == 0) { - diff = to - b.to; - } - return ((diff == 0) ? 0 : ((diff < 0) ? -1 : 1)); + if (from < b.from) return -1; + if (from > b.from) return 1; + if (to < b.to) return -1; + if (to > b.to) return 1; + return 0; } @Override diff --git a/searchlib/src/test/java/com/yahoo/searchlib/expression/IntegerBucketResultNodeTestCase.java b/searchlib/src/test/java/com/yahoo/searchlib/expression/IntegerBucketResultNodeTestCase.java index 00c0739ed9a..9c71625a85e 100644 --- a/searchlib/src/test/java/com/yahoo/searchlib/expression/IntegerBucketResultNodeTestCase.java +++ b/searchlib/src/test/java/com/yahoo/searchlib/expression/IntegerBucketResultNodeTestCase.java @@ -30,4 +30,17 @@ public class IntegerBucketResultNodeTestCase extends ResultNodeTest { assertTrue(dumpNode(bucket).contains("to: 10")); assertCorrectSerialization(bucket, new IntegerBucketResultNode()); } + + private IntegerBucketResultNode createNode(long from, long to) { + return new IntegerBucketResultNode(from, to); + } + + @Test + public void testCmp() { + assertOrder(createNode(Long.MIN_VALUE, 3), createNode(3, 9), createNode(9, Long.MAX_VALUE)); + assertOrder(createNode(6, 9), createNode(7, 9), createNode(8, 9)); + assertOrder(createNode(6, 7), createNode(6, 8), createNode(6, 9)); + assertOrder(createNode(6, 3), createNode(7, 2), createNode(8, 1)); + assertTrue(createNode(6, 8).onCmp(new NullResultNode()) != 0); + } } diff --git a/searchlib/src/tests/attribute/CMakeLists.txt b/searchlib/src/tests/attribute/CMakeLists.txt index fe838925bac..67c1452b3f2 100644 --- a/searchlib/src/tests/attribute/CMakeLists.txt +++ b/searchlib/src/tests/attribute/CMakeLists.txt @@ -4,5 +4,6 @@ vespa_add_executable(searchlib_attribute_test_app TEST attribute_test.cpp DEPENDS searchlib + GTest::GTest ) vespa_add_test(NAME searchlib_attribute_test_app COMMAND searchlib_attribute_test_app COST 250) diff --git a/searchlib/src/tests/attribute/attribute_test.cpp b/searchlib/src/tests/attribute/attribute_test.cpp index cb51487abdd..64edb30520b 100644 --- a/searchlib/src/tests/attribute/attribute_test.cpp +++ b/searchlib/src/tests/attribute/attribute_test.cpp @@ -21,7 +21,7 @@ #include <vespa/searchlib/test/weighted_type_test_utils.h> #include <vespa/searchlib/util/randomgenerator.h> #include <vespa/vespalib/io/fileutil.h> -#include <vespa/vespalib/testkit/testapp.h> +#include <vespa/vespalib/gtest/gtest.h> #include <vespa/vespalib/util/mmap_file_allocator_factory.h> #include <vespa/vespalib/util/round_up_to_page_size.h> #include <vespa/vespalib/util/size_literals.h> @@ -42,15 +42,25 @@ using search::attribute::IAttributeVector; using vespalib::stringref; using vespalib::string; -namespace search { - namespace { -string empty; string tmpDir("tmp"); string clsDir("clstmp"); string asuDir("asutmp"); +} + +namespace search { + +namespace { + +string empty; + +string make_scoped_trace_msg(string prefix, const search::attribute::Config &config) +{ + return prefix + ", basic type=" + config.basicType().asString() + ", collection type=" + config.collectionType().asString(); +} + bool isUnsignedSmallIntAttribute(const BasicType::Type &type) { @@ -74,21 +84,23 @@ template <typename BufferType> void expectZero(const BufferType &b) { - EXPECT_EQUAL(0, b); + EXPECT_EQ(0, b); } template <> void expectZero(const string &b) { - EXPECT_EQUAL(empty, b); + EXPECT_EQ(empty, b); } uint64_t statSize(const string &fileName) { FastOS_StatInfo statInfo; - if (EXPECT_TRUE(FastOS_File::Stat(fileName.c_str(), &statInfo))) { + bool stat_result = true; + EXPECT_TRUE(FastOS_File::Stat(fileName.c_str(), &statInfo)) << (stat_result = false, ""); + if (stat_result) { return statInfo._size; } else { return 0u; @@ -163,9 +175,9 @@ bool contains_value(const Container& c, size_t elems, const V& value) { using attribute::CollectionType; using attribute::Config; -class AttributeTest : public vespalib::TestApp +class AttributeTest : public ::testing::Test { -private: +protected: typedef AttributeVector::SP AttributePtr; void addDocs(const AttributePtr & v, size_t sz); @@ -284,7 +296,6 @@ private: public: AttributeTest(); - int Main() override; }; AttributeTest::AttributeTest() = default; @@ -292,27 +303,27 @@ AttributeTest::AttributeTest() = default; void AttributeTest::testBaseName() { AttributeVector::BaseName v("attr1"); - EXPECT_EQUAL(v.getAttributeName(), "attr1"); + EXPECT_EQ(v.getAttributeName(), "attr1"); EXPECT_TRUE(v.getDirName().empty()); v = "attribute/attr1/attr1"; - EXPECT_EQUAL(v.getAttributeName(), "attr1"); - EXPECT_EQUAL(v.getDirName(), "attribute/attr1"); + EXPECT_EQ(v.getAttributeName(), "attr1"); + EXPECT_EQ(v.getDirName(), "attribute/attr1"); v = "attribute/attr1/snapshot-X/attr1"; - EXPECT_EQUAL(v.getAttributeName(), "attr1"); - EXPECT_EQUAL(v.getDirName(), "attribute/attr1/snapshot-X"); + EXPECT_EQ(v.getAttributeName(), "attr1"); + EXPECT_EQ(v.getDirName(), "attribute/attr1/snapshot-X"); v = "/attribute/attr1/snapshot-X/attr1"; - EXPECT_EQUAL(v.getAttributeName(), "attr1"); - EXPECT_EQUAL(v.getDirName(), "/attribute/attr1/snapshot-X"); + EXPECT_EQ(v.getAttributeName(), "attr1"); + EXPECT_EQ(v.getDirName(), "/attribute/attr1/snapshot-X"); v = "index.1/1.ready/attribute/attr1/snapshot-X/attr1"; - EXPECT_EQUAL(v.getAttributeName(), "attr1"); - EXPECT_EQUAL(v.getDirName(), "index.1/1.ready/attribute/attr1/snapshot-X"); + EXPECT_EQ(v.getAttributeName(), "attr1"); + EXPECT_EQ(v.getDirName(), "index.1/1.ready/attribute/attr1/snapshot-X"); v = "/index.1/1.ready/attribute/attr1/snapshot-X/attr1"; - EXPECT_EQUAL(v.getAttributeName(), "attr1"); - EXPECT_EQUAL(v.getDirName(), + EXPECT_EQ(v.getAttributeName(), "attr1"); + EXPECT_EQ(v.getDirName(), "/index.1/1.ready/attribute/attr1/snapshot-X"); v = "xxxyyyy/zzz/index.1/1.ready/attribute/attr1/snapshot-X/attr1"; - EXPECT_EQUAL(v.getAttributeName(), "attr1"); - EXPECT_EQUAL(v.getDirName(), + EXPECT_EQ(v.getAttributeName(), "attr1"); + EXPECT_EQ(v.getDirName(), "xxxyyyy/zzz/index.1/1.ready/attribute/attr1/snapshot-X"); } @@ -426,7 +437,7 @@ void AttributeTest::populateSimple(IntegerAttribute & v, uint32_t docIdLow, uint template <typename VectorType, typename BufferType> void AttributeTest::compare(VectorType & a, VectorType & b) { - EXPECT_EQUAL(a.getNumDocs(), b.getNumDocs()); + EXPECT_EQ(a.getNumDocs(), b.getNumDocs()); ASSERT_TRUE(a.getNumDocs() == b.getNumDocs()); uint32_t asz(a.getMaxValueCount()); uint32_t bsz(b.getMaxValueCount()); @@ -436,10 +447,10 @@ void AttributeTest::compare(VectorType & a, VectorType & b) for (size_t i(0), m(a.getNumDocs()); i < m; i++) { ASSERT_TRUE(asz >= static_cast<uint32_t>(a.getValueCount(i))); ASSERT_TRUE(bsz >= static_cast<uint32_t>(b.getValueCount(i))); - EXPECT_EQUAL(a.getValueCount(i), b.getValueCount(i)); + EXPECT_EQ(a.getValueCount(i), b.getValueCount(i)); ASSERT_TRUE(a.getValueCount(i) == b.getValueCount(i)); - EXPECT_EQUAL(static_cast<const AttributeVector &>(a).get(i, av, asz), static_cast<uint32_t>(a.getValueCount(i))); - EXPECT_EQUAL(static_cast<const AttributeVector &>(b).get(i, bv, bsz), static_cast<uint32_t>(b.getValueCount(i))); + EXPECT_EQ(static_cast<const AttributeVector &>(a).get(i, av, asz), static_cast<uint32_t>(a.getValueCount(i))); + EXPECT_EQ(static_cast<const AttributeVector &>(b).get(i, bv, bsz), static_cast<uint32_t>(b.getValueCount(i))); const size_t min_common_value_count = std::min(a.getValueCount(i), b.getValueCount(i)); if (a.hasWeightedSetType()) { ASSERT_TRUE(b.hasWeightedSetType()); @@ -447,7 +458,7 @@ void AttributeTest::compare(VectorType & a, VectorType & b) std::sort(bv, bv + min_common_value_count, order_by_value()); } for(size_t j = 0; j < min_common_value_count; j++) { - EXPECT_EQUAL(av[j], bv[j]); + EXPECT_EQ(av[j], bv[j]); } } delete [] bv; @@ -489,19 +500,19 @@ void AttributeTest::testReload(const AttributePtr & a) EXPECT_TRUE( a->save(b->getBaseFileName()) ); a->commit(true); if (preciseEstimatedSize(*a)) { - EXPECT_EQUAL(statSize(*b), a->getEstimatedSaveByteSize()); + EXPECT_EQ(statSize(*b), a->getEstimatedSaveByteSize()); } else { double estSize = a->getEstimatedSaveByteSize(); double actSize = statSize(*b); - EXPECT_LESS_EQUAL(actSize * 1.0, estSize * 1.3); - EXPECT_GREATER_EQUAL(actSize * 1.0, estSize * 0.7); + EXPECT_LE(actSize * 1.0, estSize * 1.3); + EXPECT_GE(actSize * 1.0, estSize * 0.7); } EXPECT_TRUE( a->save(c->getBaseFileName()) ); if (preciseEstimatedSize(*a)) { - EXPECT_EQUAL(statSize(*c), a->getEstimatedSaveByteSize()); + EXPECT_EQ(statSize(*c), a->getEstimatedSaveByteSize()); } EXPECT_TRUE( b->load() ); - EXPECT_EQUAL(43u, b->getCreateSerialNum()); + EXPECT_EQ(43u, b->getCreateSerialNum()); compare<VectorType, BufferType> (*(static_cast<VectorType *>(a.get())), *(static_cast<VectorType *>(b.get()))); EXPECT_TRUE( c->load() ); @@ -777,10 +788,17 @@ AttributeTest::checkCount(const AttributePtr & ptr, uint32_t doc, uint32_t value uint32_t numValues, const BufferType & value) { std::vector<BufferType> buffer(valueCount); - if (!EXPECT_EQUAL(valueCount, ptr->getValueCount(doc))) return false; - if (!EXPECT_EQUAL(valueCount, ptr->get(doc, &buffer[0], buffer.size()))) return false; - if (!EXPECT_EQUAL(numValues, static_cast<uint32_t>(std::count(buffer.begin(), buffer.end(), value)))) return false; - return true; + bool result = true; + EXPECT_EQ(valueCount, ptr->getValueCount(doc)) << (result = false, ""); + if (!result) { + return false; + } + EXPECT_EQ(valueCount, ptr->get(doc, &buffer[0], buffer.size())) << (result = false, ""); + if (!result) { + return false; + } + EXPECT_EQ(numValues, static_cast<uint32_t>(std::count(buffer.begin(), buffer.end(), value))) << (result = false, ""); + return result; } template <typename BufferType> @@ -852,7 +870,7 @@ AttributeTest::testSingle(const AttributePtr & ptr, const std::vector<BufferType ptr->clearDoc(doc); } ptr->commit(); - EXPECT_EQUAL(1u, ptr->get(doc, &buffer[0], buffer.size())); + EXPECT_EQ(1u, ptr->get(doc, &buffer[0], buffer.size())); if (doc % 2 == 0) { if (smallUInt) { expectZero(buffer[0]); @@ -861,7 +879,7 @@ AttributeTest::testSingle(const AttributePtr & ptr, const std::vector<BufferType } } else { EXPECT_TRUE(!attribute::isUndefined<BaseType>(buffer[0])); - EXPECT_EQUAL(values[i], buffer[0]); + EXPECT_EQ(values[i], buffer[0]); } } EXPECT_TRUE(!v.clearDoc(ptr->getNumDocs())); @@ -949,8 +967,8 @@ AttributeTest::testArray(const AttributePtr & ptr, const std::vector<BufferType> // test update() - EXPECT_EQUAL(ptr->getStatus().getUpdateCount(), 0u); - EXPECT_EQUAL(ptr->getStatus().getNonIdempotentUpdateCount(), 0u); + EXPECT_EQ(ptr->getStatus().getUpdateCount(), 0u); + EXPECT_EQ(ptr->getStatus().getNonIdempotentUpdateCount(), 0u); size_t sumAppends(0); for (uint32_t doc = 0; doc < ptr->getNumDocs(); ++doc) { uint32_t valueCount = doc % numUniques; @@ -966,8 +984,8 @@ AttributeTest::testArray(const AttributePtr & ptr, const std::vector<BufferType> EXPECT_TRUE(checkCount(ptr, doc, 1, 1, values[i])); } EXPECT_TRUE(!v.update(ptr->getNumDocs(), values[0])); - EXPECT_EQUAL(ptr->getStatus().getUpdateCount(), (1 + 2)*ptr->getNumDocs() + sumAppends); - EXPECT_EQUAL(ptr->getStatus().getNonIdempotentUpdateCount(), sumAppends); + EXPECT_EQ(ptr->getStatus().getUpdateCount(), (1 + 2)*ptr->getNumDocs() + sumAppends); + EXPECT_EQ(ptr->getStatus().getNonIdempotentUpdateCount(), sumAppends); // test append() @@ -1131,8 +1149,8 @@ AttributeTest::testWeightedSet(const AttributePtr & ptr, const std::vector<Buffe std::sort(ordered_values.begin(), ordered_values.end(), order_by_weight()); // fill and check - EXPECT_EQUAL(ptr->getStatus().getUpdateCount(), 0u); - EXPECT_EQUAL(ptr->getStatus().getNonIdempotentUpdateCount(), 0u); + EXPECT_EQ(ptr->getStatus().getUpdateCount(), 0u); + EXPECT_EQ(ptr->getStatus().getNonIdempotentUpdateCount(), 0u); for (uint32_t doc = 0; doc < numDocs; ++doc) { uint32_t valueCount = doc; v.clearDoc(doc); @@ -1147,8 +1165,8 @@ AttributeTest::testWeightedSet(const AttributePtr & ptr, const std::vector<Buffe EXPECT_TRUE(buffer[j].getWeight() == ordered_values[j].getWeight()); } } - EXPECT_EQUAL(ptr->getStatus().getUpdateCount(), numDocs + (numDocs*(numDocs-1))/2); - EXPECT_EQUAL(ptr->getStatus().getNonIdempotentUpdateCount(), 0u); + EXPECT_EQ(ptr->getStatus().getUpdateCount(), numDocs + (numDocs*(numDocs-1))/2); + EXPECT_EQ(ptr->getStatus().getNonIdempotentUpdateCount(), 0u); // test append() for (uint32_t doc = 0; doc < numDocs; ++doc) { @@ -1173,8 +1191,8 @@ AttributeTest::testWeightedSet(const AttributePtr & ptr, const std::vector<Buffe ASSERT_TRUE(ptr->get(doc, &buffer[0], buffer.size()) == valueCount + 2); EXPECT_TRUE(contains(buffer, valueCount + 2, BufferType(values[doc + 1].getValue(), values[doc + 1].getWeight() + 10))); } - EXPECT_EQUAL(ptr->getStatus().getUpdateCount(), numDocs + (numDocs*(numDocs-1))/2 + numDocs*4); - EXPECT_EQUAL(ptr->getStatus().getNonIdempotentUpdateCount(), 0u); + EXPECT_EQ(ptr->getStatus().getUpdateCount(), numDocs + (numDocs*(numDocs-1))/2 + numDocs*4); + EXPECT_EQ(ptr->getStatus().getNonIdempotentUpdateCount(), 0u); // test remove() for (uint32_t doc = 0; doc < numDocs; ++doc) { @@ -1194,8 +1212,8 @@ AttributeTest::testWeightedSet(const AttributePtr & ptr, const std::vector<Buffe ASSERT_TRUE(ptr->get(doc, &buffer[0], buffer.size()) == valueCount + 1); EXPECT_FALSE(contains_value(buffer, valueCount + 1, values[doc + 1].getValue())); } - EXPECT_EQUAL(ptr->getStatus().getUpdateCount(), numDocs + (numDocs*(numDocs-1))/2 + numDocs*4 + numDocs * 2); - EXPECT_EQUAL(ptr->getStatus().getNonIdempotentUpdateCount(), 0u); + EXPECT_EQ(ptr->getStatus().getUpdateCount(), numDocs + (numDocs*(numDocs-1))/2 + numDocs*4 + numDocs * 2); + EXPECT_EQ(ptr->getStatus().getNonIdempotentUpdateCount(), 0u); } void @@ -1223,8 +1241,8 @@ AttributeTest::testWeightedSet() testWeightedSet<IntegerAttribute, AttributeVector::WeightedInt>(ptr, values); IAttributeVector::EnumHandle e; EXPECT_TRUE(ptr->findEnum("1", e)); - EXPECT_EQUAL(1u, ptr->findFoldedEnums("1").size()); - EXPECT_EQUAL(e, ptr->findFoldedEnums("1")[0]); + EXPECT_EQ(1u, ptr->findFoldedEnums("1").size()); + EXPECT_EQ(e, ptr->findFoldedEnums("1")[0]); } } @@ -1249,8 +1267,8 @@ AttributeTest::testWeightedSet() testWeightedSet<FloatingPointAttribute, AttributeVector::WeightedFloat>(ptr, values); IAttributeVector::EnumHandle e; EXPECT_TRUE(ptr->findEnum("1", e)); - EXPECT_EQUAL(1u, ptr->findFoldedEnums("1").size()); - EXPECT_EQUAL(e, ptr->findFoldedEnums("1")[0]); + EXPECT_EQ(1u, ptr->findFoldedEnums("1").size()); + EXPECT_EQ(e, ptr->findFoldedEnums("1")[0]); } } { // StringAttribute @@ -1275,8 +1293,8 @@ AttributeTest::testWeightedSet() testWeightedSet<StringAttribute, AttributeVector::WeightedString>(ptr, values); IAttributeVector::EnumHandle e; EXPECT_TRUE(ptr->findEnum("string00", e)); - EXPECT_EQUAL(1u, ptr->findFoldedEnums("StRiNg00").size()); - EXPECT_EQUAL(e, ptr->findFoldedEnums("StRiNg00")[0]); + EXPECT_EQ(1u, ptr->findFoldedEnums("StRiNg00").size()); + EXPECT_EQ(e, ptr->findFoldedEnums("StRiNg00")[0]); } } } @@ -1290,13 +1308,13 @@ AttributeTest::testArithmeticValueUpdate(const AttributePtr & ptr) typedef document::ArithmeticValueUpdate Arith; auto & vec = static_cast<VectorType &>(*ptr.get()); addDocs(ptr, 13); - EXPECT_EQUAL(ptr->getStatus().getUpdateCount(), 0u); - EXPECT_EQUAL(ptr->getStatus().getNonIdempotentUpdateCount(), 0u); + EXPECT_EQ(ptr->getStatus().getUpdateCount(), 0u); + EXPECT_EQ(ptr->getStatus().getNonIdempotentUpdateCount(), 0u); for (uint32_t doc = 0; doc < 13; ++doc) { ASSERT_TRUE(vec.update(doc, 100)); } - EXPECT_EQUAL(ptr->getStatus().getUpdateCount(), 13u); - EXPECT_EQUAL(ptr->getStatus().getNonIdempotentUpdateCount(), 0u); + EXPECT_EQ(ptr->getStatus().getUpdateCount(), 13u); + EXPECT_EQ(ptr->getStatus().getNonIdempotentUpdateCount(), 0u); ptr->commit(); EXPECT_TRUE(vec.apply(0, Arith(Arith::Add, 10))); @@ -1312,73 +1330,73 @@ AttributeTest::testArithmeticValueUpdate(const AttributePtr & ptr) EXPECT_TRUE(vec.apply(10, Arith(Arith::Mul, 1.2))); EXPECT_TRUE(vec.apply(11, Arith(Arith::Mul, 0.8))); EXPECT_TRUE(vec.apply(12, Arith(Arith::Div, 0.8))); - EXPECT_EQUAL(ptr->getStatus().getUpdateCount(), 26u); - EXPECT_EQUAL(ptr->getStatus().getNonIdempotentUpdateCount(), 13u); + EXPECT_EQ(ptr->getStatus().getUpdateCount(), 26u); + EXPECT_EQ(ptr->getStatus().getNonIdempotentUpdateCount(), 13u); ptr->commit(); std::vector<BufferType> buf(1); ptr->get(0, &buf[0], 1); - EXPECT_EQUAL(buf[0], 110); + EXPECT_EQ(buf[0], 110); ptr->get(1, &buf[0], 1); - EXPECT_EQUAL(buf[0], 90); + EXPECT_EQ(buf[0], 90); ptr->get(2, &buf[0], 1); - EXPECT_EQUAL(buf[0], 90); + EXPECT_EQ(buf[0], 90); ptr->get(3, &buf[0], 1); - EXPECT_EQUAL(buf[0], 110); + EXPECT_EQ(buf[0], 110); ptr->get(4, &buf[0], 1); - EXPECT_EQUAL(buf[0], 1000); + EXPECT_EQ(buf[0], 1000); ptr->get(5, &buf[0], 1); - EXPECT_EQUAL(buf[0], -1000); + EXPECT_EQ(buf[0], -1000); ptr->get(6, &buf[0], 1); - EXPECT_EQUAL(buf[0], 10); + EXPECT_EQ(buf[0], 10); ptr->get(7, &buf[0], 1); - EXPECT_EQUAL(buf[0], -10); + EXPECT_EQ(buf[0], -10); if (ptr->getBasicType() == BasicType::INT32) { ptr->get(8, &buf[0], 1); - EXPECT_EQUAL(buf[0], 110); + EXPECT_EQ(buf[0], 110); ptr->get(9, &buf[0], 1); - EXPECT_EQUAL(buf[0], 90); + EXPECT_EQ(buf[0], 90); } else if (ptr->getBasicType() == BasicType::FLOAT || ptr->getBasicType() == BasicType::DOUBLE) { ptr->get(8, &buf[0], 1); - EXPECT_EQUAL(buf[0], 110.5); + EXPECT_EQ(buf[0], 110.5); ptr->get(9, &buf[0], 1); - EXPECT_EQUAL(buf[0], 89.5); + EXPECT_EQ(buf[0], 89.5); } else { ASSERT_TRUE(false); } ptr->get(10, &buf[0], 1); - EXPECT_EQUAL(buf[0], 120); + EXPECT_EQ(buf[0], 120); ptr->get(11, &buf[0], 1); - EXPECT_EQUAL(buf[0], 80); + EXPECT_EQ(buf[0], 80); ptr->get(12, &buf[0], 1); - EXPECT_EQUAL(buf[0], 125); + EXPECT_EQ(buf[0], 125); // try several arithmetic operations on the same document in a single commit ASSERT_TRUE(vec.update(0, 1100)); ASSERT_TRUE(vec.update(1, 1100)); - EXPECT_EQUAL(ptr->getStatus().getUpdateCount(), 28u); - EXPECT_EQUAL(ptr->getStatus().getNonIdempotentUpdateCount(), 13u); + EXPECT_EQ(ptr->getStatus().getUpdateCount(), 28u); + EXPECT_EQ(ptr->getStatus().getNonIdempotentUpdateCount(), 13u); for (uint32_t i = 0; i < 10; ++i) { ASSERT_TRUE(vec.apply(0, Arith(Arith::Add, 10))); ASSERT_TRUE(vec.apply(1, Arith(Arith::Add, 10))); } - EXPECT_EQUAL(ptr->getStatus().getUpdateCount(), 48u); - EXPECT_EQUAL(ptr->getStatus().getNonIdempotentUpdateCount(), 33u); + EXPECT_EQ(ptr->getStatus().getUpdateCount(), 48u); + EXPECT_EQ(ptr->getStatus().getNonIdempotentUpdateCount(), 33u); ptr->commit(); ptr->get(0, &buf[0], 1); - EXPECT_EQUAL(buf[0], 1200); + EXPECT_EQ(buf[0], 1200); ptr->get(1, &buf[0], 1); - EXPECT_EQUAL(buf[0], 1200); + EXPECT_EQ(buf[0], 1200); ASSERT_TRUE(vec.update(0, 10)); ASSERT_TRUE(vec.update(1, 10)); ASSERT_TRUE(vec.update(2, 10)); ASSERT_TRUE(vec.update(3, 10)); - EXPECT_EQUAL(ptr->getStatus().getUpdateCount(), 52u); - EXPECT_EQUAL(ptr->getStatus().getNonIdempotentUpdateCount(), 33u); + EXPECT_EQ(ptr->getStatus().getUpdateCount(), 52u); + EXPECT_EQ(ptr->getStatus().getNonIdempotentUpdateCount(), 33u); for (uint32_t i = 0; i < 8; ++i) { EXPECT_TRUE(vec.apply(0, Arith(Arith::Mul, 1.2))); EXPECT_TRUE(vec.apply(1, Arith(Arith::Mul, 2.3))); @@ -1386,8 +1404,8 @@ AttributeTest::testArithmeticValueUpdate(const AttributePtr & ptr) EXPECT_TRUE(vec.apply(3, Arith(Arith::Mul, 5.6))); ptr->commit(); } - EXPECT_EQUAL(ptr->getStatus().getUpdateCount(), 84u); - EXPECT_EQUAL(ptr->getStatus().getNonIdempotentUpdateCount(), 65u); + EXPECT_EQ(ptr->getStatus().getUpdateCount(), 84u); + EXPECT_EQ(ptr->getStatus().getNonIdempotentUpdateCount(), 65u); // try divide by zero @@ -1395,26 +1413,26 @@ AttributeTest::testArithmeticValueUpdate(const AttributePtr & ptr) EXPECT_TRUE(vec.apply(0, Arith(Arith::Div, 0))); ptr->commit(); if (ptr->getClass().inherits(FloatingPointAttribute::classId)) { - EXPECT_EQUAL(ptr->getStatus().getUpdateCount(), 86u); - EXPECT_EQUAL(ptr->getStatus().getNonIdempotentUpdateCount(), 66u); + EXPECT_EQ(ptr->getStatus().getUpdateCount(), 86u); + EXPECT_EQ(ptr->getStatus().getNonIdempotentUpdateCount(), 66u); } else { // does not apply for interger attributes - EXPECT_EQUAL(ptr->getStatus().getUpdateCount(), 85u); - EXPECT_EQUAL(ptr->getStatus().getNonIdempotentUpdateCount(), 65u); + EXPECT_EQ(ptr->getStatus().getUpdateCount(), 85u); + EXPECT_EQ(ptr->getStatus().getNonIdempotentUpdateCount(), 65u); } ptr->get(0, &buf[0], 1); if (ptr->getBasicType() == BasicType::INT32) { - EXPECT_EQUAL(buf[0], 100); + EXPECT_EQ(buf[0], 100); } // try divide by zero with empty change vector EXPECT_TRUE(vec.apply(0, Arith(Arith::Div, 0))); ptr->commit(); if (ptr->getClass().inherits(FloatingPointAttribute::classId)) { - EXPECT_EQUAL(ptr->getStatus().getUpdateCount(), 87u); - EXPECT_EQUAL(ptr->getStatus().getNonIdempotentUpdateCount(), 67u); + EXPECT_EQ(ptr->getStatus().getUpdateCount(), 87u); + EXPECT_EQ(ptr->getStatus().getNonIdempotentUpdateCount(), 67u); } else { // does not apply for interger attributes - EXPECT_EQUAL(ptr->getStatus().getUpdateCount(), 85u); - EXPECT_EQUAL(ptr->getStatus().getNonIdempotentUpdateCount(), 65u); + EXPECT_EQ(ptr->getStatus().getUpdateCount(), 85u); + EXPECT_EQ(ptr->getStatus().getNonIdempotentUpdateCount(), 65u); } } @@ -1471,7 +1489,7 @@ AttributeTest::testArithmeticWithUndefinedValue(const AttributePtr & ptr, BaseTy if (ptr->getClass().inherits(FloatingPointAttribute::classId)) { EXPECT_TRUE(std::isnan(buf[0])); } else { - EXPECT_EQUAL(buf[0], after); + EXPECT_EQ(buf[0], after); } } @@ -1511,8 +1529,8 @@ AttributeTest::testMapValueUpdate(const AttributePtr & ptr, BufferType initValue for (uint32_t doc = 0; doc < 7; ++doc) { ASSERT_TRUE(vec.append(doc, initValue.getValue(), 100)); } - EXPECT_EQUAL(ptr->getStatus().getUpdateCount(), 7u); - EXPECT_EQUAL(ptr->getStatus().getNonIdempotentUpdateCount(), 0u); + EXPECT_EQ(ptr->getStatus().getUpdateCount(), 7u); + EXPECT_EQ(ptr->getStatus().getNonIdempotentUpdateCount(), 0u); EXPECT_TRUE(ptr->apply(0, MapVU(std::unique_ptr<FieldValue>(initFieldValue.clone()), std::make_unique<ArithVU>(ArithVU::Add, 10)))); EXPECT_TRUE(ptr->apply(1, MapVU(std::unique_ptr<FieldValue>(initFieldValue.clone()), std::make_unique<ArithVU>(ArithVU::Sub, 10)))); @@ -1520,60 +1538,60 @@ AttributeTest::testMapValueUpdate(const AttributePtr & ptr, BufferType initValue EXPECT_TRUE(ptr->apply(3, MapVU(std::unique_ptr<FieldValue>(initFieldValue.clone()), std::make_unique<ArithVU>(ArithVU::Div, 10)))); EXPECT_TRUE(ptr->apply(6, MapVU(std::unique_ptr<FieldValue>(initFieldValue.clone()), std::make_unique<AssignValueUpdate>(std::make_unique<IntFieldValue>(70))))); ptr->commit(); - EXPECT_EQUAL(ptr->getStatus().getUpdateCount(), 12u); - EXPECT_EQUAL(ptr->getStatus().getNonIdempotentUpdateCount(), 5u); + EXPECT_EQ(ptr->getStatus().getUpdateCount(), 12u); + EXPECT_EQ(ptr->getStatus().getNonIdempotentUpdateCount(), 5u); std::vector<BufferType> buf(2); ptr->get(0, &buf[0], 2); - EXPECT_EQUAL(buf[0].getWeight(), 110); + EXPECT_EQ(buf[0].getWeight(), 110); ptr->get(1, &buf[0], 2); - EXPECT_EQUAL(buf[0].getWeight(), 90); + EXPECT_EQ(buf[0].getWeight(), 90); ptr->get(2, &buf[0], 2); - EXPECT_EQUAL(buf[0].getWeight(), 1000); + EXPECT_EQ(buf[0].getWeight(), 1000); ptr->get(3, &buf[0], 2); - EXPECT_EQUAL(buf[0].getWeight(), 10); + EXPECT_EQ(buf[0].getWeight(), 10); ptr->get(6, &buf[0], 2); - EXPECT_EQUAL(buf[0].getWeight(), 70); + EXPECT_EQ(buf[0].getWeight(), 70); // removeifzero EXPECT_TRUE(ptr->apply(4, MapVU(std::unique_ptr<FieldValue>(initFieldValue.clone()), std::make_unique<ArithVU>(ArithVU::Sub, 100)))); ptr->commit(); if (removeIfZero) { - EXPECT_EQUAL(ptr->get(4, &buf[0], 2), uint32_t(0)); + EXPECT_EQ(ptr->get(4, &buf[0], 2), uint32_t(0)); } else { - EXPECT_EQUAL(ptr->get(4, &buf[0], 2), uint32_t(1)); - EXPECT_EQUAL(buf[0].getWeight(), 0); + EXPECT_EQ(ptr->get(4, &buf[0], 2), uint32_t(1)); + EXPECT_EQ(buf[0].getWeight(), 0); } - EXPECT_EQUAL(ptr->getStatus().getUpdateCount(), 13u); - EXPECT_EQUAL(ptr->getStatus().getNonIdempotentUpdateCount(), 6u); + EXPECT_EQ(ptr->getStatus().getUpdateCount(), 13u); + EXPECT_EQ(ptr->getStatus().getNonIdempotentUpdateCount(), 6u); // createifnonexistant EXPECT_TRUE(ptr->apply(5, MapVU(std::unique_ptr<FieldValue>(nonExistant.clone()), std::make_unique<ArithVU>(ArithVU::Add, 10)))); ptr->commit(); if (createIfNonExistant) { - EXPECT_EQUAL(ptr->get(5, &buf[0], 2), uint32_t(2)); + EXPECT_EQ(ptr->get(5, &buf[0], 2), uint32_t(2)); std::sort(buf.begin(), buf.begin() + 2, order_by_weight()); - EXPECT_EQUAL(buf[0].getWeight(), 10); - EXPECT_EQUAL(buf[1].getWeight(), 100); + EXPECT_EQ(buf[0].getWeight(), 10); + EXPECT_EQ(buf[1].getWeight(), 100); } else { - EXPECT_EQUAL(ptr->get(5, &buf[0], 2), uint32_t(1)); - EXPECT_EQUAL(buf[0].getWeight(), 100); + EXPECT_EQ(ptr->get(5, &buf[0], 2), uint32_t(1)); + EXPECT_EQ(buf[0].getWeight(), 100); } - EXPECT_EQUAL(ptr->getStatus().getUpdateCount(), 14u); - EXPECT_EQUAL(ptr->getStatus().getNonIdempotentUpdateCount(), 7u); + EXPECT_EQ(ptr->getStatus().getUpdateCount(), 14u); + EXPECT_EQ(ptr->getStatus().getNonIdempotentUpdateCount(), 7u); // try divide by zero (should be ignored) vec.clearDoc(0); - EXPECT_EQUAL(ptr->getStatus().getUpdateCount(), 15u); + EXPECT_EQ(ptr->getStatus().getUpdateCount(), 15u); ASSERT_TRUE(vec.append(0, initValue.getValue(), 12345)); - EXPECT_EQUAL(ptr->getStatus().getUpdateCount(), 16u); + EXPECT_EQ(ptr->getStatus().getUpdateCount(), 16u); EXPECT_TRUE(ptr->apply(0, MapVU(std::unique_ptr<FieldValue>(initFieldValue.clone()), std::make_unique<ArithVU>(ArithVU::Div, 0)))); - EXPECT_EQUAL(ptr->getStatus().getUpdateCount(), 16u); - EXPECT_EQUAL(ptr->getStatus().getNonIdempotentUpdateCount(), 7u); + EXPECT_EQ(ptr->getStatus().getUpdateCount(), 16u); + EXPECT_EQ(ptr->getStatus().getNonIdempotentUpdateCount(), 7u); ptr->commit(); ptr->get(0, &buf[0], 1); - EXPECT_EQUAL(buf[0].getWeight(), 12345); + EXPECT_EQ(buf[0].getWeight(), 12345); } void @@ -1664,16 +1682,16 @@ AttributeTest::testStatus() EXPECT_TRUE(appendToVector(sa, i, 1, values)); } ptr->commit(true); - EXPECT_EQUAL(ptr->getStatus().getNumDocs(), 100u); - EXPECT_EQUAL(ptr->getStatus().getNumValues(), 100u); - EXPECT_EQUAL(ptr->getStatus().getNumUniqueValues(), 1u); + EXPECT_EQ(ptr->getStatus().getNumDocs(), 100u); + EXPECT_EQ(ptr->getStatus().getNumValues(), 100u); + EXPECT_EQ(ptr->getStatus().getNumUniqueValues(), 1u); size_t expUsed = 0; expUsed += 1 * InternalNodeSize + 1 * LeafNodeSize; // enum store tree expUsed += 1 * 32; // enum store (uniquevalues * bytes per entry) // multi value mapping (numdocs * sizeof(MappingIndex) + numvalues * sizeof(EnumIndex)) expUsed += 100 * sizeof(vespalib::datastore::EntryRef) + 100 * 4; - EXPECT_GREATER_EQUAL(ptr->getStatus().getUsed(), expUsed); - EXPECT_GREATER_EQUAL(ptr->getStatus().getAllocated(), expUsed); + EXPECT_GE(ptr->getStatus().getUsed(), expUsed); + EXPECT_GE(ptr->getStatus().getAllocated(), expUsed); } { @@ -1687,17 +1705,17 @@ AttributeTest::testStatus() EXPECT_TRUE(appendToVector(sa, i, numValuesPerDoc, values)); } ptr->commit(true); - EXPECT_EQUAL(ptr->getStatus().getNumDocs(), numDocs); - EXPECT_EQUAL(ptr->getStatus().getNumValues(), numDocs*numValuesPerDoc); - EXPECT_EQUAL(ptr->getStatus().getNumUniqueValues(), numUniq); + EXPECT_EQ(ptr->getStatus().getNumDocs(), numDocs); + EXPECT_EQ(ptr->getStatus().getNumValues(), numDocs*numValuesPerDoc); + EXPECT_EQ(ptr->getStatus().getNumUniqueValues(), numUniq); size_t expUsed = 0; expUsed += 1 * InternalNodeSize + 1 * LeafNodeSize; // Approximate enum store tree expUsed += 272; // TODO Approximate... enum store (16 unique values, 17 bytes per entry) // multi value mapping (numdocs * sizeof(MappingIndex) + numvalues * sizeof(EnumIndex) + // 32 + numdocs * sizeof(Array<EnumIndex>) (due to vector vector)) expUsed += 32 + numDocs * sizeof(vespalib::datastore::EntryRef) + numDocs * numValuesPerDoc * sizeof(IEnumStore::Index) + ((numValuesPerDoc > 1024) ? numDocs * NestedVectorSize : 0); - EXPECT_GREATER_EQUAL(ptr->getStatus().getUsed(), expUsed); - EXPECT_GREATER_EQUAL(ptr->getStatus().getAllocated(), expUsed); + EXPECT_GE(ptr->getStatus().getUsed(), expUsed); + EXPECT_GE(ptr->getStatus().getAllocated(), expUsed); } } @@ -1710,16 +1728,16 @@ AttributeTest::testNullProtection() string good("good"); string evil("evil string"); string pureEvil("evil"); - EXPECT_EQUAL(strlen(evil.data()), len); - EXPECT_EQUAL(strlen(evil.c_str()), len); + EXPECT_EQ(strlen(evil.data()), len); + EXPECT_EQ(strlen(evil.c_str()), len); evil[len1] = 0; // replace space with '\0' - EXPECT_EQUAL(strlen(evil.data()), len1); - EXPECT_EQUAL(strlen(evil.c_str()), len1); - EXPECT_EQUAL(strlen(evil.data() + len1), 0u); - EXPECT_EQUAL(strlen(evil.c_str() + len1), 0u); - EXPECT_EQUAL(strlen(evil.data() + len1 + 1), len2); - EXPECT_EQUAL(strlen(evil.c_str() + len1 + 1), len2); - EXPECT_EQUAL(evil.size(), len); + EXPECT_EQ(strlen(evil.data()), len1); + EXPECT_EQ(strlen(evil.c_str()), len1); + EXPECT_EQ(strlen(evil.data() + len1), 0u); + EXPECT_EQ(strlen(evil.c_str() + len1), 0u); + EXPECT_EQ(strlen(evil.data() + len1 + 1), len2); + EXPECT_EQ(strlen(evil.c_str() + len1 + 1), len2); + EXPECT_EQ(evil.size(), len); { // string AttributeVector::DocId docId; std::vector<string> buf(16); @@ -1729,8 +1747,8 @@ AttributeTest::testNullProtection() EXPECT_TRUE(v.update(docId, evil)); v.commit(); size_t n = static_cast<const AttributeVector &>(v).get(docId, &buf[0], buf.size()); - EXPECT_EQUAL(n, 1u); - EXPECT_EQUAL(buf[0], pureEvil); + EXPECT_EQ(n, 1u); + EXPECT_EQ(buf[0], pureEvil); } { // string array AttributeVector::DocId docId; @@ -1743,10 +1761,10 @@ AttributeTest::testNullProtection() EXPECT_TRUE(v.append(0, good, 1)); v.commit(); size_t n = static_cast<const AttributeVector &>(v).get(0, &buf[0], buf.size()); - EXPECT_EQUAL(n, 3u); - EXPECT_EQUAL(buf[0], good); - EXPECT_EQUAL(buf[1], pureEvil); - EXPECT_EQUAL(buf[2], good); + EXPECT_EQ(n, 3u); + EXPECT_EQ(buf[0], good); + EXPECT_EQ(buf[1], pureEvil); + EXPECT_EQ(buf[2], good); } { // string set AttributeVector::DocId docId; @@ -1758,22 +1776,22 @@ AttributeTest::testNullProtection() EXPECT_TRUE(v.append(0, evil, 20)); v.commit(); size_t n = static_cast<const AttributeVector &>(v).get(0, &buf[0], buf.size()); - EXPECT_EQUAL(n, 2u); + EXPECT_EQ(n, 2u); if (buf[0].getValue() != good) { std::swap(buf[0], buf[1]); } - EXPECT_EQUAL(buf[0].getValue(), good); - EXPECT_EQUAL(buf[0].getWeight(), 10); - EXPECT_EQUAL(buf[1].getValue(), pureEvil); - EXPECT_EQUAL(buf[1].getWeight(), 20); + EXPECT_EQ(buf[0].getValue(), good); + EXPECT_EQ(buf[0].getWeight(), 10); + EXPECT_EQ(buf[1].getValue(), pureEvil); + EXPECT_EQ(buf[1].getWeight(), 20); // remove EXPECT_TRUE(v.remove(0, evil, 20)); v.commit(); n = static_cast<const AttributeVector &>(v).get(0, &buf[0], buf.size()); - EXPECT_EQUAL(n, 1u); - EXPECT_EQUAL(buf[0].getValue(), good); - EXPECT_EQUAL(buf[0].getWeight(), 10); + EXPECT_EQ(n, 1u); + EXPECT_EQ(buf[0].getValue(), good); + EXPECT_EQ(buf[0].getWeight(), 10); } } @@ -1784,54 +1802,54 @@ AttributeTest::testGeneration(const AttributePtr & attr, bool exactStatus) auto & ia = static_cast<IntegerAttribute &>(*attr.get()); // add docs to trigger inc generation when data vector is full AttributeVector::DocId docId; - EXPECT_EQUAL(0u, ia.getCurrentGeneration()); + EXPECT_EQ(0u, ia.getCurrentGeneration()); EXPECT_TRUE(ia.addDoc(docId)); - EXPECT_EQUAL(0u, ia.getCurrentGeneration()); + EXPECT_EQ(0u, ia.getCurrentGeneration()); EXPECT_TRUE(ia.addDoc(docId)); - EXPECT_EQUAL(0u, ia.getCurrentGeneration()); + EXPECT_EQ(0u, ia.getCurrentGeneration()); ia.commit(true); - EXPECT_EQUAL(1u, ia.getCurrentGeneration()); + EXPECT_EQ(1u, ia.getCurrentGeneration()); uint64_t lastAllocated; uint64_t lastOnHold; vespalib::MemoryUsage changeVectorMemoryUsage(attr->getChangeVectorMemoryUsage()); size_t changeVectorAllocated = changeVectorMemoryUsage.allocatedBytes(); if (exactStatus) { - EXPECT_EQUAL(2u + changeVectorAllocated, ia.getStatus().getAllocated()); - EXPECT_EQUAL(0u, ia.getStatus().getOnHold()); + EXPECT_EQ(2u + changeVectorAllocated, ia.getStatus().getAllocated()); + EXPECT_EQ(0u, ia.getStatus().getOnHold()); } else { - EXPECT_LESS(0u + changeVectorAllocated, ia.getStatus().getAllocated()); - EXPECT_EQUAL(0u, ia.getStatus().getOnHold()); + EXPECT_LT(0u + changeVectorAllocated, ia.getStatus().getAllocated()); + EXPECT_EQ(0u, ia.getStatus().getOnHold()); lastAllocated = ia.getStatus().getAllocated(); lastOnHold = ia.getStatus().getOnHold(); } { AttributeGuard ag(attr); // guard on generation 1 EXPECT_TRUE(ia.addDoc(docId)); // inc gen - EXPECT_EQUAL(2u, ia.getCurrentGeneration()); + EXPECT_EQ(2u, ia.getCurrentGeneration()); ia.commit(true); - EXPECT_EQUAL(3u, ia.getCurrentGeneration()); + EXPECT_EQ(3u, ia.getCurrentGeneration()); if (exactStatus) { - EXPECT_EQUAL(6u + changeVectorAllocated, ia.getStatus().getAllocated()); - EXPECT_EQUAL(2u, ia.getStatus().getOnHold()); // no cleanup due to guard + EXPECT_EQ(6u + changeVectorAllocated, ia.getStatus().getAllocated()); + EXPECT_EQ(2u, ia.getStatus().getOnHold()); // no cleanup due to guard } else { - EXPECT_LESS(lastAllocated, ia.getStatus().getAllocated()); - EXPECT_LESS(lastOnHold, ia.getStatus().getOnHold()); + EXPECT_LT(lastAllocated, ia.getStatus().getAllocated()); + EXPECT_LT(lastOnHold, ia.getStatus().getOnHold()); lastAllocated = ia.getStatus().getAllocated(); lastOnHold = ia.getStatus().getOnHold(); } } EXPECT_TRUE(ia.addDoc(docId)); - EXPECT_EQUAL(3u, ia.getCurrentGeneration()); + EXPECT_EQ(3u, ia.getCurrentGeneration()); { AttributeGuard ag(attr); // guard on generation 3 ia.commit(true); - EXPECT_EQUAL(4u, ia.getCurrentGeneration()); + EXPECT_EQ(4u, ia.getCurrentGeneration()); if (exactStatus) { - EXPECT_EQUAL(4u + changeVectorAllocated, ia.getStatus().getAllocated()); - EXPECT_EQUAL(0u, ia.getStatus().getOnHold()); // cleanup at end of addDoc() + EXPECT_EQ(4u + changeVectorAllocated, ia.getStatus().getAllocated()); + EXPECT_EQ(0u, ia.getStatus().getOnHold()); // cleanup at end of addDoc() } else { - EXPECT_GREATER(lastAllocated, ia.getStatus().getAllocated()); - EXPECT_GREATER(lastOnHold, ia.getStatus().getOnHold()); + EXPECT_GT(lastAllocated, ia.getStatus().getAllocated()); + EXPECT_GT(lastOnHold, ia.getStatus().getOnHold()); lastAllocated = ia.getStatus().getAllocated(); lastOnHold = ia.getStatus().getOnHold(); } @@ -1839,27 +1857,27 @@ AttributeTest::testGeneration(const AttributePtr & attr, bool exactStatus) { AttributeGuard ag(attr); // guard on generation 4 EXPECT_TRUE(ia.addDoc(docId)); // inc gen - EXPECT_EQUAL(5u, ia.getCurrentGeneration()); + EXPECT_EQ(5u, ia.getCurrentGeneration()); ia.commit(); - EXPECT_EQUAL(6u, ia.getCurrentGeneration()); + EXPECT_EQ(6u, ia.getCurrentGeneration()); if (exactStatus) { - EXPECT_EQUAL(10u + changeVectorAllocated, ia.getStatus().getAllocated()); - EXPECT_EQUAL(4u, ia.getStatus().getOnHold()); // no cleanup due to guard + EXPECT_EQ(10u + changeVectorAllocated, ia.getStatus().getAllocated()); + EXPECT_EQ(4u, ia.getStatus().getOnHold()); // no cleanup due to guard } else { - EXPECT_LESS(lastAllocated, ia.getStatus().getAllocated()); - EXPECT_LESS(lastOnHold, ia.getStatus().getOnHold()); + EXPECT_LT(lastAllocated, ia.getStatus().getAllocated()); + EXPECT_LT(lastOnHold, ia.getStatus().getOnHold()); lastAllocated = ia.getStatus().getAllocated(); lastOnHold = ia.getStatus().getOnHold(); } } ia.commit(true); - EXPECT_EQUAL(7u, ia.getCurrentGeneration()); + EXPECT_EQ(7u, ia.getCurrentGeneration()); if (exactStatus) { - EXPECT_EQUAL(6u + changeVectorAllocated, ia.getStatus().getAllocated()); - EXPECT_EQUAL(0u, ia.getStatus().getOnHold()); // cleanup at end of commit() + EXPECT_EQ(6u + changeVectorAllocated, ia.getStatus().getAllocated()); + EXPECT_EQ(0u, ia.getStatus().getOnHold()); // cleanup at end of commit() } else { - EXPECT_GREATER(lastAllocated, ia.getStatus().getAllocated()); - EXPECT_GREATER(lastOnHold, ia.getStatus().getOnHold()); + EXPECT_GT(lastAllocated, ia.getStatus().getAllocated()); + EXPECT_GT(lastOnHold, ia.getStatus().getOnHold()); } } @@ -1904,7 +1922,7 @@ AttributeTest::testCreateSerialNum() EXPECT_TRUE(attr->save()); AttributePtr attr2 = createAttribute("int32", cfg); EXPECT_TRUE(attr2->load()); - EXPECT_EQUAL(42u, attr2->getCreateSerialNum()); + EXPECT_EQ(42u, attr2->getCreateSerialNum()); } void @@ -1920,7 +1938,7 @@ AttributeTest::testPredicateHeaderTags() EXPECT_TRUE(datHeader.hasTag("predicate.arity")); EXPECT_TRUE(datHeader.hasTag("predicate.lower_bound")); EXPECT_TRUE(datHeader.hasTag("predicate.upper_bound")); - EXPECT_EQUAL(8u, datHeader.getTag("predicate.arity").asInteger()); + EXPECT_EQ(8u, datHeader.getTag("predicate.arity").asInteger()); } template <typename VectorType, typename BufferType> @@ -1946,25 +1964,25 @@ AttributeTest::testCompactLidSpace(const Config &config, auto &v2 = static_cast<VectorType &>(*attr2.get()); attr2->addDocs(trimmedDocs); populate(v2, 17); - EXPECT_EQUAL(trimmedDocs, attr2->getNumDocs()); - EXPECT_EQUAL(trimmedDocs, attr2->getCommittedDocIdLimit()); - EXPECT_EQUAL(highDocs, attr->getNumDocs()); - EXPECT_EQUAL(highDocs, attr->getCommittedDocIdLimit()); + EXPECT_EQ(trimmedDocs, attr2->getNumDocs()); + EXPECT_EQ(trimmedDocs, attr2->getCommittedDocIdLimit()); + EXPECT_EQ(highDocs, attr->getNumDocs()); + EXPECT_EQ(highDocs, attr->getCommittedDocIdLimit()); attr->compactLidSpace(trimmedDocs); - EXPECT_EQUAL(highDocs, attr->getNumDocs()); - EXPECT_EQUAL(trimmedDocs, attr->getCommittedDocIdLimit()); + EXPECT_EQ(highDocs, attr->getNumDocs()); + EXPECT_EQ(trimmedDocs, attr->getCommittedDocIdLimit()); EXPECT_TRUE(attr->save()); - EXPECT_EQUAL(highDocs, attr->getNumDocs()); - EXPECT_EQUAL(trimmedDocs, attr->getCommittedDocIdLimit()); + EXPECT_EQ(highDocs, attr->getNumDocs()); + EXPECT_EQ(trimmedDocs, attr->getCommittedDocIdLimit()); AttributePtr attr3 = AttributeFactory::createAttribute(name, cfg); EXPECT_TRUE(attr3->load()); - EXPECT_EQUAL(trimmedDocs, attr3->getNumDocs()); - EXPECT_EQUAL(trimmedDocs, attr3->getCommittedDocIdLimit()); + EXPECT_EQ(trimmedDocs, attr3->getNumDocs()); + EXPECT_EQ(trimmedDocs, attr3->getCommittedDocIdLimit()); auto &v3 = static_cast<VectorType &>(*attr3.get()); compare<VectorType, BufferType>(v2, v3); attr->shrinkLidSpace(); - EXPECT_EQUAL(trimmedDocs, attr->getNumDocs()); - EXPECT_EQUAL(trimmedDocs, attr->getCommittedDocIdLimit()); + EXPECT_EQ(trimmedDocs, attr->getNumDocs()); + EXPECT_EQ(trimmedDocs, attr->getCommittedDocIdLimit()); compare<VectorType, BufferType>(v, v3); } @@ -1995,6 +2013,7 @@ AttributeTest::testCompactLidSpaceForPredicateAttribute(const Config &config) void AttributeTest::testCompactLidSpace(const Config &config) { + SCOPED_TRACE(make_scoped_trace_msg("compact lid space", config)); switch (config.basicType().type()) { case BasicType::BOOL: case BasicType::UINT2: @@ -2035,31 +2054,31 @@ AttributeTest::testCompactLidSpace(const Config &config) void AttributeTest::testCompactLidSpace() { - TEST_DO(testCompactLidSpace(Config(BasicType::BOOL, CollectionType::SINGLE))); - TEST_DO(testCompactLidSpace(Config(BasicType::UINT2, CollectionType::SINGLE))); - TEST_DO(testCompactLidSpace(Config(BasicType::UINT4, CollectionType::SINGLE))); - TEST_DO(testCompactLidSpace(Config(BasicType::INT8, CollectionType::SINGLE))); - TEST_DO(testCompactLidSpace(Config(BasicType::INT8, CollectionType::ARRAY))); - TEST_DO(testCompactLidSpace(Config(BasicType::INT8, CollectionType::WSET))); - TEST_DO(testCompactLidSpace(Config(BasicType::INT16, CollectionType::SINGLE))); - TEST_DO(testCompactLidSpace(Config(BasicType::INT16, CollectionType::ARRAY))); - TEST_DO(testCompactLidSpace(Config(BasicType::INT16, CollectionType::WSET))); - TEST_DO(testCompactLidSpace(Config(BasicType::INT32, CollectionType::SINGLE))); - TEST_DO(testCompactLidSpace(Config(BasicType::INT32, CollectionType::ARRAY))); - TEST_DO(testCompactLidSpace(Config(BasicType::INT32, CollectionType::WSET))); - TEST_DO(testCompactLidSpace(Config(BasicType::INT64, CollectionType::SINGLE))); - TEST_DO(testCompactLidSpace(Config(BasicType::INT64, CollectionType::ARRAY))); - TEST_DO(testCompactLidSpace(Config(BasicType::INT64, CollectionType::WSET))); - TEST_DO(testCompactLidSpace(Config(BasicType::FLOAT, CollectionType::SINGLE))); - TEST_DO(testCompactLidSpace(Config(BasicType::FLOAT, CollectionType::ARRAY))); - TEST_DO(testCompactLidSpace(Config(BasicType::FLOAT, CollectionType::WSET))); - TEST_DO(testCompactLidSpace(Config(BasicType::DOUBLE, CollectionType::SINGLE))); - TEST_DO(testCompactLidSpace(Config(BasicType::DOUBLE, CollectionType::ARRAY))); - TEST_DO(testCompactLidSpace(Config(BasicType::DOUBLE, CollectionType::WSET))); - TEST_DO(testCompactLidSpace(Config(BasicType::STRING, CollectionType::SINGLE))); - TEST_DO(testCompactLidSpace(Config(BasicType::STRING, CollectionType::ARRAY))); - TEST_DO(testCompactLidSpace(Config(BasicType::STRING, CollectionType::WSET))); - TEST_DO(testCompactLidSpace(Config(BasicType::PREDICATE, CollectionType::SINGLE))); + testCompactLidSpace(Config(BasicType::BOOL, CollectionType::SINGLE)); + testCompactLidSpace(Config(BasicType::UINT2, CollectionType::SINGLE)); + testCompactLidSpace(Config(BasicType::UINT4, CollectionType::SINGLE)); + testCompactLidSpace(Config(BasicType::INT8, CollectionType::SINGLE)); + testCompactLidSpace(Config(BasicType::INT8, CollectionType::ARRAY)); + testCompactLidSpace(Config(BasicType::INT8, CollectionType::WSET)); + testCompactLidSpace(Config(BasicType::INT16, CollectionType::SINGLE)); + testCompactLidSpace(Config(BasicType::INT16, CollectionType::ARRAY)); + testCompactLidSpace(Config(BasicType::INT16, CollectionType::WSET)); + testCompactLidSpace(Config(BasicType::INT32, CollectionType::SINGLE)); + testCompactLidSpace(Config(BasicType::INT32, CollectionType::ARRAY)); + testCompactLidSpace(Config(BasicType::INT32, CollectionType::WSET)); + testCompactLidSpace(Config(BasicType::INT64, CollectionType::SINGLE)); + testCompactLidSpace(Config(BasicType::INT64, CollectionType::ARRAY)); + testCompactLidSpace(Config(BasicType::INT64, CollectionType::WSET)); + testCompactLidSpace(Config(BasicType::FLOAT, CollectionType::SINGLE)); + testCompactLidSpace(Config(BasicType::FLOAT, CollectionType::ARRAY)); + testCompactLidSpace(Config(BasicType::FLOAT, CollectionType::WSET)); + testCompactLidSpace(Config(BasicType::DOUBLE, CollectionType::SINGLE)); + testCompactLidSpace(Config(BasicType::DOUBLE, CollectionType::ARRAY)); + testCompactLidSpace(Config(BasicType::DOUBLE, CollectionType::WSET)); + testCompactLidSpace(Config(BasicType::STRING, CollectionType::SINGLE)); + testCompactLidSpace(Config(BasicType::STRING, CollectionType::ARRAY)); + testCompactLidSpace(Config(BasicType::STRING, CollectionType::WSET)); + testCompactLidSpace(Config(BasicType::PREDICATE, CollectionType::SINGLE)); } namespace { @@ -2092,12 +2111,12 @@ AttributeTest::test_default_value_ref_count_is_updated_after_shrink_lid_space() const auto & iattr = dynamic_cast<const search::IntegerAttributeTemplate<int32_t> &>(*attr); attr->addReservedDoc(); attr->addDocs(10); - EXPECT_EQUAL(11u, get_default_value_ref_count(*attr, iattr.defaultValue())); + EXPECT_EQ(11u, get_default_value_ref_count(*attr, iattr.defaultValue())); attr->compactLidSpace(6); - EXPECT_EQUAL(11u, get_default_value_ref_count(*attr, iattr.defaultValue())); + EXPECT_EQ(11u, get_default_value_ref_count(*attr, iattr.defaultValue())); attr->shrinkLidSpace(); - EXPECT_EQUAL(6u, attr->getNumDocs()); - EXPECT_EQUAL(6u, get_default_value_ref_count(*attr, iattr.defaultValue())); + EXPECT_EQ(6u, attr->getNumDocs()); + EXPECT_EQ(6u, get_default_value_ref_count(*attr, iattr.defaultValue())); } template <typename AttributeType> @@ -2117,30 +2136,30 @@ AttributeTest::requireThatAddressSpaceUsageIsReported(const Config &config, bool AddressSpaceUsage after = attrPtr->getAddressSpaceUsage(); if (attrPtr->hasEnum()) { LOG(info, "requireThatAddressSpaceUsageIsReported(%s): Has enum", attrName.c_str()); - EXPECT_EQUAL(before.enum_store_usage().used(), 1u); - EXPECT_EQUAL(before.enum_store_usage().dead(), 1u); - EXPECT_GREATER(after.enum_store_usage().used(), before.enum_store_usage().used()); - EXPECT_GREATER_EQUAL(after.enum_store_usage().limit(), before.enum_store_usage().limit()); - EXPECT_GREATER(after.enum_store_usage().limit(), 4200000000u); + EXPECT_EQ(before.enum_store_usage().used(), 1u); + EXPECT_EQ(before.enum_store_usage().dead(), 1u); + EXPECT_GT(after.enum_store_usage().used(), before.enum_store_usage().used()); + EXPECT_GE(after.enum_store_usage().limit(), before.enum_store_usage().limit()); + EXPECT_GT(after.enum_store_usage().limit(), 4200000000u); } else { LOG(info, "requireThatAddressSpaceUsageIsReported(%s): NOT enum", attrName.c_str()); - EXPECT_EQUAL(before.enum_store_usage().used(), 0u); - EXPECT_EQUAL(before.enum_store_usage().dead(), 0u); - EXPECT_EQUAL(after.enum_store_usage(), before.enum_store_usage()); - EXPECT_EQUAL(AddressSpaceComponents::default_enum_store_usage(), after.enum_store_usage()); + EXPECT_EQ(before.enum_store_usage().used(), 0u); + EXPECT_EQ(before.enum_store_usage().dead(), 0u); + EXPECT_EQ(after.enum_store_usage(), before.enum_store_usage()); + EXPECT_EQ(AddressSpaceComponents::default_enum_store_usage(), after.enum_store_usage()); } if (attrPtr->hasMultiValue()) { LOG(info, "requireThatAddressSpaceUsageIsReported(%s): Has multi-value", attrName.c_str()); - EXPECT_EQUAL(before.multi_value_usage().used(), 1u); - EXPECT_EQUAL(before.multi_value_usage().dead(), 1u); - EXPECT_GREATER_EQUAL(after.multi_value_usage().used(), before.multi_value_usage().used()); - EXPECT_GREATER(after.multi_value_usage().limit(), before.multi_value_usage().limit()); - EXPECT_GREATER((1ull << 32), after.multi_value_usage().limit()); + EXPECT_EQ(before.multi_value_usage().used(), 1u); + EXPECT_EQ(before.multi_value_usage().dead(), 1u); + EXPECT_GE(after.multi_value_usage().used(), before.multi_value_usage().used()); + EXPECT_GT(after.multi_value_usage().limit(), before.multi_value_usage().limit()); + EXPECT_GT((1ull << 32), after.multi_value_usage().limit()); } else { LOG(info, "requireThatAddressSpaceUsageIsReported(%s): NOT multi-value", attrName.c_str()); - EXPECT_EQUAL(before.multi_value_usage().used(), 0u); - EXPECT_EQUAL(after.multi_value_usage(), before.multi_value_usage()); - EXPECT_EQUAL(AddressSpaceComponents::default_multi_value_usage(), after.multi_value_usage()); + EXPECT_EQ(before.multi_value_usage().used(), 0u); + EXPECT_EQ(after.multi_value_usage(), before.multi_value_usage()); + EXPECT_EQ(AddressSpaceComponents::default_multi_value_usage(), after.multi_value_usage()); } } @@ -2148,6 +2167,7 @@ template <typename AttributeType> void AttributeTest::requireThatAddressSpaceUsageIsReported(const Config &config) { + SCOPED_TRACE(make_scoped_trace_msg("address space is reported", config)); requireThatAddressSpaceUsageIsReported<AttributeType>(config, false); requireThatAddressSpaceUsageIsReported<AttributeType>(config, true); } @@ -2155,12 +2175,12 @@ AttributeTest::requireThatAddressSpaceUsageIsReported(const Config &config) void AttributeTest::requireThatAddressSpaceUsageIsReported() { - TEST_DO(requireThatAddressSpaceUsageIsReported<IntegerAttribute>(Config(BasicType::INT32, CollectionType::SINGLE))); - TEST_DO(requireThatAddressSpaceUsageIsReported<IntegerAttribute>(Config(BasicType::INT32, CollectionType::ARRAY))); - TEST_DO(requireThatAddressSpaceUsageIsReported<FloatingPointAttribute>(Config(BasicType::FLOAT, CollectionType::SINGLE))); - TEST_DO(requireThatAddressSpaceUsageIsReported<FloatingPointAttribute>(Config(BasicType::FLOAT, CollectionType::ARRAY))); - TEST_DO(requireThatAddressSpaceUsageIsReported<StringAttribute>(Config(BasicType::STRING, CollectionType::SINGLE))); - TEST_DO(requireThatAddressSpaceUsageIsReported<StringAttribute>(Config(BasicType::STRING, CollectionType::ARRAY))); + requireThatAddressSpaceUsageIsReported<IntegerAttribute>(Config(BasicType::INT32, CollectionType::SINGLE)); + requireThatAddressSpaceUsageIsReported<IntegerAttribute>(Config(BasicType::INT32, CollectionType::ARRAY)); + requireThatAddressSpaceUsageIsReported<FloatingPointAttribute>(Config(BasicType::FLOAT, CollectionType::SINGLE)); + requireThatAddressSpaceUsageIsReported<FloatingPointAttribute>(Config(BasicType::FLOAT, CollectionType::ARRAY)); + requireThatAddressSpaceUsageIsReported<StringAttribute>(Config(BasicType::STRING, CollectionType::SINGLE)); + requireThatAddressSpaceUsageIsReported<StringAttribute>(Config(BasicType::STRING, CollectionType::ARRAY)); } template <typename AttributeType, typename BufferType> @@ -2207,6 +2227,7 @@ template <typename AttributeType, typename BufferType> void AttributeTest::testReaderDuringLastUpdate(const Config &config) { + SCOPED_TRACE(make_scoped_trace_msg("reader during last update", config)); testReaderDuringLastUpdate<AttributeType, BufferType>(config, false, false); testReaderDuringLastUpdate<AttributeType, BufferType>(config, true, false); testReaderDuringLastUpdate<AttributeType, BufferType>(config, false, true); @@ -2216,15 +2237,15 @@ AttributeTest::testReaderDuringLastUpdate(const Config &config) void AttributeTest::testReaderDuringLastUpdate() { - TEST_DO((testReaderDuringLastUpdate<IntegerAttribute,AttributeVector::largeint_t>(Config(BasicType::INT32, CollectionType::SINGLE)))); - TEST_DO((testReaderDuringLastUpdate<IntegerAttribute,AttributeVector::largeint_t>(Config(BasicType::INT32, CollectionType::ARRAY)))); - TEST_DO((testReaderDuringLastUpdate<IntegerAttribute,AttributeVector::WeightedInt>(Config(BasicType::INT32, CollectionType::WSET)))); - TEST_DO((testReaderDuringLastUpdate<FloatingPointAttribute,double>(Config(BasicType::FLOAT, CollectionType::SINGLE)))); - TEST_DO((testReaderDuringLastUpdate<FloatingPointAttribute,double>(Config(BasicType::FLOAT, CollectionType::ARRAY)))); - TEST_DO((testReaderDuringLastUpdate<FloatingPointAttribute,FloatingPointAttribute::WeightedFloat>(Config(BasicType::FLOAT, CollectionType::WSET)))); - TEST_DO((testReaderDuringLastUpdate<StringAttribute,string>(Config(BasicType::STRING, CollectionType::SINGLE)))); - TEST_DO((testReaderDuringLastUpdate<StringAttribute,string>(Config(BasicType::STRING, CollectionType::ARRAY)))); - TEST_DO((testReaderDuringLastUpdate<StringAttribute,StringAttribute::WeightedString>(Config(BasicType::STRING, CollectionType::WSET)))); + testReaderDuringLastUpdate<IntegerAttribute,AttributeVector::largeint_t>(Config(BasicType::INT32, CollectionType::SINGLE)); + testReaderDuringLastUpdate<IntegerAttribute,AttributeVector::largeint_t>(Config(BasicType::INT32, CollectionType::ARRAY)); + testReaderDuringLastUpdate<IntegerAttribute,AttributeVector::WeightedInt>(Config(BasicType::INT32, CollectionType::WSET)); + testReaderDuringLastUpdate<FloatingPointAttribute,double>(Config(BasicType::FLOAT, CollectionType::SINGLE)); + testReaderDuringLastUpdate<FloatingPointAttribute,double>(Config(BasicType::FLOAT, CollectionType::ARRAY)); + testReaderDuringLastUpdate<FloatingPointAttribute,FloatingPointAttribute::WeightedFloat>(Config(BasicType::FLOAT, CollectionType::WSET)); + testReaderDuringLastUpdate<StringAttribute,string>(Config(BasicType::STRING, CollectionType::SINGLE)); + testReaderDuringLastUpdate<StringAttribute,string>(Config(BasicType::STRING, CollectionType::ARRAY)); + testReaderDuringLastUpdate<StringAttribute,StringAttribute::WeightedString>(Config(BasicType::STRING, CollectionType::WSET)); } void @@ -2252,32 +2273,32 @@ AttributeTest::testConditionalCommit() { AttributePtr v = createAttribute("sfsint32_cc", cfg); addClearedDocs(v, 1000); auto &iv = static_cast<IntegerAttribute &>(*v.get()); - EXPECT_EQUAL(0x8000u, iv.getChangeVectorMemoryUsage().allocatedBytes()); - EXPECT_EQUAL(0u, iv.getChangeVectorMemoryUsage().usedBytes()); + EXPECT_EQ(0x8000u, iv.getChangeVectorMemoryUsage().allocatedBytes()); + EXPECT_EQ(0u, iv.getChangeVectorMemoryUsage().usedBytes()); AttributeGuard guard1(v); populateSimpleUncommitted(iv, 1, 3); - EXPECT_EQUAL(0x8000u, iv.getChangeVectorMemoryUsage().allocatedBytes()); - EXPECT_EQUAL(128u, iv.getChangeVectorMemoryUsage().usedBytes()); + EXPECT_EQ(0x8000u, iv.getChangeVectorMemoryUsage().allocatedBytes()); + EXPECT_EQ(128u, iv.getChangeVectorMemoryUsage().usedBytes()); populateSimpleUncommitted(iv, 1, 1000); - EXPECT_EQUAL(0x10000u, iv.getChangeVectorMemoryUsage().allocatedBytes()); - EXPECT_EQUAL(64064u, iv.getChangeVectorMemoryUsage().usedBytes()); + EXPECT_EQ(0x10000u, iv.getChangeVectorMemoryUsage().allocatedBytes()); + EXPECT_EQ(64064u, iv.getChangeVectorMemoryUsage().usedBytes()); EXPECT_FALSE(v->commitIfChangeVectorTooLarge()); - EXPECT_EQUAL(0x10000u, iv.getChangeVectorMemoryUsage().allocatedBytes()); - EXPECT_EQUAL(64064u, iv.getChangeVectorMemoryUsage().usedBytes()); + EXPECT_EQ(0x10000u, iv.getChangeVectorMemoryUsage().allocatedBytes()); + EXPECT_EQ(64064u, iv.getChangeVectorMemoryUsage().usedBytes()); populateSimpleUncommitted(iv, 1, 200); - EXPECT_EQUAL(0x20000u, iv.getChangeVectorMemoryUsage().allocatedBytes()); - EXPECT_EQUAL(76800u, iv.getChangeVectorMemoryUsage().usedBytes()); + EXPECT_EQ(0x20000u, iv.getChangeVectorMemoryUsage().allocatedBytes()); + EXPECT_EQ(76800u, iv.getChangeVectorMemoryUsage().usedBytes()); EXPECT_TRUE(v->commitIfChangeVectorTooLarge()); - EXPECT_EQUAL(0x2000u, iv.getChangeVectorMemoryUsage().allocatedBytes()); - EXPECT_EQUAL(0u, iv.getChangeVectorMemoryUsage().usedBytes()); + EXPECT_EQ(0x2000u, iv.getChangeVectorMemoryUsage().allocatedBytes()); + EXPECT_EQ(0u, iv.getChangeVectorMemoryUsage().usedBytes()); } int64_t AttributeTest::stat_size(const vespalib::string& swapfile) { auto stat = vespalib::stat(swapfile); - ASSERT_TRUE(stat); - return stat->_size; + EXPECT_TRUE(stat); + return stat ? stat->_size : 0u; } int @@ -2297,13 +2318,17 @@ AttributeTest::test_paged_attribute(const vespalib::string& name, const vespalib LOG(info, "test_paged_attribute '%s'", name.c_str()); auto av = createAttribute(name, cfg); auto v = std::dynamic_pointer_cast<IntegerAttribute>(av); - ASSERT_TRUE(v || (!cfg.collectionType().isMultiValue() && !cfg.fastSearch())); + bool failed = false; + EXPECT_TRUE(v || (!cfg.collectionType().isMultiValue() && !cfg.fastSearch())) << (failed = true, ""); + if (failed) { + return 0; + } auto size1 = stat_size(swapfile); // Grow mapping from lid to value or multivalue index addClearedDocs(av, lid_mapping_size); auto size2 = stat_size(swapfile); auto size3 = size2; - EXPECT_LESS(size1, size2); + EXPECT_LT(size1, size2); if (cfg.collectionType().isMultiValue()) { // Grow multi value mapping for (uint32_t lid = 1; lid < 100; ++lid) { @@ -2314,7 +2339,7 @@ AttributeTest::test_paged_attribute(const vespalib::string& name, const vespalib av->commit(); } size3 = stat_size(swapfile); - EXPECT_LESS(size2, size3); + EXPECT_LT(size2, size3); result += 2; } if (cfg.fastSearch()) { @@ -2332,7 +2357,7 @@ AttributeTest::test_paged_attribute(const vespalib::string& name, const vespalib av->commit(); } auto size4 = stat_size(swapfile); - EXPECT_LESS(size3, size4); + EXPECT_LT(size3, size4); result += 4; } return result; @@ -2345,21 +2370,21 @@ AttributeTest::test_paged_attributes() vespalib::alloc::MmapFileAllocatorFactory::instance().setup(basedir); search::attribute::Config cfg1(BasicType::INT32, CollectionType::SINGLE); cfg1.setPaged(true); - EXPECT_EQUAL(1, test_paged_attribute("std-int-sv-paged", basedir + "/0.std-int-sv-paged/swapfile", cfg1)); + EXPECT_EQ(1, test_paged_attribute("std-int-sv-paged", basedir + "/0.std-int-sv-paged/swapfile", cfg1)); search::attribute::Config cfg2(BasicType::INT32, CollectionType::ARRAY); cfg2.setPaged(true); - EXPECT_EQUAL(3, test_paged_attribute("std-int-mv-paged", basedir + "/1.std-int-mv-paged/swapfile", cfg2)); + EXPECT_EQ(3, test_paged_attribute("std-int-mv-paged", basedir + "/1.std-int-mv-paged/swapfile", cfg2)); search::attribute::Config cfg3(BasicType::INT32, CollectionType::SINGLE); cfg3.setPaged(true); cfg3.setFastSearch(true); - EXPECT_EQUAL(5, test_paged_attribute("fs-int-sv-paged", basedir + "/2.fs-int-sv-paged/swapfile", cfg3)); + EXPECT_EQ(5, test_paged_attribute("fs-int-sv-paged", basedir + "/2.fs-int-sv-paged/swapfile", cfg3)); search::attribute::Config cfg4(BasicType::INT32, CollectionType::ARRAY); cfg4.setPaged(true); cfg4.setFastSearch(true); - EXPECT_EQUAL(7, test_paged_attribute("fs-int-mv-paged", basedir + "/3.fs-int-mv-paged/swapfile", cfg4)); + EXPECT_EQ(7, test_paged_attribute("fs-int-mv-paged", basedir + "/3.fs-int-mv-paged/swapfile", cfg4)); search::attribute::Config cfg5(BasicType::BOOL, CollectionType::SINGLE); cfg5.setPaged(true); - EXPECT_EQUAL(1, test_paged_attribute("std-bool-sv-paged", basedir + "/4.std-bool-sv-paged/swapfile", cfg5)); + EXPECT_EQ(1, test_paged_attribute("std-bool-sv-paged", basedir + "/4.std-bool-sv-paged/swapfile", cfg5)); vespalib::alloc::MmapFileAllocatorFactory::instance().setup(""); vespalib::rmdir(basedir, true); } @@ -2370,14 +2395,14 @@ void testNamePrefix() { AttributeVector::SP vS1 = createAttribute("sfsint32_pc.abc", cfg); AttributeVector::SP vS2 = createAttribute("sfsint32_pc.xyz", cfg); AttributeVector::SP vSS1 = createAttribute("sfsint32_pc.xyz.abc", cfg); - EXPECT_EQUAL("sfsint32_pc", vFlat->getName()); - EXPECT_EQUAL("sfsint32_pc", vFlat->getNamePrefix()); - EXPECT_EQUAL("sfsint32_pc.abc", vS1->getName()); - EXPECT_EQUAL("sfsint32_pc", vS1->getNamePrefix()); - EXPECT_EQUAL("sfsint32_pc.xyz", vS2->getName()); - EXPECT_EQUAL("sfsint32_pc", vS2->getNamePrefix()); - EXPECT_EQUAL("sfsint32_pc.xyz.abc", vSS1->getName()); - EXPECT_EQUAL("sfsint32_pc", vSS1->getNamePrefix()); + EXPECT_EQ("sfsint32_pc", vFlat->getName()); + EXPECT_EQ("sfsint32_pc", vFlat->getNamePrefix()); + EXPECT_EQ("sfsint32_pc.abc", vS1->getName()); + EXPECT_EQ("sfsint32_pc", vS1->getNamePrefix()); + EXPECT_EQ("sfsint32_pc.xyz", vS2->getName()); + EXPECT_EQ("sfsint32_pc", vS2->getNamePrefix()); + EXPECT_EQ("sfsint32_pc.xyz.abc", vSS1->getName()); + EXPECT_EQ("sfsint32_pc", vSS1->getNamePrefix()); } class MyMultiValueAttribute : public ArrayStringAttribute { @@ -2396,62 +2421,154 @@ test_multi_value_mapping_has_free_lists_enabled() EXPECT_TRUE(attr.has_free_lists_enabled()); } -void -deleteDataDirs() +TEST_F(AttributeTest, base_name) { - vespalib::rmdir(tmpDir, true); - vespalib::rmdir(clsDir, true); - vespalib::rmdir(asuDir, true); + testBaseName(); } -void -createDataDirs() +TEST_F(AttributeTest, reload) { - vespalib::mkdir(tmpDir, true); - vespalib::mkdir(clsDir, true); - vespalib::mkdir(asuDir, true); + testReload(); } -int AttributeTest::Main() +TEST_F(AttributeTest, has_load_data) { - TEST_INIT("attribute_test"); - - if (_argc > 0) { - DummyFileHeaderContext::setCreator(_argv[0]); - } - deleteDataDirs(); - createDataDirs(); - - testBaseName(); - testReload(); testHasLoadData(); +} + +TEST_F(AttributeTest, memory_saver) +{ testMemorySaver(); +} +TEST_F(AttributeTest, single_value_attributes) +{ testSingle(); +} + +TEST_F(AttributeTest, array_attributes) +{ testArray(); +} + +TEST_F(AttributeTest, weighted_set_attributes) +{ testWeightedSet(); +} + +TEST_F(AttributeTest, arithmetic_value_update) +{ testArithmeticValueUpdate(); +} + +TEST_F(AttributeTest, arithmetic_with_undefined_value) +{ testArithmeticWithUndefinedValue(); +} + +TEST_F(AttributeTest, map_value_udpate) +{ testMapValueUpdate(); +} + +TEST_F(AttributeTest, status) +{ testStatus(); +} + +TEST_F(AttributeTest, null_protection) +{ testNullProtection(); +} + +TEST_F(AttributeTest, generation) +{ testGeneration(); +} + +TEST_F(AttributeTest, create_serial_num) +{ testCreateSerialNum(); +} + +TEST_F(AttributeTest, predicate_header_tags) +{ testPredicateHeaderTags(); - TEST_DO(testCompactLidSpace()); - TEST_DO(test_default_value_ref_count_is_updated_after_shrink_lid_space()); - TEST_DO(requireThatAddressSpaceUsageIsReported()); +} + +TEST_F(AttributeTest, compact_lid_space) +{ + testCompactLidSpace(); +} + +TEST_F(AttributeTest, default_value_ref_count_is_updated_after_shrink_lid_space) +{ + test_default_value_ref_count_is_updated_after_shrink_lid_space(); +} + +TEST_F(AttributeTest, address_space_usage_is_reported) +{ + requireThatAddressSpaceUsageIsReported(); +} + +TEST_F(AttributeTest, reader_during_last_update) +{ testReaderDuringLastUpdate(); - TEST_DO(testPendingCompaction()); - TEST_DO(testConditionalCommit()); - TEST_DO(testNamePrefix()); +} + +TEST_F(AttributeTest, pending_compaction) +{ + testPendingCompaction(); +} + +TEST_F(AttributeTest, conditional_commit) +{ + testConditionalCommit(); +} + +TEST_F(AttributeTest, name_prefix) +{ + testNamePrefix(); +} + +TEST_F(AttributeTest, multi_value_mapping_has_free_lists_enabled) +{ test_multi_value_mapping_has_free_lists_enabled(); - TEST_DO(test_paged_attributes()); +} + +TEST_F(AttributeTest, paged_attributes) +{ + test_paged_attributes(); +} - deleteDataDirs(); - TEST_DONE(); } +void +deleteDataDirs() +{ + vespalib::rmdir(tmpDir, true); + vespalib::rmdir(clsDir, true); + vespalib::rmdir(asuDir, true); } -TEST_APPHOOK(search::AttributeTest); +void +createDataDirs() +{ + vespalib::mkdir(tmpDir, true); + vespalib::mkdir(clsDir, true); + vespalib::mkdir(asuDir, true); +} + +int +main(int argc, char* argv[]) +{ + if (argc > 0) { + DummyFileHeaderContext::setCreator(argv[0]); + } + ::testing::InitGoogleTest(&argc, argv); + deleteDataDirs(); + createDataDirs(); + auto result = RUN_ALL_TESTS(); + deleteDataDirs(); + return result; +} diff --git a/searchlib/src/tests/attribute/multi_value_read_view/CMakeLists.txt b/searchlib/src/tests/attribute/multi_value_read_view/CMakeLists.txt new file mode 100644 index 00000000000..32d90273623 --- /dev/null +++ b/searchlib/src/tests/attribute/multi_value_read_view/CMakeLists.txt @@ -0,0 +1,9 @@ +# Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +vespa_add_executable(searchlib_attribute_multi_value_read_view_test_app TEST + SOURCES + multi_value_read_view_test.cpp + DEPENDS + searchlib + GTest::GTest +) +vespa_add_test(NAME searchlib_attribute_multi_value_read_view_test_app COMMAND searchlib_attribute_multi_value_read_view_test_app) diff --git a/searchlib/src/tests/attribute/multi_value_read_view/multi_value_read_view_test.cpp b/searchlib/src/tests/attribute/multi_value_read_view/multi_value_read_view_test.cpp new file mode 100644 index 00000000000..c81fea79590 --- /dev/null +++ b/searchlib/src/tests/attribute/multi_value_read_view/multi_value_read_view_test.cpp @@ -0,0 +1,454 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include <vespa/document/base/documentid.h> +#include <vespa/searchcommon/attribute/i_multi_value_attribute.h> +#include <vespa/searchcommon/attribute/multi_value_traits.h> +#include <vespa/searchlib/attribute/attributefactory.h> +#include <vespa/searchlib/attribute/attribute_read_guard.h> +#include <vespa/searchlib/attribute/extendableattributes.h> +#include <vespa/searchlib/attribute/floatbase.h> +#include <vespa/searchlib/attribute/imported_attribute_vector.h> +#include <vespa/searchlib/attribute/imported_attribute_vector_factory.h> +#include <vespa/searchlib/attribute/integerbase.h> +#include <vespa/searchlib/attribute/reference_attribute.h> +#include <vespa/searchlib/attribute/stringbase.h> +#include <vespa/searchlib/common/i_document_meta_store_context.h> +#include <vespa/searchlib/test/mock_gid_to_lid_mapping.h> +#include <vespa/vespalib/gtest/gtest.h> +#include <vespa/vespalib/util/stash.h> + +using document::GlobalId; +using document::DocumentId; + +namespace search::attribute { + +using test::MockGidToLidMapperFactory; + +namespace { + +GlobalId toGid(vespalib::stringref docId) { + return DocumentId(docId).getGlobalId(); +} + +vespalib::string doc1("id:test:music::1"); +vespalib::string doc2("id:test:music::2"); + +struct MyGidToLidMapperFactory : public MockGidToLidMapperFactory +{ + MyGidToLidMapperFactory() + : MockGidToLidMapperFactory() + { + _map.insert({toGid(doc1), 1}); + _map.insert({toGid(doc2), 2}); + } +}; + +struct MockReadGuard : public IDocumentMetaStoreContext::IReadGuard { + virtual const search::IDocumentMetaStore &get() const override { + search::IDocumentMetaStore *nullStore = nullptr; + return static_cast<search::IDocumentMetaStore &>(*nullStore); + } +}; + +struct MockDocumentMetaStoreContext : public IDocumentMetaStoreContext +{ + std::unique_ptr<IReadGuard> getReadGuard() const override; +}; + +std::unique_ptr<IDocumentMetaStoreContext::IReadGuard> +MockDocumentMetaStoreContext::getReadGuard() const +{ + return std::make_unique<MockReadGuard>(); +} + + +std::shared_ptr<ReferenceAttribute> +create_reference_attribute(const vespalib::string &name, const std::shared_ptr<IGidToLidMapperFactory> gid_to_lid_mapper_factory) +{ + auto attr = std::make_shared<ReferenceAttribute>(name, Config(BasicType::REFERENCE)); + attr->addReservedDoc(); + while (attr->getNumDocs() < 20u) { + uint32_t doc_id = 0; + attr->addDoc(doc_id); + EXPECT_NE(0u, doc_id); + } + attr->update(4, toGid(doc1)); + attr->update(11, toGid(doc2)); + attr->setGidToLidMapperFactory(gid_to_lid_mapper_factory); + attr->populateTargetLids({}); + return attr; +} + +} + +class TestParam { + BasicType _basic_type; + +public: + TestParam(BasicType basic_type_in) + : _basic_type(basic_type_in) + { + } + + BasicType basic_type() const noexcept { return _basic_type; } +}; + +std::ostream& operator<<(std::ostream& os, const TestParam& param) +{ + os << param.basic_type().asString(); + return os; +} + +class MultiValueReadViewTest : public ::testing::TestWithParam<TestParam> +{ +protected: + std::shared_ptr<IGidToLidMapperFactory> _gid_to_lid_mapper_factory; + std::shared_ptr<ReferenceAttribute> _reference_attribute; + MultiValueReadViewTest() + : ::testing::TestWithParam<TestParam>(), + _gid_to_lid_mapper_factory(std::make_shared<MyGidToLidMapperFactory>()), + _reference_attribute(create_reference_attribute("ref", _gid_to_lid_mapper_factory)) + { + } + ~MultiValueReadViewTest() override = default; + + template <typename AttributeBaseType, typename BaseType> + void populate_helper(AttributeVector& attr, const std::vector<BaseType>& values); + void populate(AttributeVector& attr); + template <typename MultiValueType> + void check_values_helper(const IAttributeVector &attr, const std::vector<multivalue::ValueType_t<MultiValueType>>& exp_values); + template <typename BasicType> + void check_integer_values(const IAttributeVector &attr); + template <typename BasicType> + void check_floating_point_values(const IAttributeVector &attr); + void check_string_values(const IAttributeVector &attr); + void check_values(const IAttributeVector& attr); + std::shared_ptr<AttributeVector> make_attribute(CollectionType collection_type, bool fast_search); + std::shared_ptr<ReadableAttributeVector> make_imported_attribute(std::shared_ptr<AttributeVector> target); + std::shared_ptr<AttributeVector> make_extendable_attribute(CollectionType collection_type); + void test_normal_attribute_vector(CollectionType collection_type, bool fast_search); + void test_imported_attribute_vector(CollectionType collection_type, bool fast_search); + void test_extendable_attribute_vector(CollectionType collection_type); +}; + +template <typename AttributeBaseType, typename BaseType> +void +MultiValueReadViewTest::populate_helper(AttributeVector& attr, const std::vector<BaseType>& values) +{ + auto extend_interface = attr.getExtendInterface(); + if (extend_interface == nullptr) { + attr.addReservedDoc(); + } else { + uint32_t doc_id = 0; + EXPECT_TRUE(attr.addDoc(doc_id)); + EXPECT_EQ(0u, doc_id); + } + uint32_t doc_id = 0; + attr.addDoc(doc_id); + EXPECT_EQ(1u, doc_id); + if (extend_interface == nullptr) { + attr.clearDoc(doc_id); + } + attr.addDoc(doc_id); + EXPECT_EQ(2u, doc_id); + if (extend_interface == nullptr) { + attr.clearDoc(doc_id); + auto& spec_attr = dynamic_cast<AttributeBaseType&>(attr); + EXPECT_TRUE(spec_attr.append(doc_id, values[0], 2)); + EXPECT_TRUE(spec_attr.append(doc_id, values[1], 7)); + attr.commit(); + } else { + EXPECT_TRUE(extend_interface->add(values[0], 2)); + EXPECT_TRUE(extend_interface->add(values[1], 7)); + } +} + +void +MultiValueReadViewTest::populate(AttributeVector& attr) +{ + switch (attr.getConfig().basicType().type()) { + case BasicType::Type::INT8: + case BasicType::Type::INT16: + case BasicType::Type::INT32: + case BasicType::Type::INT64: + populate_helper<IntegerAttribute, int64_t>(attr, {42, 44}); + break; + case BasicType::Type::FLOAT: + case BasicType::Type::DOUBLE: + populate_helper<FloatingPointAttribute, double>(attr, {42.0, 44.0}); + break; + case BasicType::Type::STRING: + populate_helper<StringAttribute, const char*>(attr, {"42", "44"}); + break; + default: + FAIL() << "Cannot populate attribute vector"; + } +} + +namespace { + +template <typename BasicType> +struct CompareValues +{ + bool operator()(const BasicType &lhs, const BasicType &rhs) const { return lhs < rhs; } + bool operator()(const multivalue::WeightedValue<BasicType>& lhs, const multivalue::WeightedValue<BasicType>& rhs) const { return lhs.value() < rhs.value(); } + bool equal(const BasicType &lhs, const BasicType &rhs) const { return lhs == rhs; } + bool equal(const multivalue::WeightedValue<BasicType>& lhs, const multivalue::WeightedValue<BasicType>& rhs) const { return lhs.value() == rhs.value(); } +}; + +template <> +struct CompareValues<const char *> +{ + bool operator()(const char *lhs, const char *rhs) const { return strcmp(lhs, rhs) < 0; } + bool operator()(const multivalue::WeightedValue<const char *>& lhs, const multivalue::WeightedValue<const char *>& rhs) const { return strcmp(lhs.value(), rhs.value()) < 0; } + bool equal(const char *lhs, const char *rhs) const { return strcmp(lhs, rhs) == 0; } + bool equal(const multivalue::WeightedValue<const char *>& lhs, const multivalue::WeightedValue<const char *>& rhs) const { return strcmp(lhs.value(), rhs.value()) == 0; } +}; + +} + +template <typename MultiValueType> +void +MultiValueReadViewTest::check_values_helper(const IAttributeVector &attr, const std::vector<multivalue::ValueType_t<MultiValueType>>& exp_values) +{ + vespalib::Stash stash; + auto mv_attr = attr.as_multi_value_attribute(); + EXPECT_NE(nullptr, mv_attr); + auto read_view = mv_attr->make_read_view(IMultiValueAttribute::Tag<MultiValueType>(), stash); + EXPECT_NE(nullptr, read_view); + bool is_imported = attr.isImported(); + auto values = read_view->get_values(is_imported ? 4 : 1); + EXPECT_TRUE(values.empty()); + values = read_view->get_values(is_imported ? 11 : 2); + std::vector<MultiValueType> values_copy(values.begin(), values.end()); + bool was_array = true; + CompareValues<multivalue::ValueType_t<MultiValueType>> compare_values; + if (attr.getCollectionType() == CollectionType::Type::WSET) { + std::sort(values_copy.begin(), values_copy.end(), compare_values); + was_array = false; + } + EXPECT_EQ(2u, values_copy.size()); + if constexpr (multivalue::is_WeightedValue_v<MultiValueType>) { + EXPECT_TRUE(compare_values.equal(exp_values[0], values_copy[0].value())); + EXPECT_EQ(was_array ? 1 : 2, values_copy[0].weight()); + EXPECT_TRUE(compare_values.equal(exp_values[1], values_copy[1].value())); + EXPECT_EQ(was_array ? 1 : 7, values_copy[1].weight()); + } else { + EXPECT_TRUE(compare_values.equal(exp_values[0], values_copy[0])); + EXPECT_TRUE(compare_values.equal(exp_values[1], values_copy[1])); + } +} + +template <typename BasicType> +void +MultiValueReadViewTest::check_integer_values(const IAttributeVector &attr) +{ + std::vector<BasicType> exp_values{42, 44}; + check_values_helper<BasicType>(attr, exp_values); + check_values_helper<multivalue::WeightedValue<BasicType>>(attr, exp_values); +} + +template <typename BasicType> +void +MultiValueReadViewTest::check_floating_point_values(const IAttributeVector &attr) +{ + std::vector<BasicType> exp_values{42.0, 44.0}; + check_values_helper<BasicType>(attr, exp_values); + check_values_helper<multivalue::WeightedValue<BasicType>>(attr, exp_values); +} + +void +MultiValueReadViewTest::check_string_values(const IAttributeVector &attr) +{ + std::vector<const char *> exp_values{"42", "44"}; + check_values_helper<const char *>(attr, exp_values); + check_values_helper<multivalue::WeightedValue<const char *>>(attr, exp_values); +} + +void +MultiValueReadViewTest::check_values(const IAttributeVector& attr) +{ + switch (attr.getBasicType()) { + case BasicType::Type::INT8: + check_integer_values<int8_t>(attr); + break; + case BasicType::Type::INT16: + check_integer_values<int16_t>(attr); + break; + case BasicType::Type::INT32: + check_integer_values<int32_t>(attr); + break; + case BasicType::Type::INT64: + check_integer_values<int64_t>(attr); + break; + case BasicType::Type::FLOAT: + check_floating_point_values<float>(attr); + break; + case BasicType::Type::DOUBLE: + check_floating_point_values<double>(attr); + break; + case BasicType::Type::STRING: + check_string_values(attr); + break; + default: + FAIL() << "Cannot check values in attribute vector"; + } +} + +std::shared_ptr<AttributeVector> +MultiValueReadViewTest::make_attribute(CollectionType collection_type, bool fast_search) +{ + auto param = GetParam(); + Config config(param.basic_type(), collection_type); + config.setFastSearch(fast_search); + auto attr = AttributeFactory::createAttribute("attr", config); + return attr; +} + +std::shared_ptr<ReadableAttributeVector> +MultiValueReadViewTest::make_imported_attribute(std::shared_ptr<AttributeVector> target) +{ + return ImportedAttributeVectorFactory::create("imported", + _reference_attribute, + std::make_shared<MockDocumentMetaStoreContext>(), + target, + std::make_shared<MockDocumentMetaStoreContext>(), + false); +} + +std::shared_ptr<AttributeVector> +MultiValueReadViewTest::make_extendable_attribute(CollectionType collection_type) +{ + vespalib::string name("attr"); + // Match strategy in streaming visitor + switch (collection_type.type()) { + case CollectionType::Type::ARRAY: + switch (GetParam().basic_type().type()) { + case BasicType::Type::INT8: + case BasicType::Type::INT16: + case BasicType::Type::INT32: + case BasicType::Type::INT64: + return std::make_shared<MultiIntegerExtAttribute>(name); + case BasicType::Type::FLOAT: + case BasicType::Type::DOUBLE: + return std::make_shared<MultiFloatExtAttribute>(name); + case BasicType::Type::STRING: + return std::make_shared<MultiStringExtAttribute>(name); + default: + ; + } + break; + case CollectionType::Type::WSET: + switch (GetParam().basic_type().type()) { + case BasicType::Type::INT8: + case BasicType::Type::INT16: + case BasicType::Type::INT32: + case BasicType::Type::INT64: + return std::make_shared<WeightedSetIntegerExtAttribute>(name); + case BasicType::Type::FLOAT: + case BasicType::Type::DOUBLE: + return std::make_shared<WeightedSetFloatExtAttribute>(name); + case BasicType::Type::STRING: + return std::make_shared<WeightedSetStringExtAttribute>(name); + default: + ; + } + break; + default: + ; + } + return {}; +} + +void +MultiValueReadViewTest::test_normal_attribute_vector(CollectionType collection_type, bool fast_search) +{ + auto attr = make_attribute(collection_type, fast_search); + populate(*attr); + check_values(*attr); +} + +void +MultiValueReadViewTest::test_imported_attribute_vector(CollectionType collection_type, bool fast_search) +{ + auto attr = make_attribute(collection_type, fast_search); + populate(*attr); + auto imported_attr = make_imported_attribute(attr); + auto guard = imported_attr->makeReadGuard(false); + check_values(**guard); +} + +void +MultiValueReadViewTest::test_extendable_attribute_vector(CollectionType collection_type) +{ + auto attr = make_extendable_attribute(collection_type); + if (attr == nullptr) { + FAIL() << "Cannot create extend attribute"; + } + populate(*attr); + check_values(*attr); +} + +TEST_P(MultiValueReadViewTest, test_array) +{ + test_normal_attribute_vector(CollectionType::Type::ARRAY, false); +}; + +TEST_P(MultiValueReadViewTest, test_enumerated_array) +{ + test_normal_attribute_vector(CollectionType::Type::ARRAY, true); +}; + +TEST_P(MultiValueReadViewTest, test_weighted_set) +{ + test_normal_attribute_vector(CollectionType::Type::WSET, false); +}; + +TEST_P(MultiValueReadViewTest, test_enumerated_weighted_set) +{ + test_normal_attribute_vector(CollectionType::Type::WSET, true); +}; + +TEST_P(MultiValueReadViewTest, test_imported_array) +{ + test_imported_attribute_vector(CollectionType::Type::ARRAY, false); +}; + +TEST_P(MultiValueReadViewTest, test_imported_enumerated_array) +{ + test_imported_attribute_vector(CollectionType::Type::ARRAY, true); +}; + +TEST_P(MultiValueReadViewTest, test_importe_weighted_set) +{ + test_imported_attribute_vector(CollectionType::Type::WSET, false); +}; + +TEST_P(MultiValueReadViewTest, test_imported_enumerated_weighted_set) +{ + test_imported_attribute_vector(CollectionType::Type::WSET, true); +}; + +TEST_P(MultiValueReadViewTest, test_extendable_array) +{ + test_extendable_attribute_vector(CollectionType::Type::ARRAY); +} + +TEST_P(MultiValueReadViewTest, test_extendable_weighted_set) +{ + test_extendable_attribute_vector(CollectionType::Type::WSET); +} + +auto test_values = ::testing::Values(TestParam(BasicType::Type::INT8), + TestParam(BasicType::Type::INT16), + TestParam(BasicType::Type::INT32), + TestParam(BasicType::Type::INT64), + TestParam(BasicType::Type::FLOAT), + TestParam(BasicType::Type::DOUBLE), + TestParam(BasicType::Type::STRING)); + +VESPA_GTEST_INSTANTIATE_TEST_SUITE_P(ReadView, MultiValueReadViewTest, test_values,testing::PrintToStringParamName()); + +} + +GTEST_MAIN_RUN_ALL_TESTS() diff --git a/searchlib/src/vespa/searchlib/attribute/CMakeLists.txt b/searchlib/src/vespa/searchlib/attribute/CMakeLists.txt index 79c68ba4fe3..5fa691394fb 100644 --- a/searchlib/src/vespa/searchlib/attribute/CMakeLists.txt +++ b/searchlib/src/vespa/searchlib/attribute/CMakeLists.txt @@ -26,6 +26,7 @@ vespa_add_library(searchlib_attribute OBJECT bitvector_search_cache.cpp changevector.cpp configconverter.cpp + copy_multi_value_read_view.cpp createarrayfastsearch.cpp createarraystd.cpp createsetfastsearch.cpp @@ -45,7 +46,12 @@ vespa_add_library(searchlib_attribute OBJECT enum_store_dictionary.cpp enum_store_loaders.cpp enumstore.cpp + enumerated_multi_value_read_view.cpp extendableattributes.cpp + extendable_numeric_array_multi_value_read_view.cpp + extendable_numeric_weighted_set_multi_value_read_view.cpp + extendable_string_array_multi_value_read_view.cpp + extendable_string_weighted_set_multi_value_read_view.cpp fixedsourceselector.cpp flagattribute.cpp floatbase.cpp @@ -56,6 +62,7 @@ vespa_add_library(searchlib_attribute OBJECT imported_attribute_vector.cpp imported_attribute_vector_factory.cpp imported_attribute_vector_read_guard.cpp + imported_multi_value_read_view.cpp imported_search_context.cpp integerbase.cpp ipostinglistsearchcontext.cpp diff --git a/searchlib/src/vespa/searchlib/attribute/attributevector.h b/searchlib/src/vespa/searchlib/attribute/attributevector.h index 61f578d9f2b..5f336ab921f 100644 --- a/searchlib/src/vespa/searchlib/attribute/attributevector.h +++ b/searchlib/src/vespa/searchlib/attribute/attributevector.h @@ -167,39 +167,6 @@ protected: void performCompactionWarning(); - void getByType(DocId doc, const char *&v) const { - char tmp[1024]; v = getString(doc, tmp, sizeof(tmp)); - } - - void getByType(DocId doc, vespalib::string &v) const { - char tmp[1024]; v = getString(doc, tmp, sizeof(tmp)); - } - - void getByType(DocId doc, largeint_t & v) const { - v = getInt(doc); - } - - void getByType(DocId doc, double &v) const { - v = getFloat(doc); - } - - uint32_t getByType(DocId doc, const char **v, uint32_t sz) const { - return get(doc, v, sz); - } - - uint32_t getByType(DocId doc, vespalib::string *v, uint32_t sz) const { - return get(doc, v, sz); - } - - uint32_t getByType(DocId doc, largeint_t * v, uint32_t sz) const { - return get(doc, v, sz); - } - - uint32_t getByType(DocId doc, double *v, uint32_t sz) const { - return get(doc, v, sz); - } - - AttributeVector(vespalib::stringref baseFileName, const Config & c); void checkSetMaxValueCount(int index) { diff --git a/searchlib/src/vespa/searchlib/attribute/copy_multi_value_read_view.cpp b/searchlib/src/vespa/searchlib/attribute/copy_multi_value_read_view.cpp new file mode 100644 index 00000000000..a2cead2e1ad --- /dev/null +++ b/searchlib/src/vespa/searchlib/attribute/copy_multi_value_read_view.cpp @@ -0,0 +1,54 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include "copy_multi_value_read_view.h" + +using vespalib::datastore::AtomicEntryRef; + +namespace search::attribute { + +template <typename MultiValueType, typename RawMultiValueType> +CopyMultiValueReadView<MultiValueType, RawMultiValueType>::CopyMultiValueReadView(MultiValueMappingReadView<RawMultiValueType> mv_mapping_read_view) + : _mv_mapping_read_view(mv_mapping_read_view), + _copy() +{ +} + +template <typename MultiValueType, typename RawMultiValueType> +CopyMultiValueReadView<MultiValueType, RawMultiValueType>::~CopyMultiValueReadView() = default; + +template <typename MultiValueType, typename RawMultiValueType> +vespalib::ConstArrayRef<MultiValueType> +CopyMultiValueReadView<MultiValueType, RawMultiValueType>::get_values(uint32_t docid) const +{ + auto raw = _mv_mapping_read_view.get(docid); + if (_copy.size() < raw.size()) { + _copy.resize(raw.size()); + } + auto dst = _copy.data(); + for (auto &src : raw) { + ValueType v = multivalue::get_value_ref(src); + *dst = multivalue::ValueBuilder<MultiValueType>::build(v, multivalue::get_weight(src)); + ++dst; + } + return vespalib::ConstArrayRef(_copy.data(), raw.size()); +} + +using multivalue::WeightedValue; + +template class CopyMultiValueReadView<int8_t, WeightedValue<int8_t>>; +template class CopyMultiValueReadView<int16_t, WeightedValue<int16_t>>; +template class CopyMultiValueReadView<int32_t, WeightedValue<int32_t>>; +template class CopyMultiValueReadView<int64_t, WeightedValue<int64_t>>; +template class CopyMultiValueReadView<float, WeightedValue<float>>; +template class CopyMultiValueReadView<double, WeightedValue<double>>; +template class CopyMultiValueReadView<AtomicEntryRef, WeightedValue<AtomicEntryRef>>; + +template class CopyMultiValueReadView<WeightedValue<int8_t>, int8_t>; +template class CopyMultiValueReadView<WeightedValue<int16_t>, int16_t>; +template class CopyMultiValueReadView<WeightedValue<int32_t>, int32_t>; +template class CopyMultiValueReadView<WeightedValue<int64_t>, int64_t>; +template class CopyMultiValueReadView<WeightedValue<float>, float>; +template class CopyMultiValueReadView<WeightedValue<double>, double>; +template class CopyMultiValueReadView<WeightedValue<AtomicEntryRef>, AtomicEntryRef>; + +} diff --git a/searchlib/src/vespa/searchlib/attribute/copy_multi_value_read_view.h b/searchlib/src/vespa/searchlib/attribute/copy_multi_value_read_view.h new file mode 100644 index 00000000000..e8786357738 --- /dev/null +++ b/searchlib/src/vespa/searchlib/attribute/copy_multi_value_read_view.h @@ -0,0 +1,31 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#pragma once + +#include "multi_value_mapping_read_view.h" +#include "enumstore.h" +#include <vespa/searchcommon/attribute/i_multi_value_read_view.h> +#include <vespa/searchcommon/attribute/multi_value_traits.h> + +namespace search::attribute { + +/** + * Read view for the data stored in a multi-value attribute that handles + * addition and removal of weight. + * @tparam MultiValueType The multi-value type of the data to access. + * @tparam RawMultiValueType The multi-value type of the raw data to access. + */ +template <typename MultiValueType, typename RawMultiValueType> +class CopyMultiValueReadView : public IMultiValueReadView<MultiValueType> +{ + static_assert(std::is_same_v<multivalue::ValueType_t<MultiValueType>, multivalue::ValueType_t<RawMultiValueType>>); + using ValueType = multivalue::ValueType_t<MultiValueType>; + MultiValueMappingReadView<RawMultiValueType> _mv_mapping_read_view; + mutable std::vector<MultiValueType> _copy; +public: + CopyMultiValueReadView(MultiValueMappingReadView<RawMultiValueType> mv_mapping_read_view); + ~CopyMultiValueReadView() override; + vespalib::ConstArrayRef<MultiValueType> get_values(uint32_t docid) const override; +}; + +} diff --git a/searchlib/src/vespa/searchlib/attribute/enumerated_multi_value_read_view.cpp b/searchlib/src/vespa/searchlib/attribute/enumerated_multi_value_read_view.cpp new file mode 100644 index 00000000000..b0459d83ecf --- /dev/null +++ b/searchlib/src/vespa/searchlib/attribute/enumerated_multi_value_read_view.cpp @@ -0,0 +1,73 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include "enumerated_multi_value_read_view.h" + +using vespalib::datastore::AtomicEntryRef; + +namespace search::attribute { + +template <typename MultiValueType, typename RawMultiValueType, typename EnumEntryType> +EnumeratedMultiValueReadView<MultiValueType, RawMultiValueType, EnumEntryType>::EnumeratedMultiValueReadView(MultiValueMappingReadView<RawMultiValueType> mv_mapping_read_view, const EnumStoreT<EnumEntryType>& enum_store) + : _mv_mapping_read_view(mv_mapping_read_view), + _enum_store(enum_store), + _copy() +{ +} + +template <typename MultiValueType, typename RawMultiValueType, typename EnumEntryType> +EnumeratedMultiValueReadView<MultiValueType, RawMultiValueType, EnumEntryType>::~EnumeratedMultiValueReadView() = default; + +template <typename MultiValueType, typename RawMultiValueType, typename EnumEntryType> +vespalib::ConstArrayRef<MultiValueType> +EnumeratedMultiValueReadView<MultiValueType, RawMultiValueType, EnumEntryType>::get_values(uint32_t docid) const +{ + auto raw = _mv_mapping_read_view.get(docid); + if (_copy.size() < raw.size()) { + _copy.resize(raw.size()); + } + auto dst = _copy.data(); + for (auto &src : raw) { + EnumEntryType v = _enum_store.get_value(multivalue::get_value_ref(src).load_acquire()); + *dst = multivalue::ValueBuilder<MultiValueType>::build(v, multivalue::get_weight(src)); + ++dst; + } + return vespalib::ConstArrayRef(_copy.data(), raw.size()); +} + +using multivalue::WeightedValue; + +using WeightedAtomicEntryRef = WeightedValue<AtomicEntryRef>; + +template class EnumeratedMultiValueReadView<int8_t, AtomicEntryRef>; +template class EnumeratedMultiValueReadView<int16_t, AtomicEntryRef>; +template class EnumeratedMultiValueReadView<int32_t, AtomicEntryRef>; +template class EnumeratedMultiValueReadView<int64_t, AtomicEntryRef>; +template class EnumeratedMultiValueReadView<float, AtomicEntryRef>; +template class EnumeratedMultiValueReadView<double, AtomicEntryRef>; +template class EnumeratedMultiValueReadView<const char*, AtomicEntryRef>; + +template class EnumeratedMultiValueReadView<int8_t, WeightedAtomicEntryRef>; +template class EnumeratedMultiValueReadView<int16_t, WeightedAtomicEntryRef>; +template class EnumeratedMultiValueReadView<int32_t, WeightedAtomicEntryRef>; +template class EnumeratedMultiValueReadView<int64_t, WeightedAtomicEntryRef>; +template class EnumeratedMultiValueReadView<float, WeightedAtomicEntryRef>; +template class EnumeratedMultiValueReadView<double, WeightedAtomicEntryRef>; +template class EnumeratedMultiValueReadView<const char*, WeightedAtomicEntryRef>; + +template class EnumeratedMultiValueReadView<WeightedValue<int8_t>, WeightedAtomicEntryRef>; +template class EnumeratedMultiValueReadView<WeightedValue<int16_t>, WeightedAtomicEntryRef>; +template class EnumeratedMultiValueReadView<WeightedValue<int32_t>, WeightedAtomicEntryRef>; +template class EnumeratedMultiValueReadView<WeightedValue<int64_t>, WeightedAtomicEntryRef>; +template class EnumeratedMultiValueReadView<WeightedValue<float>, WeightedAtomicEntryRef>; +template class EnumeratedMultiValueReadView<WeightedValue<double>, WeightedAtomicEntryRef>; +template class EnumeratedMultiValueReadView<WeightedValue<const char*>, WeightedAtomicEntryRef>; + +template class EnumeratedMultiValueReadView<WeightedValue<int8_t>, AtomicEntryRef>; +template class EnumeratedMultiValueReadView<WeightedValue<int16_t>, AtomicEntryRef>; +template class EnumeratedMultiValueReadView<WeightedValue<int32_t>, AtomicEntryRef>; +template class EnumeratedMultiValueReadView<WeightedValue<int64_t>, AtomicEntryRef>; +template class EnumeratedMultiValueReadView<WeightedValue<float>, AtomicEntryRef>; +template class EnumeratedMultiValueReadView<WeightedValue<double>, AtomicEntryRef>; +template class EnumeratedMultiValueReadView<WeightedValue<const char*>, AtomicEntryRef>; + +} diff --git a/searchlib/src/vespa/searchlib/attribute/enumerated_multi_value_read_view.h b/searchlib/src/vespa/searchlib/attribute/enumerated_multi_value_read_view.h new file mode 100644 index 00000000000..77c279700c9 --- /dev/null +++ b/searchlib/src/vespa/searchlib/attribute/enumerated_multi_value_read_view.h @@ -0,0 +1,32 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#pragma once + +#include "multi_value_mapping_read_view.h" +#include "enumstore.h" +#include <vespa/searchcommon/attribute/i_multi_value_read_view.h> +#include <vespa/searchcommon/attribute/multi_value_traits.h> + +namespace search::attribute { + +/** + * Read view for the data stored in a multi-value attribute that handles + * mapping from enumerated value to value. + * @tparam MultiValueType The multi-value type of the data to access. + * @tparam RawMultiValueType The multi-value type of the raw data to access. + * @tparam EnumEntryType The enum store entry type. + */ +template <typename MultiValueType, typename RawMultiValueType, typename EnumEntryType = multivalue::ValueType_t<MultiValueType>> +class EnumeratedMultiValueReadView : public IMultiValueReadView<MultiValueType> +{ + using AtomicEntryRef = vespalib::datastore::AtomicEntryRef; + MultiValueMappingReadView<RawMultiValueType> _mv_mapping_read_view; + const EnumStoreT<EnumEntryType>& _enum_store; + mutable std::vector<MultiValueType> _copy; +public: + EnumeratedMultiValueReadView(MultiValueMappingReadView<RawMultiValueType> mv_mapping_read_view, const EnumStoreT<EnumEntryType>& enum_store); + ~EnumeratedMultiValueReadView() override; + vespalib::ConstArrayRef<MultiValueType> get_values(uint32_t docid) const override; +}; + +} diff --git a/searchlib/src/vespa/searchlib/attribute/extendable_numeric_array_multi_value_read_view.cpp b/searchlib/src/vespa/searchlib/attribute/extendable_numeric_array_multi_value_read_view.cpp new file mode 100644 index 00000000000..94c0855c47b --- /dev/null +++ b/searchlib/src/vespa/searchlib/attribute/extendable_numeric_array_multi_value_read_view.cpp @@ -0,0 +1,48 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include "extendable_numeric_array_multi_value_read_view.h" + +namespace search::attribute { + +template <class MultiValueType, typename BaseType> +ExtendableNumericArrayMultiValueReadView<MultiValueType, BaseType>::ExtendableNumericArrayMultiValueReadView(const std::vector<BaseType>& data, const std::vector<uint32_t>& idx) + : attribute::IMultiValueReadView<MultiValueType>(), + _data(data), + _idx(idx), + _copy() +{ +} + +template <class MultiValueType, typename BaseType> +ExtendableNumericArrayMultiValueReadView<MultiValueType, BaseType>::~ExtendableNumericArrayMultiValueReadView() = default; + +template <class MultiValueType, typename BaseType> +vespalib::ConstArrayRef<MultiValueType> +ExtendableNumericArrayMultiValueReadView<MultiValueType, BaseType>::get_values(uint32_t doc_id) const +{ + auto offset = _idx[doc_id]; + auto next_offset = _idx[doc_id + 1]; + vespalib::ConstArrayRef<BaseType> raw(_data.data() + offset, next_offset - offset); + if (_copy.size() < raw.size()) { + _copy.resize(raw.size()); + } + auto dst = _copy.data(); + for (auto &src : raw) { + *dst = multivalue::ValueBuilder<MultiValueType>::build(src, 1); + ++dst; + } + return vespalib::ConstArrayRef(_copy.data(), raw.size()); +} + +template class ExtendableNumericArrayMultiValueReadView<int8_t, int8_t>; +template class ExtendableNumericArrayMultiValueReadView<int16_t, int16_t>; +template class ExtendableNumericArrayMultiValueReadView<int32_t, int32_t>; +template class ExtendableNumericArrayMultiValueReadView<int64_t, int64_t>; +template class ExtendableNumericArrayMultiValueReadView<double, double>; +template class ExtendableNumericArrayMultiValueReadView<multivalue::WeightedValue<int8_t>, int8_t>; +template class ExtendableNumericArrayMultiValueReadView<multivalue::WeightedValue<int16_t>, int16_t>; +template class ExtendableNumericArrayMultiValueReadView<multivalue::WeightedValue<int32_t>, int32_t>; +template class ExtendableNumericArrayMultiValueReadView<multivalue::WeightedValue<int64_t>, int64_t>; +template class ExtendableNumericArrayMultiValueReadView<multivalue::WeightedValue<double>, double>; + +} diff --git a/searchlib/src/vespa/searchlib/attribute/extendable_numeric_array_multi_value_read_view.h b/searchlib/src/vespa/searchlib/attribute/extendable_numeric_array_multi_value_read_view.h new file mode 100644 index 00000000000..09c5649e385 --- /dev/null +++ b/searchlib/src/vespa/searchlib/attribute/extendable_numeric_array_multi_value_read_view.h @@ -0,0 +1,28 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#pragma once + +#include <vespa/searchcommon/attribute/i_multi_value_read_view.h> + +namespace search::attribute { + +/** + * Read view for the data stored in an extendable multi-value numeric + * array attribute vector (used by streaming visitor) that handles + * optional addition of weight. + * @tparam MultiValueType The multi-value type of the data to access. + * @tparam BaseType The base type of the raw data to access. + */ +template <typename MultiValueType, typename BaseType> +class ExtendableNumericArrayMultiValueReadView : public attribute::IMultiValueReadView<MultiValueType> +{ + const std::vector<BaseType>& _data; + const std::vector<uint32_t>& _idx; + mutable std::vector<MultiValueType> _copy; +public: + ExtendableNumericArrayMultiValueReadView(const std::vector<BaseType>& data, const std::vector<uint32_t>& idx); + ~ExtendableNumericArrayMultiValueReadView() override; + vespalib::ConstArrayRef<MultiValueType> get_values(uint32_t doc_id) const override; +}; + +} diff --git a/searchlib/src/vespa/searchlib/attribute/extendable_numeric_weighted_set_multi_value_read_view.cpp b/searchlib/src/vespa/searchlib/attribute/extendable_numeric_weighted_set_multi_value_read_view.cpp new file mode 100644 index 00000000000..0e8b2f62f9d --- /dev/null +++ b/searchlib/src/vespa/searchlib/attribute/extendable_numeric_weighted_set_multi_value_read_view.cpp @@ -0,0 +1,43 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include "extendable_numeric_weighted_set_multi_value_read_view.h" + +namespace search::attribute { + +template <class MultiValueType, typename BaseType> +ExtendableNumericWeightedSetMultiValueReadView<MultiValueType, BaseType>::ExtendableNumericWeightedSetMultiValueReadView(const std::vector<BaseType>& data, const std::vector<uint32_t>& idx, const std::vector<int32_t>& weights) + : attribute::IMultiValueReadView<MultiValueType>(), + _data(data), + _idx(idx), + _weights(weights), + _copy() +{ +} + +template <class MultiValueType, typename BaseType> +ExtendableNumericWeightedSetMultiValueReadView<MultiValueType, BaseType>::~ExtendableNumericWeightedSetMultiValueReadView() = default; + +template <class MultiValueType, typename BaseType> +vespalib::ConstArrayRef<MultiValueType> +ExtendableNumericWeightedSetMultiValueReadView<MultiValueType, BaseType>::get_values(uint32_t doc_id) const +{ + auto offset = _idx[doc_id]; + auto next_offset = _idx[doc_id + 1]; + vespalib::ConstArrayRef<BaseType> raw(_data.data() + offset, next_offset - offset); + if (_copy.size() < raw.size()) { + _copy.resize(raw.size()); + } + auto dst = _copy.data(); + auto* src_weight = _weights.data() + offset; + for (auto &src : raw) { + *dst = multivalue::ValueBuilder<MultiValueType>::build(src, *src_weight); + ++src_weight; + ++dst; + } + return vespalib::ConstArrayRef(_copy.data(), raw.size()); +} + +template class ExtendableNumericWeightedSetMultiValueReadView<multivalue::WeightedValue<int64_t>, int64_t>; +template class ExtendableNumericWeightedSetMultiValueReadView<multivalue::WeightedValue<double>, double>; + +} diff --git a/searchlib/src/vespa/searchlib/attribute/extendable_numeric_weighted_set_multi_value_read_view.h b/searchlib/src/vespa/searchlib/attribute/extendable_numeric_weighted_set_multi_value_read_view.h new file mode 100644 index 00000000000..735bb754408 --- /dev/null +++ b/searchlib/src/vespa/searchlib/attribute/extendable_numeric_weighted_set_multi_value_read_view.h @@ -0,0 +1,29 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#pragma once + +#include <vespa/searchcommon/attribute/i_multi_value_read_view.h> + +namespace search::attribute { + +/** + * Read view for the data stored in an extendable multi-value numeric + * weighted set attribute vector (used by streaming visitor) that handles + * optional removal of weight. + * @tparam MultiValueType The multi-value type of the data to access. + * @tparam BaseType The base type of the raw data to access. + */ +template <typename MultiValueType, typename BaseType> +class ExtendableNumericWeightedSetMultiValueReadView : public attribute::IMultiValueReadView<MultiValueType> +{ + const std::vector<BaseType>& _data; + const std::vector<uint32_t>& _idx; + const std::vector<int32_t>& _weights; + mutable std::vector<MultiValueType> _copy; +public: + ExtendableNumericWeightedSetMultiValueReadView(const std::vector<BaseType>& data, const std::vector<uint32_t>& idx, const std::vector<int32_t>& weights); + ~ExtendableNumericWeightedSetMultiValueReadView() override; + vespalib::ConstArrayRef<MultiValueType> get_values(uint32_t doc_id) const override; +}; + +} diff --git a/searchlib/src/vespa/searchlib/attribute/extendable_string_array_multi_value_read_view.cpp b/searchlib/src/vespa/searchlib/attribute/extendable_string_array_multi_value_read_view.cpp new file mode 100644 index 00000000000..fef05a8ce08 --- /dev/null +++ b/searchlib/src/vespa/searchlib/attribute/extendable_string_array_multi_value_read_view.cpp @@ -0,0 +1,41 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include "extendable_string_array_multi_value_read_view.h" + +namespace search::attribute { + +template <class MultiValueType> +ExtendableStringArrayMultiValueReadView<MultiValueType>::ExtendableStringArrayMultiValueReadView(const std::vector<char>& buffer, const vespalib::Array<uint32_t>& offsets, const std::vector<uint32_t>& idx) + : attribute::IMultiValueReadView<MultiValueType>(), + _buffer(buffer), + _offsets(offsets), + _idx(idx), + _copy() +{ +} + +template <class MultiValueType> +ExtendableStringArrayMultiValueReadView<MultiValueType>::~ExtendableStringArrayMultiValueReadView() = default; + +template <class MultiValueType> +vespalib::ConstArrayRef<MultiValueType> +ExtendableStringArrayMultiValueReadView<MultiValueType>::get_values(uint32_t doc_id) const +{ + auto offset = _idx[doc_id]; + auto next_offset = _idx[doc_id + 1]; + vespalib::ConstArrayRef<uint32_t> raw(&_offsets[offset], next_offset - offset); + if (_copy.size() < raw.size()) { + _copy.resize(raw.size()); + } + auto dst = _copy.data(); + for (auto &src : raw) { + *dst = multivalue::ValueBuilder<MultiValueType>::build(_buffer.data() + src, 1); + ++dst; + } + return vespalib::ConstArrayRef(_copy.data(), raw.size()); +} + +template class ExtendableStringArrayMultiValueReadView<const char*>; +template class ExtendableStringArrayMultiValueReadView<multivalue::WeightedValue<const char*>>; + +} diff --git a/searchlib/src/vespa/searchlib/attribute/extendable_string_array_multi_value_read_view.h b/searchlib/src/vespa/searchlib/attribute/extendable_string_array_multi_value_read_view.h new file mode 100644 index 00000000000..6d8c64a706d --- /dev/null +++ b/searchlib/src/vespa/searchlib/attribute/extendable_string_array_multi_value_read_view.h @@ -0,0 +1,28 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#pragma once + +#include <vespa/searchcommon/attribute/i_multi_value_read_view.h> + +namespace search::attribute { + +/** + * Read view for the data stored in an extendable multi-value string + * array attribute vector (used by streaming visitor) that handles + * optional addition of weight. + * @tparam MultiValueType The multi-value type of the data to access. + */ +template <typename MultiValueType> +class ExtendableStringArrayMultiValueReadView : public attribute::IMultiValueReadView<MultiValueType> +{ + const std::vector<char>& _buffer; + const vespalib::Array<uint32_t>& _offsets; + const std::vector<uint32_t>& _idx; + mutable std::vector<MultiValueType> _copy; +public: + ExtendableStringArrayMultiValueReadView(const std::vector<char>& buffer, const vespalib::Array<uint32_t>& offsets, const std::vector<uint32_t>& idx); + ~ExtendableStringArrayMultiValueReadView() override; + vespalib::ConstArrayRef<MultiValueType> get_values(uint32_t doc_id) const override; +}; + +} diff --git a/searchlib/src/vespa/searchlib/attribute/extendable_string_weighted_set_multi_value_read_view.cpp b/searchlib/src/vespa/searchlib/attribute/extendable_string_weighted_set_multi_value_read_view.cpp new file mode 100644 index 00000000000..eda50dd7172 --- /dev/null +++ b/searchlib/src/vespa/searchlib/attribute/extendable_string_weighted_set_multi_value_read_view.cpp @@ -0,0 +1,43 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include "extendable_string_weighted_set_multi_value_read_view.h" + +namespace search::attribute { + +template <class MultiValueType> +ExtendableStringWeightedSetMultiValueReadView<MultiValueType>::ExtendableStringWeightedSetMultiValueReadView(const std::vector<char>& buffer, const vespalib::Array<uint32_t>& offsets, const std::vector<uint32_t>& idx, const std::vector<int32_t>& weights) + : attribute::IMultiValueReadView<MultiValueType>(), + _buffer(buffer), + _offsets(offsets), + _idx(idx), + _weights(weights), + _copy() +{ +} + +template <class MultiValueType> +ExtendableStringWeightedSetMultiValueReadView<MultiValueType>::~ExtendableStringWeightedSetMultiValueReadView() = default; + +template <class MultiValueType> +vespalib::ConstArrayRef<MultiValueType> +ExtendableStringWeightedSetMultiValueReadView<MultiValueType>::get_values(uint32_t doc_id) const +{ + auto offset = _idx[doc_id]; + auto next_offset = _idx[doc_id + 1]; + vespalib::ConstArrayRef<uint32_t> raw(&_offsets[offset], next_offset - offset); + if (_copy.size() < raw.size()) { + _copy.resize(raw.size()); + } + auto dst = _copy.data(); + auto* src_weight = _weights.data() + offset; + for (auto &src : raw) { + *dst = multivalue::ValueBuilder<MultiValueType>::build(_buffer.data() + src, *src_weight); + ++src_weight; + ++dst; + } + return vespalib::ConstArrayRef(_copy.data(), raw.size()); +} + +template class ExtendableStringWeightedSetMultiValueReadView<multivalue::WeightedValue<const char*>>; + +} diff --git a/searchlib/src/vespa/searchlib/attribute/extendable_string_weighted_set_multi_value_read_view.h b/searchlib/src/vespa/searchlib/attribute/extendable_string_weighted_set_multi_value_read_view.h new file mode 100644 index 00000000000..1d631b06c1c --- /dev/null +++ b/searchlib/src/vespa/searchlib/attribute/extendable_string_weighted_set_multi_value_read_view.h @@ -0,0 +1,29 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#pragma once + +#include <vespa/searchcommon/attribute/i_multi_value_read_view.h> + +namespace search::attribute { + +/** + * Read view for the data stored in an extendable multi-value string + * weighted set attribute vector (used by streaming visitor) that handles + * optional removal of weight. + * @tparam MultiValueType The multi-value type of the data to access. + */ +template <typename MultiValueType> +class ExtendableStringWeightedSetMultiValueReadView : public attribute::IMultiValueReadView<MultiValueType> +{ + const std::vector<char>& _buffer; + const vespalib::Array<uint32_t>& _offsets; + const std::vector<uint32_t>& _idx; + const std::vector<int32_t>& _weights; + mutable std::vector<MultiValueType> _copy; +public: + ExtendableStringWeightedSetMultiValueReadView(const std::vector<char>& buffer, const vespalib::Array<uint32_t>& offsets, const std::vector<uint32_t>& idx, const std::vector<int32_t>& weights); + ~ExtendableStringWeightedSetMultiValueReadView() override; + vespalib::ConstArrayRef<MultiValueType> get_values(uint32_t doc_id) const override; +}; + +} diff --git a/searchlib/src/vespa/searchlib/attribute/extendableattributes.cpp b/searchlib/src/vespa/searchlib/attribute/extendableattributes.cpp index 12507a73564..9c944a70b94 100644 --- a/searchlib/src/vespa/searchlib/attribute/extendableattributes.cpp +++ b/searchlib/src/vespa/searchlib/attribute/extendableattributes.cpp @@ -2,6 +2,11 @@ #include "extendableattributes.h" #include "attrvector.hpp" +#include "extendable_numeric_array_multi_value_read_view.h" +#include "extendable_numeric_weighted_set_multi_value_read_view.h" +#include "extendable_string_array_multi_value_read_view.h" +#include "extendable_string_weighted_set_multi_value_read_view.h" +#include <vespa/vespalib/util/stash.h> #include <vespa/log/log.h> LOG_SETUP(".searchlib.attribute.extendable_attributes"); @@ -41,6 +46,26 @@ bool SingleStringExtAttribute::add(const char * v, int32_t) //******************** CollectionType::ARRAY ********************// +template <typename T> +const attribute::IMultiValueAttribute* +MultiExtAttribute<T>::as_multi_value_attribute() const +{ + return this; +} + +template <typename T> +const attribute::IMultiValueReadView<T>* +MultiExtAttribute<T>::make_read_view(attribute::IMultiValueAttribute::Tag<T>, vespalib::Stash& stash) const +{ + return &stash.create<attribute::ExtendableNumericArrayMultiValueReadView<T, T>>(this->_data, this->_idx); +} + +template <typename T> +const attribute::IMultiValueReadView<multivalue::WeightedValue<T>>* +MultiExtAttribute<T>::make_read_view(attribute::IMultiValueAttribute::Tag<multivalue::WeightedValue<T>>, vespalib::Stash& stash) const +{ + return &stash.create<attribute::ExtendableNumericArrayMultiValueReadView<multivalue::WeightedValue<T>, T>>(this->_data, this->_idx); +} MultiStringExtAttribute::MultiStringExtAttribute(const vespalib::string & name, const CollectionType & ctype) : StringDirectAttrVector< AttrVector::Features<true> > @@ -79,6 +104,23 @@ bool MultiStringExtAttribute::add(const char * v, int32_t) return true; } +const attribute::IMultiValueAttribute* +MultiStringExtAttribute::as_multi_value_attribute() const +{ + return this; +} + +const attribute::IMultiValueReadView<const char*>* +MultiStringExtAttribute::make_read_view(attribute::IMultiValueAttribute::Tag<const char*>, vespalib::Stash& stash) const +{ + return &stash.create<attribute::ExtendableStringArrayMultiValueReadView<const char*>>(this->_buffer, this->_offsets, this->_idx); +} + +const attribute::IMultiValueReadView<multivalue::WeightedValue<const char*>>* +MultiStringExtAttribute::make_read_view(attribute::IMultiValueAttribute::Tag<multivalue::WeightedValue<const char*>>, vespalib::Stash& stash) const +{ + return &stash.create<attribute::ExtendableStringArrayMultiValueReadView<multivalue::WeightedValue<const char*>>>(this->_buffer, this->_offsets, this->_idx); +} //******************** CollectionType::WSET ********************// @@ -108,6 +150,12 @@ WeightedSetIntegerExtAttribute::get(DocId doc, AttributeVector::WeightedInt * v, return valueCount; } +const attribute::IMultiValueReadView<multivalue::WeightedValue<int64_t>>* +WeightedSetIntegerExtAttribute::make_read_view(attribute::IMultiValueAttribute::Tag<multivalue::WeightedValue<int64_t>>, vespalib::Stash& stash) const +{ + return &stash.create<attribute::ExtendableNumericWeightedSetMultiValueReadView<multivalue::WeightedValue<int64_t>, int64_t>>(this->_data, this->_idx, this->get_weights()); +} + WeightedSetFloatExtAttribute::WeightedSetFloatExtAttribute(const vespalib::string & name) : WeightedSetExtAttributeBase<MultiFloatExtAttribute>(name) { @@ -134,6 +182,12 @@ WeightedSetFloatExtAttribute::get(DocId doc, AttributeVector::WeightedFloat * v, return valueCount; } +const attribute::IMultiValueReadView<multivalue::WeightedValue<double>>* +WeightedSetFloatExtAttribute::make_read_view(attribute::IMultiValueAttribute::Tag<multivalue::WeightedValue<double>>, vespalib::Stash& stash) const +{ + return &stash.create<attribute::ExtendableNumericWeightedSetMultiValueReadView<multivalue::WeightedValue<double>, double>>(this->_data, this->_idx, this->get_weights()); +} + WeightedSetStringExtAttribute::WeightedSetStringExtAttribute(const vespalib::string & name) : WeightedSetExtAttributeBase<MultiStringExtAttribute>(name) { @@ -162,4 +216,16 @@ WeightedSetStringExtAttribute::get(DocId doc, AttributeVector::WeightedConstChar return getAllHelper(doc, v, sz); } +const attribute::IMultiValueReadView<multivalue::WeightedValue<const char*>>* +WeightedSetStringExtAttribute::make_read_view(attribute::IMultiValueAttribute::Tag<multivalue::WeightedValue<const char*>>, vespalib::Stash& stash) const +{ + return &stash.create<attribute::ExtendableStringWeightedSetMultiValueReadView<multivalue::WeightedValue<const char*>>>(this->_buffer, this->_offsets, this->_idx, this->get_weights()); +} + +template class MultiExtAttribute<int8_t>; +template class MultiExtAttribute<int16_t>; +template class MultiExtAttribute<int32_t>; +template class MultiExtAttribute<int64_t>; +template class MultiExtAttribute<double>; + } diff --git a/searchlib/src/vespa/searchlib/attribute/extendableattributes.h b/searchlib/src/vespa/searchlib/attribute/extendableattributes.h index 5a1d005686e..ebd1d1d764c 100644 --- a/searchlib/src/vespa/searchlib/attribute/extendableattributes.h +++ b/searchlib/src/vespa/searchlib/attribute/extendableattributes.h @@ -8,6 +8,7 @@ #include "attrvector.h" #include "attrvector.hpp" +#include <vespa/searchcommon/attribute/i_multi_value_attribute.h> namespace search { @@ -99,7 +100,8 @@ public: template <typename T> class MultiExtAttribute : public NumericDirectAttrVector<AttrVector::Features<true>, typename AttributeTemplate<T>::Type>, - public IExtendAttribute + public IExtendAttribute, + public attribute::IMultiValueAttribute { protected: typedef typename MultiExtAttribute<T>::NumDirectAttrVec Super; @@ -146,6 +148,11 @@ public: void onAddDocs(uint32_t lidLimit) override { this->_data.reserve(lidLimit); } + const attribute::IMultiValueAttribute* as_multi_value_attribute() const override; + + // Implements attribute::IMultiValueAttribute + const attribute::IMultiValueReadView<T>* make_read_view(attribute::IMultiValueAttribute::Tag<T>, vespalib::Stash& stash) const override; + const attribute::IMultiValueReadView<multivalue::WeightedValue<T>>* make_read_view(attribute::IMultiValueAttribute::Tag<multivalue::WeightedValue<T>>, vespalib::Stash& stash) const override; }; template <typename T> @@ -161,7 +168,8 @@ typedef MultiInt64ExtAttribute MultiIntegerExtAttribute; class MultiStringExtAttribute : public StringDirectAttrVector< AttrVector::Features<true> >, - public IExtendAttribute + public IExtendAttribute, + public attribute::IMultiValueAttribute { IExtendAttribute * getExtendInterface() override { return this; } protected: @@ -175,6 +183,10 @@ public: return false; // Emulate that this attribute is never loaded } void onAddDocs(DocId ) override { } + const attribute::IMultiValueAttribute* as_multi_value_attribute() const override; + // Implements attribute::IMultiValueAttribute + const attribute::IMultiValueReadView<const char*>* make_read_view(attribute::IMultiValueAttribute::Tag<const char*>, vespalib::Stash& stash) const override; + const attribute::IMultiValueReadView<multivalue::WeightedValue<const char*>>* make_read_view(attribute::IMultiValueAttribute::Tag<multivalue::WeightedValue<const char*>>, vespalib::Stash& stash) const override; }; @@ -198,6 +210,7 @@ protected: _weights() {} ~WeightedSetExtAttributeBase() {} + const std::vector<int32_t>& get_weights() const noexcept { return _weights; } }; class WeightedSetIntegerExtAttribute @@ -215,6 +228,8 @@ public: ~WeightedSetIntegerExtAttribute(); bool add(int64_t v, int32_t w = 1) override; uint32_t get(DocId doc, AttributeVector::WeightedInt * v, uint32_t sz) const override; + // Implements attribute::IMultiValueAttribute + const attribute::IMultiValueReadView<multivalue::WeightedValue<int64_t>>* make_read_view(attribute::IMultiValueAttribute::Tag<multivalue::WeightedValue<int64_t>>, vespalib::Stash& stash) const override; }; class WeightedSetFloatExtAttribute @@ -232,6 +247,8 @@ public: ~WeightedSetFloatExtAttribute(); bool add(double v, int32_t w = 1) override; uint32_t get(DocId doc, AttributeVector::WeightedFloat * v, uint32_t sz) const override; + // Implements attribute::IMultiValueAttribute + const attribute::IMultiValueReadView<multivalue::WeightedValue<double>>* make_read_view(attribute::IMultiValueAttribute::Tag<multivalue::WeightedValue<double>>, vespalib::Stash& stash) const override; }; class WeightedSetStringExtAttribute @@ -258,6 +275,8 @@ public: bool add(const char * v, int32_t w = 1) override; uint32_t get(DocId doc, AttributeVector::WeightedString * v, uint32_t sz) const override; uint32_t get(DocId doc, AttributeVector::WeightedConstChar * v, uint32_t sz) const override; + // Implements attribute::IMultiValueAttribute + const attribute::IMultiValueReadView<multivalue::WeightedValue<const char*>>* make_read_view(attribute::IMultiValueAttribute::Tag<multivalue::WeightedValue<const char*>>, vespalib::Stash& stash) const override; }; } // namespace search diff --git a/searchlib/src/vespa/searchlib/attribute/imported_attribute_vector_read_guard.cpp b/searchlib/src/vespa/searchlib/attribute/imported_attribute_vector_read_guard.cpp index f1ae5252031..c8a6309c282 100644 --- a/searchlib/src/vespa/searchlib/attribute/imported_attribute_vector_read_guard.cpp +++ b/searchlib/src/vespa/searchlib/attribute/imported_attribute_vector_read_guard.cpp @@ -2,9 +2,11 @@ #include "imported_attribute_vector_read_guard.h" #include "imported_attribute_vector.h" +#include "imported_multi_value_read_view.h" #include "imported_search_context.h" #include "reference_attribute.h" #include <vespa/searchlib/query/query_term_simple.h> +#include <vespa/vespalib/util/stash.h> namespace search::attribute { @@ -120,7 +122,7 @@ const tensor::ITensorAttribute *ImportedAttributeVectorReadGuard::asTensorAttrib } const attribute::IMultiValueAttribute* ImportedAttributeVectorReadGuard::as_multi_value_attribute() const { - return nullptr; + return this; } BasicType::Type ImportedAttributeVectorReadGuard::getBasicType() const { @@ -156,6 +158,117 @@ bool ImportedAttributeVectorReadGuard::isImported() const return true; } +template <typename MultiValueType> +const IMultiValueReadView<MultiValueType>* +ImportedAttributeVectorReadGuard::make_read_view_helper(Tag<MultiValueType> tag, vespalib::Stash &stash) const +{ + auto target_mv_attr = _target_attribute.as_multi_value_attribute(); + if (target_mv_attr == nullptr) { + return nullptr; + } + auto target_read_view = target_mv_attr->make_read_view(tag, stash); + if (target_read_view == nullptr) { + return nullptr; + } + return &stash.create<ImportedMultiValueReadView<MultiValueType>>(_targetLids, target_read_view); +} + +const IArrayReadView<int8_t>* +ImportedAttributeVectorReadGuard::make_read_view(ArrayTag<int8_t> tag, vespalib::Stash& stash) const +{ + return make_read_view_helper(tag, stash); +} + +const IArrayReadView<int16_t>* +ImportedAttributeVectorReadGuard::make_read_view(ArrayTag<int16_t> tag, vespalib::Stash& stash) const +{ + return make_read_view_helper(tag, stash); +} + +const IArrayReadView<int32_t>* +ImportedAttributeVectorReadGuard::make_read_view(ArrayTag<int32_t> tag, vespalib::Stash& stash) const +{ + return make_read_view_helper(tag, stash); +} + +const IArrayReadView<int64_t>* +ImportedAttributeVectorReadGuard::make_read_view(ArrayTag<int64_t> tag, vespalib::Stash& stash) const +{ + return make_read_view_helper(tag, stash); +} + +const IArrayReadView<float>* +ImportedAttributeVectorReadGuard::make_read_view(ArrayTag<float> tag, vespalib::Stash& stash) const +{ + return make_read_view_helper(tag, stash); +} + +const IArrayReadView<double>* +ImportedAttributeVectorReadGuard::make_read_view(ArrayTag<double> tag, vespalib::Stash& stash) const +{ + return make_read_view_helper(tag, stash); +} + +const IArrayReadView<const char*>* +ImportedAttributeVectorReadGuard::make_read_view(ArrayTag<const char*> tag, vespalib::Stash& stash) const +{ + return make_read_view_helper(tag, stash); +} + +const IWeightedSetReadView<int8_t>* +ImportedAttributeVectorReadGuard::make_read_view(WeightedSetTag<int8_t> tag, vespalib::Stash& stash) const +{ + return make_read_view_helper(tag, stash); +} + +const IWeightedSetReadView<int16_t>* +ImportedAttributeVectorReadGuard::make_read_view(WeightedSetTag<int16_t> tag, vespalib::Stash& stash) const +{ + return make_read_view_helper(tag, stash); +} + +const IWeightedSetReadView<int32_t>* +ImportedAttributeVectorReadGuard::make_read_view(WeightedSetTag<int32_t> tag, vespalib::Stash& stash) const +{ + return make_read_view_helper(tag, stash); +} + +const IWeightedSetReadView<int64_t>* +ImportedAttributeVectorReadGuard::make_read_view(WeightedSetTag<int64_t> tag, vespalib::Stash& stash) const +{ + return make_read_view_helper(tag, stash); +} + +const IWeightedSetReadView<float>* +ImportedAttributeVectorReadGuard::make_read_view(WeightedSetTag<float> tag, vespalib::Stash& stash) const +{ + return make_read_view_helper(tag, stash); +} + +const IWeightedSetReadView<double>* +ImportedAttributeVectorReadGuard:: make_read_view(WeightedSetTag<double> tag, vespalib::Stash& stash) const +{ + return make_read_view_helper(tag, stash); +} + +const IWeightedSetReadView<const char*>* +ImportedAttributeVectorReadGuard::make_read_view(WeightedSetTag<const char*> tag, vespalib::Stash& stash) const +{ + return make_read_view_helper(tag, stash); +} + +const IArrayEnumReadView* +ImportedAttributeVectorReadGuard::make_read_view(ArrayEnumTag tag, vespalib::Stash& stash) const +{ + return make_read_view_helper(tag, stash); +} + +const IWeightedSetEnumReadView* +ImportedAttributeVectorReadGuard::make_read_view(WeightedSetEnumTag tag, vespalib::Stash& stash) const +{ + return make_read_view_helper(tag, stash); +} + bool ImportedAttributeVectorReadGuard::isUndefined(DocId doc) const { return _target_attribute.isUndefined(getTargetLid(doc)); } diff --git a/searchlib/src/vespa/searchlib/attribute/imported_attribute_vector_read_guard.h b/searchlib/src/vespa/searchlib/attribute/imported_attribute_vector_read_guard.h index f5b896e2da5..9e596078678 100644 --- a/searchlib/src/vespa/searchlib/attribute/imported_attribute_vector_read_guard.h +++ b/searchlib/src/vespa/searchlib/attribute/imported_attribute_vector_read_guard.h @@ -5,6 +5,7 @@ #include "attribute_read_guard.h" #include "attributeguard.h" #include <vespa/searchcommon/attribute/iattributevector.h> +#include <vespa/searchcommon/attribute/i_multi_value_attribute.h> #include <vespa/searchlib/common/i_document_meta_store_context.h> #include <vespa/vespalib/datastore/atomic_value_wrapper.h> #include <vespa/vespalib/util/arrayref.h> @@ -27,7 +28,8 @@ class ReferenceAttribute; * boundary check is setup during construction. */ class ImportedAttributeVectorReadGuard : public IAttributeVector, - public AttributeReadGuard + public AttributeReadGuard, + public IMultiValueAttribute { private: using AtomicTargetLid = vespalib::datastore::AtomicValueWrapper<uint32_t>; @@ -86,6 +88,24 @@ public: uint32_t getCommittedDocIdLimit() const override; bool isImported() const override; bool isUndefined(DocId doc) const override; + template <typename MultiValueType> + const IMultiValueReadView<MultiValueType>* make_read_view_helper(Tag<MultiValueType> tag, vespalib::Stash& stash) const; + const IArrayReadView<int8_t>* make_read_view(ArrayTag<int8_t> tag, vespalib::Stash& stash) const override; + const IArrayReadView<int16_t>* make_read_view(ArrayTag<int16_t> tag, vespalib::Stash& stash) const override; + const IArrayReadView<int32_t>* make_read_view(ArrayTag<int32_t> tag, vespalib::Stash& stash) const override; + const IArrayReadView<int64_t>* make_read_view(ArrayTag<int64_t> tag, vespalib::Stash& stash) const override; + const IArrayReadView<float>* make_read_view(ArrayTag<float> tag, vespalib::Stash& stash) const override; + const IArrayReadView<double>* make_read_view(ArrayTag<double> tag, vespalib::Stash& stash) const override; + const IArrayReadView<const char*>* make_read_view(ArrayTag<const char*> tag, vespalib::Stash& stash) const override; + const IWeightedSetReadView<int8_t>* make_read_view(WeightedSetTag<int8_t> tag, vespalib::Stash& stash) const override; + const IWeightedSetReadView<int16_t>* make_read_view(WeightedSetTag<int16_t> tag, vespalib::Stash& stash) const override; + const IWeightedSetReadView<int32_t>* make_read_view(WeightedSetTag<int32_t> tag, vespalib::Stash& stash) const override; + const IWeightedSetReadView<int64_t>* make_read_view(WeightedSetTag<int64_t> tag, vespalib::Stash& stash) const override; + const IWeightedSetReadView<float>* make_read_view(WeightedSetTag<float> tag, vespalib::Stash& stash) const override; + const IWeightedSetReadView<double>* make_read_view(WeightedSetTag<double> tag, vespalib::Stash& stash) const override; + const IWeightedSetReadView<const char*>* make_read_view(WeightedSetTag<const char*> tag, vespalib::Stash& stash) const override; + const IArrayEnumReadView* make_read_view(ArrayEnumTag tag, vespalib::Stash& stash) const override; + const IWeightedSetEnumReadView* make_read_view(WeightedSetEnumTag tag, vespalib::Stash& stash) const override; protected: long onSerializeForAscendingSort(DocId doc, void * serTo, long available, diff --git a/searchlib/src/vespa/searchlib/attribute/imported_multi_value_read_view.cpp b/searchlib/src/vespa/searchlib/attribute/imported_multi_value_read_view.cpp new file mode 100644 index 00000000000..b36d321694e --- /dev/null +++ b/searchlib/src/vespa/searchlib/attribute/imported_multi_value_read_view.cpp @@ -0,0 +1,47 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include "imported_multi_value_read_view.h" + +using vespalib::datastore::AtomicEntryRef; + +namespace search::attribute { + +template <typename MultiValueType> +ImportedMultiValueReadView<MultiValueType>::ImportedMultiValueReadView(TargetLids target_lids, const IMultiValueReadView<MultiValueType>* target_read_view) + : _target_lids(target_lids), + _target_read_view(target_read_view) +{ +} + +template <typename MultiValueType> +ImportedMultiValueReadView<MultiValueType>::~ImportedMultiValueReadView() = default; + +template <typename MultiValueType> +vespalib::ConstArrayRef<MultiValueType> +ImportedMultiValueReadView<MultiValueType>::get_values(uint32_t docid) const +{ + auto target_lid = get_target_lid(docid); + return _target_read_view->get_values(target_lid); +} + +using multivalue::WeightedValue; + +template class ImportedMultiValueReadView<int8_t>; +template class ImportedMultiValueReadView<int16_t>; +template class ImportedMultiValueReadView<int32_t>; +template class ImportedMultiValueReadView<int64_t>; +template class ImportedMultiValueReadView<float>; +template class ImportedMultiValueReadView<double>; +template class ImportedMultiValueReadView<AtomicEntryRef>; +template class ImportedMultiValueReadView<const char*>; + +template class ImportedMultiValueReadView<WeightedValue<int8_t>>; +template class ImportedMultiValueReadView<WeightedValue<int16_t>>; +template class ImportedMultiValueReadView<WeightedValue<int32_t>>; +template class ImportedMultiValueReadView<WeightedValue<int64_t>>; +template class ImportedMultiValueReadView<WeightedValue<float>>; +template class ImportedMultiValueReadView<WeightedValue<double>>; +template class ImportedMultiValueReadView<WeightedValue<AtomicEntryRef>>; +template class ImportedMultiValueReadView<WeightedValue<const char*>>; + +} diff --git a/searchlib/src/vespa/searchlib/attribute/imported_multi_value_read_view.h b/searchlib/src/vespa/searchlib/attribute/imported_multi_value_read_view.h new file mode 100644 index 00000000000..9ec54feca9e --- /dev/null +++ b/searchlib/src/vespa/searchlib/attribute/imported_multi_value_read_view.h @@ -0,0 +1,33 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#pragma once + +#include <vespa/searchcommon/attribute/i_multi_value_read_view.h> +#include <vespa/vespalib/datastore/atomic_value_wrapper.h> + +namespace search::attribute { + +/** + * Multi value read view adapter for imported atributes vectors. + * Performs lid mapping. + * @tparam MultiValueType The multi-value type of the data to access. + */ +template <typename MultiValueType> +class ImportedMultiValueReadView : public IMultiValueReadView<MultiValueType> +{ + using AtomicTargetLid = vespalib::datastore::AtomicValueWrapper<uint32_t>; + using TargetLids = vespalib::ConstArrayRef<AtomicTargetLid>; + TargetLids _target_lids; + const IMultiValueReadView<MultiValueType>* _target_read_view; + + uint32_t get_target_lid(uint32_t lid) const { + // Check range to avoid reading memory beyond end of mapping array + return lid < _target_lids.size() ? _target_lids[lid].load_acquire() : 0u; + } +public: + ImportedMultiValueReadView(TargetLids target_lids, const IMultiValueReadView<MultiValueType>* target_read_view); + ~ImportedMultiValueReadView() override; + vespalib::ConstArrayRef<MultiValueType> get_values(uint32_t docid) const override; +}; + +} diff --git a/searchlib/src/vespa/searchlib/attribute/multinumericenumattribute.h b/searchlib/src/vespa/searchlib/attribute/multinumericenumattribute.h index 1786429af8e..9f8506d3cb4 100644 --- a/searchlib/src/vespa/searchlib/attribute/multinumericenumattribute.h +++ b/searchlib/src/vespa/searchlib/attribute/multinumericenumattribute.h @@ -106,6 +106,10 @@ public: return getWeightedHelper<WeightedFloat, double>(doc, v, sz); } + // Implements attribute::IMultiValueAttribute + const attribute::IMultiValueReadView<T>* make_read_view(attribute::IMultiValueAttribute::Tag<T>, vespalib::Stash& stash) const override; + const attribute::IMultiValueReadView<multivalue::WeightedValue<T>>* make_read_view(attribute::IMultiValueAttribute::Tag<multivalue::WeightedValue<T>>, vespalib::Stash& stash) const override; + private: using AttributeReader = PrimitiveReader<typename B::LoadedValueType>; void loadAllAtOnce(AttributeReader & attrReader, size_t numDocs, size_t numValues); diff --git a/searchlib/src/vespa/searchlib/attribute/multinumericenumattribute.hpp b/searchlib/src/vespa/searchlib/attribute/multinumericenumattribute.hpp index 3323440dd0d..81f8c1c910e 100644 --- a/searchlib/src/vespa/searchlib/attribute/multinumericenumattribute.hpp +++ b/searchlib/src/vespa/searchlib/attribute/multinumericenumattribute.hpp @@ -4,10 +4,12 @@ #include "load_utils.h" #include "loadednumericvalue.h" +#include "enumerated_multi_value_read_view.h" #include "multinumericenumattribute.h" #include "multi_numeric_enum_search_context.h" #include <vespa/searchlib/query/query_term_simple.h> #include <vespa/searchlib/util/fileutil.hpp> +#include <vespa/vespalib/util/stash.h> namespace search { @@ -114,6 +116,20 @@ MultiValueNumericEnumAttribute<B, M>::onLoad(vespalib::Executor *) } template <typename B, typename M> +const attribute::IMultiValueReadView<typename B::BaseClass::BaseType>* +MultiValueNumericEnumAttribute<B, M>::make_read_view(attribute::IMultiValueAttribute::Tag<typename B::BaseClass::BaseType>, vespalib::Stash& stash) const +{ + return &stash.create<attribute::EnumeratedMultiValueReadView<T, M>>(this->_mvMapping.make_read_view(this->getCommittedDocIdLimit()), this->_enumStore); +} + +template <typename B, typename M> +const attribute::IMultiValueReadView<multivalue::WeightedValue<typename B::BaseClass::BaseType>>* +MultiValueNumericEnumAttribute<B, M>::make_read_view(attribute::IMultiValueAttribute::Tag<multivalue::WeightedValue<typename B::BaseClass::BaseType>>, vespalib::Stash& stash) const +{ + return &stash.create<attribute::EnumeratedMultiValueReadView<multivalue::WeightedValue<T>, M>>(this->_mvMapping.make_read_view(this->getCommittedDocIdLimit()), this->_enumStore); +} + +template <typename B, typename M> std::unique_ptr<attribute::SearchContext> MultiValueNumericEnumAttribute<B, M>::getSearch(QueryTermSimple::UP qTerm, const attribute::SearchContextParams & params) const diff --git a/searchlib/src/vespa/searchlib/attribute/multistringattribute.h b/searchlib/src/vespa/searchlib/attribute/multistringattribute.h index 6f94843e1b8..532af930220 100644 --- a/searchlib/src/vespa/searchlib/attribute/multistringattribute.h +++ b/searchlib/src/vespa/searchlib/attribute/multistringattribute.h @@ -105,6 +105,10 @@ public: std::unique_ptr<attribute::SearchContext> getSearch(QueryTermSimpleUP term, const attribute::SearchContextParams & params) const override; + + // Implements attribute::IMultiValueAttribute + const attribute::IMultiValueReadView<const char*>* make_read_view(attribute::IMultiValueAttribute::Tag<const char*>, vespalib::Stash& stash) const override; + const attribute::IMultiValueReadView<multivalue::WeightedValue<const char*>>* make_read_view(attribute::IMultiValueAttribute::Tag<multivalue::WeightedValue<const char*>>, vespalib::Stash& stash) const override; }; diff --git a/searchlib/src/vespa/searchlib/attribute/multistringattribute.hpp b/searchlib/src/vespa/searchlib/attribute/multistringattribute.hpp index 2d60887c23b..0edd459efc7 100644 --- a/searchlib/src/vespa/searchlib/attribute/multistringattribute.hpp +++ b/searchlib/src/vespa/searchlib/attribute/multistringattribute.hpp @@ -5,12 +5,14 @@ #include "stringattribute.h" #include "multistringattribute.h" #include "enumattribute.hpp" +#include "enumerated_multi_value_read_view.h" #include "multienumattribute.hpp" #include "multi_string_enum_hint_search_context.h" #include <vespa/vespalib/text/utf8.h> #include <vespa/vespalib/text/lowercase.h> #include <vespa/searchlib/util/bufferwriter.h> #include <vespa/vespalib/util/regexp.h> +#include <vespa/vespalib/util/stash.h> #include <vespa/searchlib/query/query_term_ucs4.h> namespace search { @@ -47,5 +49,19 @@ MultiValueStringAttributeT<B, M>::getSearch(QueryTermSimpleUP qTerm, return std::make_unique<attribute::MultiStringEnumHintSearchContext<M>>(std::move(qTerm), cased, *this, this->_mvMapping.make_read_view(doc_id_limit), this->_enumStore, doc_id_limit, this->getStatus().getNumValues()); } +template <typename B, typename M> +const attribute::IMultiValueReadView<const char*>* +MultiValueStringAttributeT<B, M>::make_read_view(attribute::IMultiValueAttribute::Tag<const char*>, vespalib::Stash& stash) const +{ + return &stash.create<attribute::EnumeratedMultiValueReadView<const char*, M>>(this->_mvMapping.make_read_view(this->getCommittedDocIdLimit()), this->_enumStore); +} + +template <typename B, typename M> +const attribute::IMultiValueReadView<multivalue::WeightedValue<const char*>>* +MultiValueStringAttributeT<B, M>::make_read_view(attribute::IMultiValueAttribute::Tag<multivalue::WeightedValue<const char*>>, vespalib::Stash& stash) const +{ + return &stash.create<attribute::EnumeratedMultiValueReadView<multivalue::WeightedValue<const char*>, M>>(this->_mvMapping.make_read_view(this->getCommittedDocIdLimit()), this->_enumStore); +} + } // namespace search diff --git a/searchlib/src/vespa/searchlib/attribute/multivalueattribute.h b/searchlib/src/vespa/searchlib/attribute/multivalueattribute.h index 3844c0f9b02..5a2ee5c80d9 100644 --- a/searchlib/src/vespa/searchlib/attribute/multivalueattribute.h +++ b/searchlib/src/vespa/searchlib/attribute/multivalueattribute.h @@ -82,7 +82,8 @@ public: const IMultiValueAttribute* as_multi_value_attribute() const override; // Implements attribute::IMultiValueAttribute - const attribute::IMultiValueReadView<MultiValueType>* make_read_view(attribute::IMultiValueAttribute::Tag<MultiValueType>, vespalib::Stash& stash) const override; + const attribute::IMultiValueReadView<ValueType>* make_read_view(attribute::IMultiValueAttribute::Tag<ValueType>, vespalib::Stash& stash) const override; + const attribute::IMultiValueReadView<multivalue::WeightedValue<ValueType>>* make_read_view(attribute::IMultiValueAttribute::Tag<multivalue::WeightedValue<ValueType>>, vespalib::Stash& stash) const override; }; } // namespace search diff --git a/searchlib/src/vespa/searchlib/attribute/multivalueattribute.hpp b/searchlib/src/vespa/searchlib/attribute/multivalueattribute.hpp index 04194f662d9..4e0e460da9c 100644 --- a/searchlib/src/vespa/searchlib/attribute/multivalueattribute.hpp +++ b/searchlib/src/vespa/searchlib/attribute/multivalueattribute.hpp @@ -4,6 +4,7 @@ #include "address_space_components.h" #include "raw_multi_value_read_view.h" +#include "copy_multi_value_read_view.h" #include <vespa/searchlib/attribute/multivalueattribute.h> #include <vespa/vespalib/stllike/hash_map.h> #include <vespa/vespalib/stllike/hash_map.hpp> @@ -299,10 +300,25 @@ MultiValueAttribute<B, M>::as_multi_value_attribute() const } template <typename B, typename M> -const attribute::IMultiValueReadView<M>* -MultiValueAttribute<B, M>::make_read_view(attribute::IMultiValueAttribute::Tag<MultiValueType>, vespalib::Stash& stash) const +const attribute::IMultiValueReadView<multivalue::ValueType_t<M>>* +MultiValueAttribute<B, M>::make_read_view(attribute::IMultiValueAttribute::Tag<ValueType>, vespalib::Stash& stash) const { - return &stash.create<attribute::RawMultiValueReadView<MultiValueType>>(this->_mvMapping.make_read_view(this->getCommittedDocIdLimit())); + if constexpr (std::is_same_v<MultiValueType, ValueType>) { + return &stash.create<attribute::RawMultiValueReadView<MultiValueType>>(this->_mvMapping.make_read_view(this->getCommittedDocIdLimit())); + } else { + return &stash.create<attribute::CopyMultiValueReadView<ValueType, MultiValueType>>(this->_mvMapping.make_read_view(this->getCommittedDocIdLimit())); + } +} + +template <typename B, typename M> +const attribute::IMultiValueReadView<multivalue::WeightedValue<multivalue::ValueType_t<M>>>* +MultiValueAttribute<B, M>::make_read_view(attribute::IMultiValueAttribute::Tag<multivalue::WeightedValue<ValueType>>, vespalib::Stash& stash) const +{ + if constexpr (std::is_same_v<MultiValueType, multivalue::WeightedValue<ValueType>>) { + return &stash.create<attribute::RawMultiValueReadView<MultiValueType>>(this->_mvMapping.make_read_view(this->getCommittedDocIdLimit())); + } else { + return &stash.create<attribute::CopyMultiValueReadView<multivalue::WeightedValue<ValueType>, MultiValueType>>(this->_mvMapping.make_read_view(this->getCommittedDocIdLimit())); + } } } // namespace search diff --git a/searchlib/src/vespa/searchlib/features/dotproductfeature.cpp b/searchlib/src/vespa/searchlib/features/dotproductfeature.cpp index 8f75b6ecc7d..1ee5fe90773 100644 --- a/searchlib/src/vespa/searchlib/features/dotproductfeature.cpp +++ b/searchlib/src/vespa/searchlib/features/dotproductfeature.cpp @@ -52,49 +52,6 @@ template class VectorBase<uint32_t, uint32_t, double>; template class IntegerVectorT<int64_t>; - -template <typename Vector, typename Buffer> -DotProductExecutorByCopy<Vector, Buffer>::DotProductExecutorByCopy(const IAttributeVector * attribute, const Vector & queryVector) : - FeatureExecutor(), - _attribute(attribute), - _queryVector(queryVector), - _end(_queryVector.getDimMap().end()), - _buffer(), - _backing() -{ - _buffer.allocate(_attribute->getMaxValueCount()); -} - -template <typename Vector, typename Buffer> -DotProductExecutorByCopy<Vector, Buffer>::DotProductExecutorByCopy(const IAttributeVector * attribute, std::unique_ptr<Vector> queryVector) : - FeatureExecutor(), - _attribute(attribute), - _queryVector(*queryVector), - _end(_queryVector.getDimMap().end()), - _buffer(), - _backing(std::move(queryVector)) -{ - _buffer.allocate(_attribute->getMaxValueCount()); -} - -template <typename Vector, typename Buffer> -DotProductExecutorByCopy<Vector, Buffer>::~DotProductExecutorByCopy() = default; - -template <typename Vector, typename Buffer> -void -DotProductExecutorByCopy<Vector, Buffer>::execute(uint32_t docId) -{ - feature_t val = 0; - _buffer.fill(*_attribute, docId); - for (size_t i = 0; i < _buffer.size(); ++i) { - auto itr = _queryVector.getDimMap().find(_buffer[i].getValue()); - if (itr != _end) { - val += _buffer[i].getWeight() * itr->second; - } - } - outputs().set_number(0, val); -} - StringVector::StringVector() = default; StringVector::~StringVector() = default; @@ -229,6 +186,7 @@ template <typename BaseType> class SingleDotProductByWeightedValueExecutor final : public fef::FeatureExecutor { public: using WeightedSetReadView = attribute::IWeightedSetReadView<BaseType>; + using StoredKeyType = std::conditional_t<std::is_same_v<BaseType,const char*>,vespalib::string,BaseType>; SingleDotProductByWeightedValueExecutor(const WeightedSetReadView * weighted_set_read_view, BaseType key, feature_t value) : _weighted_set_read_view(weighted_set_read_view), _key(key), @@ -247,7 +205,7 @@ public: } private: const WeightedSetReadView* _weighted_set_read_view; - BaseType _key; + StoredKeyType _key; feature_t _value; }; @@ -650,23 +608,38 @@ std::pair<T, feature_t> extractElem(const std::unique_ptr<dotproduct::wset::Inte return extractElem(*v, idx); } -template <typename A, typename V> +size_t extractSize(const dotproduct::wset::StringVector& v) { + return v.getVector().size(); +} + +std::pair<const char*, feature_t> extractElem(const dotproduct::wset::StringVector& v, size_t idx) { + const auto & pair = v.getVector()[idx]; + return std::pair<const char*, feature_t>(pair.first.c_str(), pair.second); +} + +size_t extractSize(const std::unique_ptr<dotproduct::wset::StringVector>& v) { + return extractSize(*v); +} + +std::pair<const char*, feature_t> extractElem(const std::unique_ptr<dotproduct::wset::StringVector>& v, size_t idx) { + return extractElem(*v, idx); +} + +template <typename T, typename V> FeatureExecutor & createForDirectWSetImpl(const IAttributeVector * attribute, V && vector, vespalib::Stash & stash) { using namespace dotproduct::wset; - using T = typename A::BaseType; - const A * iattr = dynamic_cast<const A *>(attribute); using VT = multivalue::WeightedValue<T>; auto weighted_set_read_view = make_multi_value_read_view<VT>(*attribute, stash); - if (!attribute->isImported() && (iattr != nullptr) && weighted_set_read_view != nullptr) { + if (weighted_set_read_view != nullptr) { if (extractSize(vector) == 1) { auto elem = extractElem(vector, 0ul); return stash.create<SingleDotProductByWeightedValueExecutor<T>>(weighted_set_read_view, elem.first, elem.second); } return stash.create<DotProductByWeightedSetReadViewExecutor<T>>(weighted_set_read_view, std::forward<V>(vector)); } - return stash.create<DotProductExecutorByCopy<IntegerVectorT<T>, WeightedIntegerContent>>(attribute, std::forward<V>(vector)); + return stash.create<SingleZeroValueExecutor>(); } template <typename T> @@ -676,7 +649,7 @@ createForDirectIntegerWSet(const IAttributeVector * attribute, const dotproduct: using namespace dotproduct::wset; return vector.empty() ? stash.create<SingleZeroValueExecutor>() - : createForDirectWSetImpl<IntegerAttributeTemplate<T>>(attribute, vector, stash); + : createForDirectWSetImpl<T>(attribute, vector, stash); } FeatureExecutor & @@ -727,14 +700,13 @@ createFromObject(const IAttributeVector * attribute, const fef::Anything & objec } return stash.create<DotProductExecutorByEnum>(weighted_set_enum_read_view, vector); } - return stash.create<DotProductExecutorByCopy<EnumVector, WeightedEnumContent>>(attribute, vector); } else { if (attribute->isStringType()) { const auto & vector = dynamic_cast<const StringVector &>(object); if (vector.empty()) { return stash.create<SingleZeroValueExecutor>(); } - return stash.create<DotProductExecutorByCopy<StringVector, WeightedConstCharContent>>(attribute, vector); + return createForDirectWSetImpl<const char*>(attribute, vector, stash); } else if (attribute->isIntegerType()) { if (attribute->getBasicType() == BasicType::INT32) { return createForDirectIntegerWSet<int32_t>(attribute, dynamic_cast<const IntegerVectorT<int32_t> &>(object), stash); @@ -800,7 +772,7 @@ createForDirectIntegerWSet(const IAttributeVector * attribute, const Property & vector->syncMap(); return vector->empty() ? stash.create<SingleZeroValueExecutor>() - : createForDirectWSetImpl<IntegerAttributeTemplate<T>>(attribute, std::move(vector), stash); + : createForDirectWSetImpl<T>(attribute, std::move(vector), stash); } FeatureExecutor & @@ -822,7 +794,6 @@ createTypedWsetExecutor(const IAttributeVector * attribute, const Property & pro } return stash.create<DotProductExecutorByEnum>(weighted_set_enum_read_view, std::move(vector)); } - return stash.create<DotProductExecutorByCopy<EnumVector, WeightedEnumContent>>(attribute, std::move(vector)); } else { if (attribute->isStringType()) { auto vector = std::make_unique<StringVector>(); @@ -831,7 +802,7 @@ createTypedWsetExecutor(const IAttributeVector * attribute, const Property & pro return stash.create<SingleZeroValueExecutor>(); } vector->syncMap(); - return stash.create<DotProductExecutorByCopy<StringVector, WeightedConstCharContent>>(attribute, std::move(vector)); + return createForDirectWSetImpl<const char*>(attribute, std::move(vector), stash); } else if (attribute->isIntegerType()) { if (attribute->getBasicType() == BasicType::INT32) { return createForDirectIntegerWSet<int32_t>(attribute, prop, stash); diff --git a/searchlib/src/vespa/searchlib/features/dotproductfeature.h b/searchlib/src/vespa/searchlib/features/dotproductfeature.h index 051a08025d8..ee0a85e689b 100644 --- a/searchlib/src/vespa/searchlib/features/dotproductfeature.h +++ b/searchlib/src/vespa/searchlib/features/dotproductfeature.h @@ -65,11 +65,14 @@ public: bool empty() const { return _vector.empty(); } }; +template <typename T> +using NumericVectorBaseT = VectorBase<T, T, feature_t>; + /** * Represents a vector where the dimensions are integers. **/ template<typename T> -class IntegerVectorT : public VectorBase<T, T, feature_t> { +class IntegerVectorT : public NumericVectorBaseT<T> { public: void insert(vespalib::stringref label, vespalib::stringref value) { this->_vector.emplace_back(util::strToNum<T>(label), util::strToNum<feature_t>(value)); @@ -82,10 +85,12 @@ extern template class IntegerVectorT<int64_t>; using IntegerVector = IntegerVectorT<int64_t>; +using StringVectorBase = VectorBase<vespalib::string, const char*, feature_t, ConstCharComparator>; + /** * Represents a vector where the dimensions are string values. **/ -class StringVector : public VectorBase<vespalib::string, const char *, feature_t, ConstCharComparator> { +class StringVector : public StringVectorBase { public: StringVector(); StringVector(StringVector &&) = default; @@ -121,7 +126,7 @@ template <typename BaseType> class DotProductExecutorBase : public fef::FeatureExecutor { public: using AT = multivalue::WeightedValue<BaseType>; - using V = VectorBase<BaseType, BaseType, feature_t>; + using V = std::conditional_t<std::is_same_v<BaseType,const char*>,StringVectorBase,NumericVectorBaseT<BaseType>>; private: const V & _queryVector; const typename V::HashMap::const_iterator _end; @@ -149,25 +154,6 @@ public: ~DotProductByWeightedSetReadViewExecutor(); }; - -/** - * Implements the executor for the dotproduct feature. - */ -template <typename Vector, typename Buffer> -class DotProductExecutorByCopy final : public fef::FeatureExecutor { -private: - const attribute::IAttributeVector * _attribute; - const Vector & _queryVector; - const typename Vector::HashMap::const_iterator _end; - Buffer _buffer; - std::unique_ptr<Vector> _backing; -public: - DotProductExecutorByCopy(const attribute::IAttributeVector * attribute, const Vector & queryVector); - DotProductExecutorByCopy(const attribute::IAttributeVector * attribute, std::unique_ptr<Vector> queryVector); - ~DotProductExecutorByCopy() override; - void execute(uint32_t docId) override; -}; - } namespace array { diff --git a/searchlib/src/vespa/searchlib/tensor/hnsw_index.cpp b/searchlib/src/vespa/searchlib/tensor/hnsw_index.cpp index 5ae26757b0d..8327db769d7 100644 --- a/searchlib/src/vespa/searchlib/tensor/hnsw_index.cpp +++ b/searchlib/src/vespa/searchlib/tensor/hnsw_index.cpp @@ -157,6 +157,7 @@ HnswIndex::shrink_if_needed(uint32_t docid, uint32_t level) uint32_t max_links = max_links_for_level(level); if (old_links.size() > max_links) { HnswCandidateVector neighbors; + neighbors.reserve(old_links.size()); for (uint32_t neighbor_docid : old_links) { double dist = calc_distance(docid, neighbor_docid); neighbors.emplace_back(neighbor_docid, dist); diff --git a/serviceview/OWNERS b/serviceview/OWNERS deleted file mode 100644 index 338ed581212..00000000000 --- a/serviceview/OWNERS +++ /dev/null @@ -1 +0,0 @@ -hmusum diff --git a/serviceview/README.md b/serviceview/README.md deleted file mode 100644 index e74c32feab4..00000000000 --- a/serviceview/README.md +++ /dev/null @@ -1,2 +0,0 @@ -<!-- Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. --> -# REST API for getting a view of services and hosts for an application diff --git a/serviceview/pom.xml b/serviceview/pom.xml deleted file mode 100644 index fb124f49624..00000000000 --- a/serviceview/pom.xml +++ /dev/null @@ -1,44 +0,0 @@ -<?xml version="1.0"?> -<!-- Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. --> -<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd"> - <modelVersion>4.0.0</modelVersion> - - <parent> - <groupId>com.yahoo.vespa</groupId> - <artifactId>parent</artifactId> - <version>7-SNAPSHOT</version> - <relativePath>../parent/pom.xml</relativePath> - </parent> - <artifactId>serviceview</artifactId> - <packaging>container-plugin</packaging> - <version>7-SNAPSHOT</version> - <name>serviceview</name> - <description>REST API that is shared between the Vespa Config Server and others.</description> - <dependencies> - <dependency> - <groupId>com.yahoo.vespa</groupId> - <artifactId>container-dev</artifactId> - <version>${project.version}</version> - <scope>provided</scope> - </dependency> - <dependency> - <groupId>junit</groupId> - <artifactId>junit</artifactId> - <scope>test</scope> - </dependency> - </dependencies> - <build> - <plugins> - <plugin> - <groupId>com.yahoo.vespa</groupId> - <artifactId>bundle-plugin</artifactId> - <extensions>true</extensions> - </plugin> - <plugin> - <!-- Explicit for IntelliJ to detect correct language level from parent --> - <groupId>org.apache.maven.plugins</groupId> - <artifactId>maven-compiler-plugin</artifactId> - </plugin> - </plugins> - </build> -</project> diff --git a/serviceview/src/main/java/com/yahoo/vespa/serviceview/bindings/ApplicationView.java b/serviceview/src/main/java/com/yahoo/vespa/serviceview/bindings/ApplicationView.java deleted file mode 100644 index 2b6c2a718c3..00000000000 --- a/serviceview/src/main/java/com/yahoo/vespa/serviceview/bindings/ApplicationView.java +++ /dev/null @@ -1,15 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.serviceview.bindings; - -import java.util.List; - -/** - * All clusters of a deployed application. - * - * @author Steinar Knutsen - */ -public class ApplicationView { - - public List<ClusterView> clusters; - -} diff --git a/serviceview/src/main/java/com/yahoo/vespa/serviceview/bindings/ClusterView.java b/serviceview/src/main/java/com/yahoo/vespa/serviceview/bindings/ClusterView.java deleted file mode 100644 index 47dbc5193be..00000000000 --- a/serviceview/src/main/java/com/yahoo/vespa/serviceview/bindings/ClusterView.java +++ /dev/null @@ -1,20 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.serviceview.bindings; - -import java.util.List; - -import com.fasterxml.jackson.annotation.JsonInclude; -import com.fasterxml.jackson.annotation.JsonInclude.Include; - -/** - * The response binding for a complete cluster. - * - * @author Steinar Knutsen - */ -public class ClusterView { - public String name; - public String type; - @JsonInclude(value = Include.NON_NULL) - public String url; - public List<ServiceView> services; -} diff --git a/serviceview/src/main/java/com/yahoo/vespa/serviceview/bindings/ConfigClient.java b/serviceview/src/main/java/com/yahoo/vespa/serviceview/bindings/ConfigClient.java deleted file mode 100644 index f0dff70eece..00000000000 --- a/serviceview/src/main/java/com/yahoo/vespa/serviceview/bindings/ConfigClient.java +++ /dev/null @@ -1,25 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.serviceview.bindings; - -import javax.ws.rs.GET; -import javax.ws.rs.Path; -import javax.ws.rs.PathParam; -import javax.ws.rs.Produces; -import javax.ws.rs.core.MediaType; - -/** - * Client to fetch the model config from the configserver. - * - * @author Steinar Knutsen - */ -public interface ConfigClient { - - @GET - @Path("/config/v2/tenant/{tenantName}/application/{applicationName}/environment/{environmentName}/region/{regionName}/instance/{instanceName}/cloud.config.model") - @Produces(MediaType.APPLICATION_JSON) - ModelResponse getServiceModel(@PathParam("tenantName") String tenantName, - @PathParam("applicationName") String applicationName, - @PathParam("environmentName") String environmentName, - @PathParam("regionName") String regionName, - @PathParam("instanceName") String instanceName); -} diff --git a/serviceview/src/main/java/com/yahoo/vespa/serviceview/bindings/HealthClient.java b/serviceview/src/main/java/com/yahoo/vespa/serviceview/bindings/HealthClient.java deleted file mode 100644 index 41b7e10f511..00000000000 --- a/serviceview/src/main/java/com/yahoo/vespa/serviceview/bindings/HealthClient.java +++ /dev/null @@ -1,23 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.serviceview.bindings; - -import java.util.HashMap; - -import javax.ws.rs.GET; -import javax.ws.rs.Path; -import javax.ws.rs.Produces; -import javax.ws.rs.core.MediaType; - -/** - * Client to fetch the model config from the configserver. - * - * @author Steinar Knutsen - */ -public interface HealthClient { - - @SuppressWarnings("rawtypes") - @GET - @Path("") - @Produces(MediaType.APPLICATION_JSON) - public HashMap getHealthInfo(); -} diff --git a/serviceview/src/main/java/com/yahoo/vespa/serviceview/bindings/HostService.java b/serviceview/src/main/java/com/yahoo/vespa/serviceview/bindings/HostService.java deleted file mode 100644 index 97425235cef..00000000000 --- a/serviceview/src/main/java/com/yahoo/vespa/serviceview/bindings/HostService.java +++ /dev/null @@ -1,19 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.serviceview.bindings; - -import java.util.List; - -import com.fasterxml.jackson.annotation.JsonIgnoreProperties; - -/** - * View of {@link com.yahoo.cloud.config.ModelConfig.Hosts}. - * - * @author mortent - */ -@JsonIgnoreProperties(ignoreUnknown = true) -public class HostService { - - public String name; - public List<Service> services; - -} diff --git a/serviceview/src/main/java/com/yahoo/vespa/serviceview/bindings/ModelResponse.java b/serviceview/src/main/java/com/yahoo/vespa/serviceview/bindings/ModelResponse.java deleted file mode 100644 index 9ce331605ef..00000000000 --- a/serviceview/src/main/java/com/yahoo/vespa/serviceview/bindings/ModelResponse.java +++ /dev/null @@ -1,17 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.serviceview.bindings; - -import java.util.List; - -import com.fasterxml.jackson.annotation.JsonIgnoreProperties; - -/** - * View of {@link com.yahoo.cloud.config.ModelConfig}. - * - * @author mortent - */ -@JsonIgnoreProperties(ignoreUnknown = true) -public class ModelResponse { - public List<HostService> hosts; - public String vespaVersion; -} diff --git a/serviceview/src/main/java/com/yahoo/vespa/serviceview/bindings/Service.java b/serviceview/src/main/java/com/yahoo/vespa/serviceview/bindings/Service.java deleted file mode 100644 index 4f38f1275f0..00000000000 --- a/serviceview/src/main/java/com/yahoo/vespa/serviceview/bindings/Service.java +++ /dev/null @@ -1,23 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.serviceview.bindings; - -import java.util.List; - -import com.fasterxml.jackson.annotation.JsonIgnoreProperties; - -/** - * View of {@link com.yahoo.cloud.config.ModelConfig.Hosts.Services}. - * - * @author mortent - */ -@JsonIgnoreProperties(ignoreUnknown = true) -public class Service { - public String name; - public String type; - public String configid; - public String clustertype; - public String clustername; - public long index; - public List<ServicePort> ports; - -} diff --git a/serviceview/src/main/java/com/yahoo/vespa/serviceview/bindings/ServicePort.java b/serviceview/src/main/java/com/yahoo/vespa/serviceview/bindings/ServicePort.java deleted file mode 100644 index 29fa07b46e7..00000000000 --- a/serviceview/src/main/java/com/yahoo/vespa/serviceview/bindings/ServicePort.java +++ /dev/null @@ -1,41 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.serviceview.bindings; - -import java.util.List; - -import com.fasterxml.jackson.annotation.JsonIgnoreProperties; -import com.google.common.base.Splitter; - -/** - * View of {@link com.yahoo.cloud.config.ModelConfig.Hosts.Services.Ports}. - * - * @author mortent - * @author Steinar Knutsen - */ -@JsonIgnoreProperties(value = { "splitOnSpace" }, ignoreUnknown = true) -public class ServicePort { - public int number; - public String tags; - private static final Splitter splitOnSpace = Splitter.on(' '); - - /** - * Return true if all argument tags are present for this port. - * - * @param tag - * one or more tag names to check for - * @return true if all argument tags are present for this port, false - * otherwise - */ - public boolean hasTags(String... tag) { - if (tags == null) { - return false; - } - List<String> isTaggedWith = splitOnSpace.splitToList(tags); - for (String t : tag) { - if (!isTaggedWith.contains(t)) { - return false; - } - } - return true; - } -} diff --git a/serviceview/src/main/java/com/yahoo/vespa/serviceview/bindings/ServiceView.java b/serviceview/src/main/java/com/yahoo/vespa/serviceview/bindings/ServiceView.java deleted file mode 100644 index e9fece65758..00000000000 --- a/serviceview/src/main/java/com/yahoo/vespa/serviceview/bindings/ServiceView.java +++ /dev/null @@ -1,20 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.serviceview.bindings; - -import com.fasterxml.jackson.annotation.JsonInclude; -import com.fasterxml.jackson.annotation.JsonInclude.Include; - -/** - * The response wrapper for the link to a single service state API. - * - * @author Steinar Knutsen - */ -public class ServiceView { - public String url; - public String serviceType; - public String serviceName; - public String configId; - public String host; - @JsonInclude(value = Include.NON_NULL) - public String legacyStatusPages; -} diff --git a/serviceview/src/main/java/com/yahoo/vespa/serviceview/bindings/StateClient.java b/serviceview/src/main/java/com/yahoo/vespa/serviceview/bindings/StateClient.java deleted file mode 100644 index 66e518e23e5..00000000000 --- a/serviceview/src/main/java/com/yahoo/vespa/serviceview/bindings/StateClient.java +++ /dev/null @@ -1,41 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.serviceview.bindings; - -import javax.ws.rs.GET; -import javax.ws.rs.Path; -import javax.ws.rs.PathParam; -import javax.ws.rs.Produces; -import javax.ws.rs.core.MediaType; -import java.util.HashMap; - -/** - * @author Steinar Knutsen - */ -public interface StateClient { - - @GET - @Path("v1/") - @Produces(MediaType.APPLICATION_JSON) - ApplicationView getDefaultUserInfo(); - - @GET - @Path("v1/tenant/{tenantName}/application/{applicationName}/environment/{environmentName}/region/{regionName}/instance/{instanceName}") - @Produces(MediaType.APPLICATION_JSON) - ApplicationView getUserInfo(@PathParam("tenantName") String tenantName, - @PathParam("applicationName") String applicationName, - @PathParam("environmentName") String environmentName, - @PathParam("regionName") String regionName, - @PathParam("instanceName") String instanceName); - - @SuppressWarnings("rawtypes") - @GET - @Path("v1/tenant/{tenantName}/application/{applicationName}/environment/{environmentName}/region/{regionName}/instance/{instanceName}/service/{serviceIdentifier}/{apiParams: .*}") - @Produces(MediaType.APPLICATION_JSON) - HashMap singleService(@PathParam("tenantName") String tenantName, - @PathParam("applicationName") String applicationName, - @PathParam("environmentName") String environmentName, - @PathParam("regionName") String regionName, - @PathParam("instanceName") String instanceName, - @PathParam("serviceIdentifier") String identifier, - @PathParam("apiParams") String apiParams); -} diff --git a/serviceview/src/main/java/com/yahoo/vespa/serviceview/bindings/package-info.java b/serviceview/src/main/java/com/yahoo/vespa/serviceview/bindings/package-info.java deleted file mode 100644 index d73a9023b7f..00000000000 --- a/serviceview/src/main/java/com/yahoo/vespa/serviceview/bindings/package-info.java +++ /dev/null @@ -1,11 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -/** - * The API bindings for the serviceview and underlying state APIs. - * - * <p>Do note this package is in its prototyping stage and classes <i>will</i> - * be renamed and moved around a little.</p> - */ -@ExportPackage -package com.yahoo.vespa.serviceview.bindings; - -import com.yahoo.osgi.annotation.ExportPackage; diff --git a/serviceview/src/test/java/com/yahoo/vespa/serviceview/bindings/ServicePortTest.java b/serviceview/src/test/java/com/yahoo/vespa/serviceview/bindings/ServicePortTest.java deleted file mode 100644 index 2e3d0315c4a..00000000000 --- a/serviceview/src/test/java/com/yahoo/vespa/serviceview/bindings/ServicePortTest.java +++ /dev/null @@ -1,78 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.serviceview.bindings; - -import static org.junit.Assert.*; - -import org.junit.Test; - -/** - * Functional test of tag checking. - * - * @author <a href="mailto:steinar@yahoo-inc.com">Steinar Knutsen</a> - */ -public class ServicePortTest { - - @Test - public final void testNullTags() { - ServicePort s = new ServicePort(); - s.tags = null; - assertFalse(s.hasTags("http")); - } - - @Test - public final void testEmptyTags() { - ServicePort s = new ServicePort(); - s.tags = ""; - assertFalse(s.hasTags("http")); - } - - @Test - public final void testSubsetInArgs() { - ServicePort s = new ServicePort(); - s.tags = "http state status"; - assertTrue(s.hasTags("http", "state")); - } - - @Test - public final void testSupersetInArgs() { - ServicePort s = new ServicePort(); - s.tags = "http"; - assertFalse(s.hasTags("http", "rpc")); - } - - @Test - public final void testIdenticalInArgs() { - ServicePort s = new ServicePort(); - s.tags = "http rpc"; - assertTrue(s.hasTags("http", "rpc")); - } - - @Test - public final void testDisjunctBetweenArgsAndTags() { - ServicePort s = new ServicePort(); - s.tags = "http state moo"; - assertFalse(s.hasTags("http", "state", "rpc")); - } - - @Test - public final void testTagNameStartsWithOther() { - ServicePort s = new ServicePort(); - s.tags = "http state moo"; - assertFalse(s.hasTags("htt")); - } - - @Test - public final void testTagNameEndsWithOther() { - ServicePort s = new ServicePort(); - s.tags = "http state moo"; - assertFalse(s.hasTags("tp")); - } - - @Test - public final void testTagNameIsSubstringofOther() { - ServicePort s = new ServicePort(); - s.tags = "http state moo"; - assertFalse(s.hasTags("tt")); - } - -} diff --git a/storage/src/vespa/storage/config/stor-distributormanager.def b/storage/src/vespa/storage/config/stor-distributormanager.def index 1d2d4babf74..0a858ba37c3 100644 --- a/storage/src/vespa/storage/config/stor-distributormanager.def +++ b/storage/src/vespa/storage/config/stor-distributormanager.def @@ -298,4 +298,4 @@ use_unordered_merge_chaining bool default=true ## cluster state bundle indicates that global merges are pending in the cluster, i.e. ## one or more nodes is in maintenance mode in the default bucket space but marked up in ## the global bucket space. -inhibit_default_merges_when_global_merges_pending bool default=false +inhibit_default_merges_when_global_merges_pending bool default=true diff --git a/vespa-application-maven-plugin/src/main/java/com/yahoo/container/plugin/mojo/ApplicationMojo.java b/vespa-application-maven-plugin/src/main/java/com/yahoo/container/plugin/mojo/ApplicationMojo.java index 1bbdfc9914a..6fdcf6efe87 100644 --- a/vespa-application-maven-plugin/src/main/java/com/yahoo/container/plugin/mojo/ApplicationMojo.java +++ b/vespa-application-maven-plugin/src/main/java/com/yahoo/container/plugin/mojo/ApplicationMojo.java @@ -2,6 +2,7 @@ package com.yahoo.container.plugin.mojo; import org.apache.commons.io.FileUtils; +import org.apache.maven.artifact.Artifact; import org.apache.maven.plugin.AbstractMojo; import org.apache.maven.plugin.MojoExecutionException; import org.apache.maven.plugins.annotations.LifecyclePhase; @@ -56,15 +57,30 @@ public class ApplicationMojo extends AbstractMojo { /** Writes meta data about this package if the destination directory exists. */ private void addBuildMetaData(File applicationDestination) throws MojoExecutionException { - if ( ! applicationDestination.exists()) - return; + if ( ! applicationDestination.exists()) return; - if (vespaversion == null) // Get the build version of the parent project unless specifically set. - vespaversion = project.getProperties().getProperty("vespaversion"); + if (vespaversion == null) + vespaversion = project.getPlugin("com.yahoo.vespa:vespa-application-maven-plugin").getVersion(); - String metaData = String.format("{\"compileVersion\": \"%s\",\n \"buildTime\": %d}", - vespaversion, - System.currentTimeMillis()); + Version compileVersion = Version.from(vespaversion); + + MavenProject current = project; + while (current.getParent() != null && current.getParent().getParentArtifact() != null) + current = current.getParent(); + + boolean hasVespaParent = false; + Artifact parentArtifact = current.getParentArtifact(); + if (parentArtifact != null && (parentArtifact.getGroupId().startsWith("com.yahoo.vespa.") || parentArtifact.getGroupId().startsWith("ai.vespa."))) { + hasVespaParent = true; + Version parentVersion = Version.from(parentArtifact.getVersion()); + if (parentVersion.compareTo(compileVersion) < 0) + throw new IllegalArgumentException("compile version (" + compileVersion + ") cannot be higher than parent version (" + parentVersion + ")"); + } + + String metaData = String.format("{\n \"compileVersion\": \"%s\",\n \"buildTime\": %d,\n \"hasVespaParent\": %b\n}", + compileVersion, + System.currentTimeMillis(), + hasVespaParent); try { Files.write(applicationDestination.toPath().resolve("build-meta.json"), metaData.getBytes(StandardCharsets.UTF_8)); diff --git a/vespa-application-maven-plugin/src/main/java/com/yahoo/container/plugin/mojo/Version.java b/vespa-application-maven-plugin/src/main/java/com/yahoo/container/plugin/mojo/Version.java new file mode 100644 index 00000000000..050871043ca --- /dev/null +++ b/vespa-application-maven-plugin/src/main/java/com/yahoo/container/plugin/mojo/Version.java @@ -0,0 +1,80 @@ +package com.yahoo.container.plugin.mojo; + +import java.util.Comparator; +import java.util.Objects; + +import static java.lang.Integer.min; +import static java.lang.Integer.parseInt; + +/** + * @author jonmv + */ +public class Version implements Comparable<Version> { + + private static final Comparator<Version> comparator = Comparator.comparingInt(Version::major) + .thenComparing(Version::isSnapshot) + .thenComparing(Version::minor) + .thenComparing(Version::micro); + + private final int major; + private final int minor; + private final int micro; + private final boolean snapshot; + + private Version(int major, int minor, int micro, boolean snapshot) { + if (major < 0 || minor < 0 || micro < 0) throw new IllegalArgumentException("version numbers must all be non-negative"); + this.major = major; + this.minor = minor; + this.micro = micro; + this.snapshot = snapshot; + } + + public static Version of(int major, int minor, int micro) { + return new Version(major, minor, micro, false); + } + + public static Version ofSnapshot(int major) { + return new Version(major, 0, 0, true); + } + + public static Version from(String version) { + if (version.endsWith("-SNAPSHOT")) { + String[] parts = version.split("-"); + if (parts.length != 2) throw new IllegalArgumentException("snapshot version must only specify major, e.g., \"1-SNAPSHOT\""); + return ofSnapshot(parseInt(parts[0])); + } + String[] parts = version.split("\\."); + if (parts.length != 3) throw new IllegalArgumentException("release versions must specify major, minor and micro, separated by '.', e.g., \"1.2.0\""); + return of(parseInt(parts[0]), parseInt(parts[1]), parseInt(parts[2])); + } + + public int major() { return major; } + public int minor() { return minor; } + public int micro() { return micro; } + public boolean isSnapshot() { return snapshot; } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + Version version = (Version) o; + return major == version.major && minor == version.minor && micro == version.micro && snapshot == version.snapshot; + } + + @Override + public int hashCode() { + return Objects.hash(major, minor, micro, snapshot); + } + + /** Snapshots sort last within their major, and sorting is on major, then minor, then micro otherwise. */ + @Override + public int compareTo(Version other) { + return comparator.compare(this, other); + } + + @Override + public String toString() { + return isSnapshot() ? major + "-SNAPSHOT" : major + "." + minor + "." + micro; + } + +} diff --git a/vespa-application-maven-plugin/src/test/java/com/yahoo/container/plugin/mojo/VersionTest.java b/vespa-application-maven-plugin/src/test/java/com/yahoo/container/plugin/mojo/VersionTest.java new file mode 100644 index 00000000000..ed31f43d081 --- /dev/null +++ b/vespa-application-maven-plugin/src/test/java/com/yahoo/container/plugin/mojo/VersionTest.java @@ -0,0 +1,31 @@ +package com.yahoo.container.plugin.mojo; + +import org.junit.Test; + +import java.util.List; + +import static org.junit.Assert.assertEquals; + +/** + * @author jonmv + */ +public class VersionTest { + + @Test + public void test() { + assertEquals("1.2.0", Version.from("1.2.0").toString()); + assertEquals("3-SNAPSHOT", Version.from("3-SNAPSHOT").toString()); + + List<Version> versions = List.of(Version.of(1, 2, 3), + Version.of(1, 2, 4), + Version.of(1, 3, 2), + Version.ofSnapshot(1), + Version.of(2, 1, 0)); + + for (int i = 0; i < versions.size(); i++) + for (int j = 0; j < versions.size(); j++) + assertEquals(versions.get(i) + " should be less than " + versions.get(j), + Integer.compare(i, j), versions.get(i).compareTo(versions.get(j))); + } + +} diff --git a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/api/AthenzDomain.java b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/api/AthenzDomain.java index 769e81d7732..155ba8ab66a 100644 --- a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/api/AthenzDomain.java +++ b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/api/AthenzDomain.java @@ -12,7 +12,7 @@ import java.util.regex.Pattern; */ public class AthenzDomain { - private static final Pattern NAME_PATTERN = Pattern.compile("[a-zA-Z0-9_][a-zA-Z0-9_\\-.]*[a-zA-Z0-9_]"); + private static final Pattern NAME_PATTERN = Pattern.compile("[a-zA-Z0-9_][a-zA-Z0-9_.-]*[a-zA-Z0-9_]"); private final String name; diff --git a/vespa-maven-plugin/src/main/java/ai/vespa/hosted/plugin/SubmitMojo.java b/vespa-maven-plugin/src/main/java/ai/vespa/hosted/plugin/SubmitMojo.java index d2bad008003..f6ef17bc1b8 100644 --- a/vespa-maven-plugin/src/main/java/ai/vespa/hosted/plugin/SubmitMojo.java +++ b/vespa-maven-plugin/src/main/java/ai/vespa/hosted/plugin/SubmitMojo.java @@ -6,8 +6,6 @@ import org.apache.maven.plugins.annotations.Mojo; import org.apache.maven.plugins.annotations.Parameter; import java.nio.file.Paths; -import java.util.Optional; -import java.util.OptionalLong; /** * Submits a Vespa application package and corresponding test jars to the hosted Vespa API. @@ -41,6 +39,12 @@ public class SubmitMojo extends AbstractVespaMojo { @Parameter(property = "projectId") private String projectId; + @Parameter(property = "risk") + private String risk; + + @Parameter(property = "description") + private String description; + @Override public void doExecute() { applicationZip = firstNonBlank(applicationZip, projectPathOf("target", "application.zip")).orElseThrow(); @@ -48,7 +52,8 @@ public class SubmitMojo extends AbstractVespaMojo { Submission submission = new Submission(optionalOf(repository), optionalOf(branch), optionalOf(commit), optionalOf(sourceUrl), optionalOf(authorEmail), Paths.get(applicationZip), Paths.get(applicationTestZip), - optionalOf(projectId, Long::parseLong)); + optionalOf(projectId, Long::parseLong), optionalOf(risk, Integer::parseInt), + optionalOf(description)); getLog().info(controller.submit(submission, id.tenant(), id.application())); } diff --git a/vespabase/src/rhel-prestart.sh b/vespabase/src/rhel-prestart.sh index dbc77879efe..62e416c2cc2 100755 --- a/vespabase/src/rhel-prestart.sh +++ b/vespabase/src/rhel-prestart.sh @@ -89,29 +89,21 @@ fixdir () { echo "fixdir: Expected 4 params, got:" "$@" exit 1 fi - mkdir -p "$4" - if [ "${VESPA_UNPRIVILEGED}" != yes ]; then - chown $1 "$4" - chgrp $2 "$4" - fi - chmod $3 "$4" + + parent=`dirname "$4"` + [ -d "$parent" ] || fixdir "$1" "$2" "$3" "$parent" + [ -d "$4" ] || mkdir --mode "$3" "$4" + [ "${VESPA_UNPRIVILEGED}" == yes ] || chown $1:$2 "$4" } # BEGIN directory fixups fixdir ${VESPA_USER} ${VESPA_GROUP} 755 libexec/vespa/plugins/qrs -fixdir ${VESPA_USER} ${VESPA_GROUP} 755 logs -fixdir ${VESPA_USER} ${VESPA_GROUP} 755 logs/vespa fixdir ${VESPA_USER} ${VESPA_GROUP} 755 logs/vespa/configserver fixdir ${VESPA_USER} ${VESPA_GROUP} 755 logs/vespa/qrs fixdir ${VESPA_USER} ${VESPA_GROUP} 755 logs/vespa/search -fixdir ${VESPA_USER} ${VESPA_GROUP} 755 tmp fixdir ${VESPA_USER} ${VESPA_GROUP} 755 tmp/vespa -fixdir ${VESPA_USER} ${VESPA_GROUP} 755 var fixdir ${VESPA_USER} ${VESPA_GROUP} 755 var/crash -fixdir ${VESPA_USER} ${VESPA_GROUP} 755 var/db/vespa -fixdir ${VESPA_USER} ${VESPA_GROUP} 755 var/db/vespa/config_server -fixdir ${VESPA_USER} ${VESPA_GROUP} 755 var/db/vespa/config_server/serverdb fixdir ${VESPA_USER} ${VESPA_GROUP} 755 var/db/vespa/config_server/serverdb/tenants fixdir ${VESPA_USER} ${VESPA_GROUP} 755 var/db/vespa/filedistribution fixdir ${VESPA_USER} ${VESPA_GROUP} 755 var/db/vespa/index @@ -120,11 +112,9 @@ fixdir ${VESPA_USER} ${VESPA_GROUP} 755 var/db/vespa/search fixdir ${VESPA_USER} ${VESPA_GROUP} 755 var/db/vespa/tmp fixdir ${VESPA_USER} ${VESPA_GROUP} 755 var/jdisc_container fixdir ${VESPA_USER} ${VESPA_GROUP} 755 var/run -fixdir ${VESPA_USER} ${VESPA_GROUP} 755 var/vespa fixdir ${VESPA_USER} ${VESPA_GROUP} 755 var/vespa/application -fixdir ${VESPA_USER} ${VESPA_GROUP} 755 var/vespa/bundlecache fixdir ${VESPA_USER} ${VESPA_GROUP} 755 var/vespa/bundlecache/configserver -fixdir ${VESPA_USER} ${VESPA_GROUP} 755 var/vespa/cache/config/ +fixdir ${VESPA_USER} ${VESPA_GROUP} 755 var/vespa/cache/config if [ "${VESPA_UNPRIVILEGED}" != yes ]; then chown -hR ${VESPA_USER} logs/vespa diff --git a/vespaclient-container-plugin/src/main/java/com/yahoo/document/restapi/resource/DocumentV1ApiHandler.java b/vespaclient-container-plugin/src/main/java/com/yahoo/document/restapi/resource/DocumentV1ApiHandler.java index 9ec25f8b6a6..51707a05401 100644 --- a/vespaclient-container-plugin/src/main/java/com/yahoo/document/restapi/resource/DocumentV1ApiHandler.java +++ b/vespaclient-container-plugin/src/main/java/com/yahoo/document/restapi/resource/DocumentV1ApiHandler.java @@ -919,7 +919,7 @@ public class DocumentV1ApiHandler extends AbstractRequestHandler { return false; if (result.type() == Result.ResultType.FATAL_ERROR) - throw new DispatchException(result.getError()); + throw new DispatchException(new Throwable(result.error().toString())); outstanding.incrementAndGet(); return true; @@ -1202,7 +1202,7 @@ public class DocumentV1ApiHandler extends AbstractRequestHandler { return false; if (result.type() == Result.ResultType.FATAL_ERROR) - onError.accept(result.getError().getMessage()); + onError.accept(result.error().getMessage()); else outstanding.incrementAndGet(); diff --git a/vespaclient-container-plugin/src/test/java/com/yahoo/document/restapi/resource/DocumentV1ApiTest.java b/vespaclient-container-plugin/src/test/java/com/yahoo/document/restapi/resource/DocumentV1ApiTest.java index 232d49129d7..eaed95b97e1 100644 --- a/vespaclient-container-plugin/src/test/java/com/yahoo/document/restapi/resource/DocumentV1ApiTest.java +++ b/vespaclient-container-plugin/src/test/java/com/yahoo/document/restapi/resource/DocumentV1ApiTest.java @@ -42,7 +42,6 @@ import com.yahoo.documentapi.VisitorParameters; import com.yahoo.documentapi.VisitorResponse; import com.yahoo.documentapi.VisitorSession; import com.yahoo.documentapi.messagebus.protocol.PutDocumentMessage; -import com.yahoo.jdisc.Metric; import com.yahoo.jdisc.test.MockMetric; import com.yahoo.messagebus.StaticThrottlePolicy; import com.yahoo.messagebus.Trace; @@ -364,7 +363,7 @@ public class DocumentV1ApiTest { assertEquals(expectedUpdate, update); parameters.responseHandler().get().handleResponse(new UpdateResponse(0, false)); assertEquals(parameters().withRoute("content"), parameters); - return new Result(Result.ResultType.SUCCESS, null); + return new Result(); }); response = driver.sendRequest("http://localhost/document/v1/space/music/docid?selection=true&cluster=content&timeChunk=10", PUT, "{" + @@ -415,7 +414,7 @@ public class DocumentV1ApiTest { assertEquals(expectedRemove, remove); assertEquals(parameters().withRoute("content"), parameters); parameters.responseHandler().get().handleResponse(new DocumentIdResponse(0, doc2.getId(), "boom", Response.Outcome.ERROR)); - return new Result(Result.ResultType.SUCCESS, null); + return new Result(); }); response = driver.sendRequest("http://localhost/document/v1/space/music/docid?selection=false&cluster=content", DELETE); assertSameJson("{" + @@ -476,7 +475,7 @@ public class DocumentV1ApiTest { assertEquals(doc1.getId(), id); assertEquals(parameters().withRoute("content").withFieldSet("go"), parameters); parameters.responseHandler().get().handleResponse(new DocumentResponse(0, null)); - return new Result(Result.ResultType.SUCCESS, null); + return new Result(); }); response = driver.sendRequest("http://localhost/document/v1/space/music/docid/one?cluster=content&fieldSet=go&timeout=123"); assertSameJson("{" + @@ -490,7 +489,7 @@ public class DocumentV1ApiTest { assertEquals(doc1.getId(), id); assertEquals(parameters().withFieldSet("music:[document]"), parameters); parameters.responseHandler().get().handleResponse(new DocumentResponse(0, doc1)); - return new Result(Result.ResultType.SUCCESS, null); + return new Result(); }); response = driver.sendRequest("http://localhost/document/v1/space/music/docid/one?"); assertSameJson("{" + @@ -507,7 +506,7 @@ public class DocumentV1ApiTest { assertEquals(new DocumentId("id:space:music::one/two/three"), id); assertEquals(parameters().withFieldSet("music:[document]"), parameters); parameters.responseHandler().get().handleResponse(new DocumentResponse(0)); - return new Result(Result.ResultType.SUCCESS, null); + return new Result(); }); response = driver.sendRequest("http://localhost/document/v1/space/music/docid/one/two/three"); assertSameJson("{" + @@ -528,7 +527,7 @@ public class DocumentV1ApiTest { .addChild("Fast Car") .addChild("Baby Can I Hold You")); parameters.responseHandler().get().handleResponse(new DocumentResponse(0, doc2, trace)); - return new Result(Result.ResultType.SUCCESS, null); + return new Result(); }); response = driver.sendRequest("http://localhost/document/v1/space/music/number/1/two?condition=test%20it&tracelevel=9", POST, "{" + @@ -565,7 +564,7 @@ public class DocumentV1ApiTest { assertEquals(expectedUpdate, update); assertEquals(parameters(), parameters); parameters.responseHandler().get().handleResponse(new UpdateResponse(0, true)); - return new Result(Result.ResultType.SUCCESS, null); + return new Result(); }); response = driver.sendRequest("http://localhost/document/v1/space/music/group/a/three?create=true&timeout=1e1s", PUT, "{" + @@ -617,7 +616,7 @@ public class DocumentV1ApiTest { // PUT on document which is not found is a 200 access.session.expect((update, parameters) -> { parameters.responseHandler().get().handleResponse(new UpdateResponse(0, false)); - return new Result(Result.ResultType.SUCCESS, null); + return new Result(); }); response = driver.sendRequest("http://localhost/document/v1/space/music/docid/sonny", PUT, "{" + @@ -638,7 +637,7 @@ public class DocumentV1ApiTest { assertEquals(expectedRemove, remove); assertEquals(parameters().withRoute("route"), parameters); parameters.responseHandler().get().handleResponse(new DocumentIdResponse(0, doc2.getId())); - return new Result(Result.ResultType.SUCCESS, null); + return new Result(); }); response = driver.sendRequest("http://localhost/document/v1/space/music/number/1/two?route=route&condition=false", DELETE); assertSameJson("{" + @@ -669,7 +668,7 @@ public class DocumentV1ApiTest { access.session.expect((id, parameters) -> { assertEquals(clock.instant().plusSeconds(1000), parameters.deadline().get()); parameters.responseHandler().get().handleResponse(new Response(0, "timeout", Response.Outcome.TIMEOUT)); - return new Result(Result.ResultType.SUCCESS, null); + return new Result(); }); response = driver.sendRequest("http://localhost/document/v1/space/music/number/1/two?timeout=1ks"); assertSameJson("{" + @@ -682,7 +681,7 @@ public class DocumentV1ApiTest { // INSUFFICIENT_STORAGE is a 507 access.session.expect((id, parameters) -> { parameters.responseHandler().get().handleResponse(new Response(0, "disk full", Response.Outcome.INSUFFICIENT_STORAGE)); - return new Result(Result.ResultType.SUCCESS, null); + return new Result(); }); response = driver.sendRequest("http://localhost/document/v1/space/music/number/1/two", DELETE); assertSameJson("{" + @@ -695,7 +694,7 @@ public class DocumentV1ApiTest { // PRECONDITION_FAILED is a 412 access.session.expect((id, parameters) -> { parameters.responseHandler().get().handleResponse(new Response(0, "no dice", Response.Outcome.CONDITION_FAILED)); - return new Result(Result.ResultType.SUCCESS, null); + return new Result(); }); response = driver.sendRequest("http://localhost/document/v1/space/music/number/1/two", DELETE); assertSameJson("{" + @@ -722,7 +721,7 @@ public class DocumentV1ApiTest { assertEquals(405, response.getStatus()); // OVERLOAD is a 429 - access.session.expect((id, parameters) -> new Result(Result.ResultType.TRANSIENT_ERROR, new Error("overload"))); + access.session.expect((id, parameters) -> new Result(Result.ResultType.TRANSIENT_ERROR, Result.toError(Result.ResultType.TRANSIENT_ERROR))); var response1 = driver.sendRequest("http://localhost/document/v1/space/music/number/1/two", POST, "{\"fields\": {}}"); var response2 = driver.sendRequest("http://localhost/document/v1/space/music/number/1/two", POST, "{\"fields\": {}}"); var response3 = driver.sendRequest("http://localhost/document/v1/space/music/number/1/two", POST, "{\"fields\": {}}"); @@ -731,16 +730,16 @@ public class DocumentV1ApiTest { " \"message\": \"Rejecting execution due to overload: 2 requests already enqueued\"" + "}", response3.readAll()); assertEquals(429, response3.getStatus()); - access.session.expect((id, parameters) -> new Result(Result.ResultType.FATAL_ERROR, new Error("error"))); + access.session.expect((id, parameters) -> new Result(Result.ResultType.FATAL_ERROR, Result.toError(Result.ResultType.FATAL_ERROR))); handler.dispatchEnqueued(); assertSameJson("{" + " \"pathId\": \"/document/v1/space/music/number/1/two\"," + - " \"message\": \"error\"" + + " \"message\": \"[FATAL_ERROR @ localhost]: FATAL_ERROR\"" + "}", response1.readAll()); assertEquals(502, response1.getStatus()); assertSameJson("{" + " \"pathId\": \"/document/v1/space/music/number/1/two\"," + - " \"message\": \"error\"" + + " \"message\": \"[FATAL_ERROR @ localhost]: FATAL_ERROR\"" + "}", response2.readAll()); assertEquals(502, response2.getStatus()); @@ -748,7 +747,7 @@ public class DocumentV1ApiTest { AtomicReference<ResponseHandler> handler = new AtomicReference<>(); access.session.expect((id, parameters) -> { handler.set(parameters.responseHandler().get()); - return new Result(Result.ResultType.SUCCESS, null); + return new Result(); }); try { var response4 = driver.sendRequest("http://localhost/document/v1/space/music/docid/one?timeout=1ms"); @@ -808,7 +807,7 @@ public class DocumentV1ApiTest { CountDownLatch setup = new CountDownLatch(queueFill); access.session.expect((id, parameters) -> { setup.countDown(); - return new Result(Result.ResultType.TRANSIENT_ERROR, new Error()); + return new Result(Result.ResultType.TRANSIENT_ERROR, Result.toError(Result.ResultType.TRANSIENT_ERROR)); }); for (int i = 0; i < queueFill; i++) { int j = i; diff --git a/vespajlib/src/test/java/ai/vespa/http/HttpURLTest.java b/vespajlib/src/test/java/ai/vespa/http/HttpURLTest.java index 8e3029009b0..ee0b98c00ed 100644 --- a/vespajlib/src/test/java/ai/vespa/http/HttpURLTest.java +++ b/vespajlib/src/test/java/ai/vespa/http/HttpURLTest.java @@ -161,6 +161,10 @@ class HttpURLTest { assertEquals("path segments cannot be \"\", \".\", or \"..\", but got: '..'", assertThrows(IllegalArgumentException.class, () -> Path.empty().append("%2E%25252E")).getMessage()); + + assertEquals("path segments cannot be \"\", \".\", or \"..\", but got: ''", + assertThrows(IllegalArgumentException.class, + () -> Path.parse("//")).getMessage()); } @Test |
