diff options
163 files changed, 1238 insertions, 921 deletions
diff --git a/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java b/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java index c01bf3c234b..4536257f8a3 100644 --- a/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java +++ b/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java @@ -80,10 +80,10 @@ public interface ModelContext { @ModelFeatureFlag(owners = {"baldersheim"}, removeAfter="7.last") default boolean skipCommunicationManagerThread() { return true; } @ModelFeatureFlag(owners = {"baldersheim"}, removeAfter="7.last") default boolean skipMbusRequestThread() { return true; } @ModelFeatureFlag(owners = {"baldersheim"}, removeAfter="7.last") default boolean skipMbusReplyThread() { return true; } - @ModelFeatureFlag(owners = {"baldersheim"}) default boolean mbusDispatchOnDecode() { return true; } - @ModelFeatureFlag(owners = {"baldersheim"}) default boolean mbusDispatchOnEncode() { return true; } + @ModelFeatureFlag(owners = {"baldersheim"}, removeAfter="8.15") default boolean mbusDispatchOnDecode() { return true; } + @ModelFeatureFlag(owners = {"baldersheim"}, removeAfter="8.15") default boolean mbusDispatchOnEncode() { return true; } @ModelFeatureFlag(owners = {"baldersheim"}) default int mbusNetworkThreads() { return 1; } - @ModelFeatureFlag(owners = {"baldersheim"}) default int mbusThreads() { return 4; } + @ModelFeatureFlag(owners = {"baldersheim"}, removeAfter="8.15") default int mbusThreads() { return 4; } @ModelFeatureFlag(owners = {"baldersheim"}) default int mbusJavaRpcNumTargets() { return 1; } @ModelFeatureFlag(owners = {"baldersheim"}) default int mbusJavaEventsBeforeWakeup() { return 1; } @ModelFeatureFlag(owners = {"baldersheim"}) default int mbusCppRpcNumTargets() { return 1; } @@ -114,7 +114,7 @@ public interface ModelContext { @ModelFeatureFlag(owners = {"arnej"}) default boolean useV8GeoPositions() { return false; } @ModelFeatureFlag(owners = {"baldersheim", "geirst", "toregge"}) default int maxCompactBuffers() { return 1; } @ModelFeatureFlag(owners = {"arnej", "andreer"}) default List<String> ignoredHttpUserAgents() { return List.of(); } - @ModelFeatureFlag(owners = {"bjorncs"}) default boolean enableServerOcspStapling() { return false; } + @ModelFeatureFlag(owners = {"bjorncs"}, removeAfter="7.last") default boolean enableServerOcspStapling() { return true; } @ModelFeatureFlag(owners = {"vekterli"}) default String mergeThrottlingPolicy() { throw new UnsupportedOperationException("TODO specify default value"); } @ModelFeatureFlag(owners = {"vekterli"}) default double persistenceThrottlingWsDecrementFactor() { throw new UnsupportedOperationException("TODO specify default value"); } @ModelFeatureFlag(owners = {"vekterli"}) default double persistenceThrottlingWsBackoff() { throw new UnsupportedOperationException("TODO specify default value"); } diff --git a/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java b/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java index 5d64dfe7041..400aea86834 100644 --- a/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java +++ b/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java @@ -78,15 +78,12 @@ public class TestProperties implements ModelContext.Properties, ModelContext.Fea private boolean enableBitVectors = false; private boolean loadCodeAsHugePages = false; private boolean sharedStringRepoNoReclaim = false; - private boolean mbus_dispatch_on_decode = true; - private boolean mbus_dispatch_on_encode = true; private int mbus_java_num_targets = 1; private int mbus_java_events_before_wakeup = 1; private int mbus_cpp_num_targets = 1; private int mbus_cpp_events_before_wakeup = 1; private int rpc_num_targets = 1; private int rpc_events_before_wakeup = 1; - private int mbus_threads = 4; private int mbus_network_threads = 1; private Architecture adminClusterNodeResourcesArchitecture = Architecture.getDefault(); @@ -143,10 +140,7 @@ public class TestProperties implements ModelContext.Properties, ModelContext.Fea @Override public Architecture adminClusterArchitecture() { return adminClusterNodeResourcesArchitecture; } @Override public boolean sharedStringRepoNoReclaim() { return sharedStringRepoNoReclaim; } @Override public boolean loadCodeAsHugePages() { return loadCodeAsHugePages; } - @Override public boolean mbusDispatchOnDecode() { return mbus_dispatch_on_decode; } - @Override public boolean mbusDispatchOnEncode() { return mbus_dispatch_on_encode; } @Override public int mbusNetworkThreads() { return mbus_network_threads; } - @Override public int mbusThreads() { return mbus_threads; } @Override public int mbusJavaRpcNumTargets() { return mbus_java_num_targets; } @Override public int mbusJavaEventsBeforeWakeup() { return mbus_java_events_before_wakeup; } @Override public int mbusCppRpcNumTargets() { return mbus_cpp_num_targets; } @@ -379,20 +373,6 @@ public class TestProperties implements ModelContext.Properties, ModelContext.Fea return this; } - public TestProperties setMbusDispatchOnDecode(boolean value) { - this.mbus_dispatch_on_decode = value; - return this; - } - public TestProperties setMbusDispatchOnEncode(boolean value) { - this.mbus_dispatch_on_encode = value; - return this; - } - - public TestProperties setMbusThreads(int value) { - this.mbus_threads = value; - return this; - } - public TestProperties setMbusNetworkThreads(int value) { this.mbus_network_threads = value; return this; diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/Admin.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/Admin.java index 79f65264249..1928fb0f9ad 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/admin/Admin.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/Admin.java @@ -79,19 +79,19 @@ public class Admin extends AbstractConfigProducer<Admin> implements Serializable private ZooKeepersConfigProvider zooKeepersConfigProvider; private final FileDistributionConfigProducer fileDistribution; - private final boolean multitenant; + private final boolean multiTenant; public Admin(AbstractConfigProducer<?> parent, Monitoring monitoring, Metrics metrics, - boolean multitenant, + boolean multiTenant, boolean isHostedVespa, ApplicationType applicationType) { super(parent, "admin"); this.isHostedVespa = isHostedVespa; this.monitoring = monitoring; this.metrics = metrics; - this.multitenant = multitenant; + this.multiTenant = multiTenant; this.fileDistribution = new FileDistributionConfigProducer(parent); this.applicationType = applicationType; } @@ -149,7 +149,7 @@ public class Admin extends AbstractConfigProducer<Admin> implements Serializable public void setClusterControllers(ClusterControllerContainerCluster clusterControllers, DeployState deployState) { this.clusterControllers = clusterControllers; if (isHostedVespa) { - // Prefer to put Slobroks on the admin cluster running cluster controllers to avoid unnecessary + // Prefer to put slobroks on the admin cluster running cluster controllers to avoid unnecessary // movement of the slobroks when there are changes to the content cluster nodes removeSlobroks(); addSlobroks(createSlobroksOn(clusterControllers, deployState)); @@ -250,9 +250,9 @@ public class Admin extends AbstractConfigProducer<Admin> implements Serializable boolean actuallyAdd = true; var membership = host.spec().membership(); if (membership.isPresent()) { - var clustertype = membership.get().cluster().type(); + var clusterType = membership.get().cluster().type(); // XXX should skip only if this.isHostedVespa is true? - if (clustertype == ClusterSpec.Type.admin) { + if (clusterType == ClusterSpec.Type.admin) { actuallyAdd = logForwarderIncludeAdmin; } } @@ -263,8 +263,7 @@ public class Admin extends AbstractConfigProducer<Admin> implements Serializable } private void addConfigSentinel(DeployState deployState, HostResource host, - ApplicationId applicationId, Zone zone, ModelContext.FeatureFlags featureFlags) - { + ApplicationId applicationId, Zone zone, ModelContext.FeatureFlags featureFlags) { ConfigSentinel configSentinel = new ConfigSentinel(host.getHost(), applicationId, zone, featureFlags); addAndInitializeService(deployState, host, configSentinel); host.getHost().setConfigSentinel(configSentinel); @@ -320,9 +319,7 @@ public class Admin extends AbstractConfigProducer<Admin> implements Serializable return slobs; } - public boolean multitenant() { - return multitenant; - } + public boolean multiTenant() { return multiTenant; } public ApplicationType getApplicationType() { return applicationType; } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomAdminBuilderBase.java b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomAdminBuilderBase.java index 4c74282c061..c3bfeca91ae 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomAdminBuilderBase.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomAdminBuilderBase.java @@ -1,7 +1,6 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.model.builder.xml.dom; -import com.yahoo.config.application.api.DeployLogger; import com.yahoo.config.model.ConfigModelContext.ApplicationType; import com.yahoo.config.model.api.ConfigServerSpec; import com.yahoo.config.model.deploy.DeployState; @@ -20,7 +19,6 @@ import com.yahoo.vespa.model.admin.monitoring.builder.Metrics; import com.yahoo.vespa.model.admin.monitoring.builder.PredefinedMetricSets; import com.yahoo.vespa.model.admin.monitoring.builder.xml.MetricsBuilder; import org.w3c.dom.Element; - import java.util.ArrayList; import java.util.List; import java.util.Optional; @@ -35,13 +33,11 @@ public abstract class DomAdminBuilderBase extends VespaDomBuilder.DomConfigProdu private final ApplicationType applicationType; protected final List<ConfigServerSpec> configServerSpecs; - protected final boolean multitenant; + protected final boolean multiTenant; - DomAdminBuilderBase(ApplicationType applicationType, - boolean multitenant, - List<ConfigServerSpec> configServerSpecs) { + DomAdminBuilderBase(ApplicationType applicationType, boolean multiTenant, List<ConfigServerSpec> configServerSpecs) { this.applicationType = applicationType; - this.multitenant = multitenant; + this.multiTenant = multiTenant; this.configServerSpecs = configServerSpecs; } @@ -65,7 +61,7 @@ public abstract class DomAdminBuilderBase extends VespaDomBuilder.DomConfigProdu Monitoring monitoring = getMonitoring(XML.getChild(adminElement,"monitoring"), deployState.isHosted()); Metrics metrics = new MetricsBuilder(applicationType, PredefinedMetricSets.get()) .buildMetrics(XML.getChild(adminElement, "metrics")); - Admin admin = new Admin(parent, monitoring, metrics, multitenant, deployState.isHosted(), applicationType); + Admin admin = new Admin(parent, monitoring, metrics, multiTenant, deployState.isHosted(), applicationType); doBuildAdmin(deployState, admin, adminElement); new ModelConfigProvider(admin); diff --git a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomAdminV2Builder.java b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomAdminV2Builder.java index e081848d01c..68e34f7bf06 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomAdminV2Builder.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomAdminV2Builder.java @@ -1,7 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.model.builder.xml.dom; -import com.yahoo.config.model.ConfigModelContext; +import com.yahoo.config.model.ConfigModelContext.ApplicationType; import com.yahoo.config.model.api.ConfigServerSpec; import com.yahoo.config.model.deploy.DeployState; import com.yahoo.config.model.producer.AbstractConfigProducer; @@ -17,7 +17,6 @@ import com.yahoo.vespa.model.admin.clustercontroller.ClusterControllerContainerC import com.yahoo.vespa.model.builder.xml.dom.VespaDomBuilder.DomConfigProducerBuilder; import com.yahoo.vespa.model.container.Container; import org.w3c.dom.Element; - import java.util.ArrayList; import java.util.List; import java.util.Objects; @@ -32,45 +31,39 @@ public class DomAdminV2Builder extends DomAdminBuilderBase { private static final String ATTRIBUTE_CLUSTER_CONTROLLER_STANDALONE_ZK = "standalone-zookeeper"; - public DomAdminV2Builder(ConfigModelContext.ApplicationType applicationType, - boolean multitenant, - List<ConfigServerSpec> configServerSpecs) { - super(applicationType, multitenant, configServerSpecs); + public DomAdminV2Builder(ApplicationType applicationType, boolean multiTenant, List<ConfigServerSpec> configServerSpecs) { + super(applicationType, multiTenant, configServerSpecs); } @Override protected void doBuildAdmin(DeployState deployState, Admin admin, Element adminE) { - List<Configserver> configservers = parseConfigservers(deployState, admin, adminE); admin.setLogserver(parseLogserver(deployState, admin, adminE)); - admin.addConfigservers(configservers); + admin.addConfigservers(parseConfigServers(deployState, admin, adminE)); admin.addSlobroks(getSlobroks(deployState, admin, XML.getChild(adminE, "slobroks"))); - if ( ! admin.multitenant()) + if ( ! admin.multiTenant()) admin.setClusterControllers(addConfiguredClusterControllers(deployState, admin, adminE), deployState); - addLogForwarders(new ModelElement(adminE).child("logforwarding"), admin); } - private List<Configserver> parseConfigservers(DeployState deployState, Admin admin, Element adminE) { - List<Configserver> configservers; - if (multitenant) - configservers = getConfigServersFromSpec(deployState, admin); - else - configservers = getConfigServers(deployState, admin, adminE); - if (configservers.isEmpty() && ! multitenant) - configservers = createSingleConfigServer(deployState, admin); - if (configservers.size() % 2 == 0) + private List<Configserver> parseConfigServers(DeployState deployState, Admin admin, Element adminE) { + List<Configserver> configServers = multiTenant + ? getConfigServersFromSpec(deployState, admin) + : getConfigServers(deployState, admin, adminE); + + if (configServers.isEmpty() && !multiTenant) + configServers = createSingleConfigServer(deployState, admin); + if (configServers.size() % 2 == 0) deployState.getDeployLogger().logApplicationPackage(Level.WARNING, - "An even number (" + configservers.size() + + "An even number (" + configServers.size() + ") of config servers have been configured. " + "This is discouraged, see doc for configuration server "); - return configservers; + return configServers; } private Logserver parseLogserver(DeployState deployState, Admin admin, Element adminE) { Element logserverE = XML.getChild(adminE, "logserver"); - if (logserverE == null) { + if (logserverE == null) logserverE = XML.getChild(adminE, "adminserver"); - } return new LogserverBuilder().build(deployState, admin, logserverE); } @@ -83,10 +76,9 @@ public class DomAdminV2Builder extends DomAdminBuilderBase { List<Element> controllers = XML.getChildren(controllersElements, "cluster-controller"); if (controllers.isEmpty()) return null; - boolean standaloneZooKeeper = "true".equals(controllersElements.getAttribute(ATTRIBUTE_CLUSTER_CONTROLLER_STANDALONE_ZK)) || multitenant; - if (standaloneZooKeeper) { + boolean standaloneZooKeeper = "true".equals(controllersElements.getAttribute(ATTRIBUTE_CLUSTER_CONTROLLER_STANDALONE_ZK)) || multiTenant; + if (standaloneZooKeeper) parent = new ClusterControllerCluster(parent, "standalone", deployState); - } var cluster = new ClusterControllerContainerCluster(parent, "cluster-controllers", "cluster-controllers", diff --git a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomAdminV4Builder.java b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomAdminV4Builder.java index dae2f7e8cb8..b6ee08cb5ec 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomAdminV4Builder.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomAdminV4Builder.java @@ -33,9 +33,9 @@ public class DomAdminV4Builder extends DomAdminBuilderBase { private final Collection<ContainerModel> containerModels; private final ConfigModelContext context; - public DomAdminV4Builder(ConfigModelContext context, boolean multitenant, List<ConfigServerSpec> configServerSpecs, + public DomAdminV4Builder(ConfigModelContext context, boolean multiTenant, List<ConfigServerSpec> configServerSpecs, Collection<ContainerModel> containerModels) { - super(context.getApplicationType(), multitenant, configServerSpecs); + super(context.getApplicationType(), multiTenant, configServerSpecs); this.containerModels = containerModels; this.context = context; } @@ -131,7 +131,7 @@ public class DomAdminV4Builder extends DomAdminBuilderBase { * The list returns the same nodes on each invocation given the same available nodes. * * @param count the desired number of nodes. More nodes may be returned to ensure a smooth transition - * on topology changes, and less nodes may be returned if fewer are available + * on topology changes, and fewer nodes may be returned if fewer are available * @param minHostsPerContainerCluster the desired number of hosts per cluster */ private List<HostResource> pickContainerHostsForSlobrok(int count, int minHostsPerContainerCluster) { diff --git a/config-model/src/main/java/com/yahoo/vespa/model/clients/ContainerDocumentApi.java b/config-model/src/main/java/com/yahoo/vespa/model/clients/ContainerDocumentApi.java index 15a0e060e1c..33c125dcecf 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/clients/ContainerDocumentApi.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/clients/ContainerDocumentApi.java @@ -7,10 +7,12 @@ import com.yahoo.container.handler.threadpool.ContainerThreadpoolConfig; import com.yahoo.osgi.provider.model.ComponentModel; import com.yahoo.vespa.model.container.ContainerCluster; import com.yahoo.vespa.model.container.ContainerThreadpool; +import com.yahoo.vespa.model.container.PlatformBundles; import com.yahoo.vespa.model.container.component.Handler; import com.yahoo.vespa.model.container.component.SystemBindingPattern; import com.yahoo.vespa.model.container.component.UserBindingPattern; +import java.nio.file.Path; import java.util.Collection; import java.util.Collections; @@ -21,6 +23,8 @@ import java.util.Collections; public class ContainerDocumentApi { public static final String DOCUMENT_V1_PREFIX = "/document/v1"; + public static final Path VESPACLIENT_CONTAINER_BUNDLE = + PlatformBundles.absoluteBundlePath("vespaclient-container-plugin"); private final boolean ignoreUndefinedFields; @@ -28,6 +32,11 @@ public class ContainerDocumentApi { this.ignoreUndefinedFields = ignoreUndefinedFields; addRestApiHandler(cluster, handlerOptions); addFeedHandler(cluster, handlerOptions); + addVespaClientContainerBundle(cluster); + } + + public static void addVespaClientContainerBundle(ContainerCluster<?> c) { + c.addPlatformBundle(VESPACLIENT_CONTAINER_BUNDLE); } private static void addFeedHandler(ContainerCluster<?> cluster, HandlerOptions handlerOptions) { diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/ApplicationContainer.java b/config-model/src/main/java/com/yahoo/vespa/model/container/ApplicationContainer.java index 47dab37cc14..9997b20d205 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/ApplicationContainer.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/ApplicationContainer.java @@ -27,7 +27,6 @@ public final class ApplicationContainer extends Container implements private static final String defaultHostedJVMArgs = "-XX:+SuppressFatalErrorMessage"; private final boolean isHostedVespa; - private final boolean enableServerOcspStapling; public ApplicationContainer(AbstractConfigProducer<?> parent, String name, int index, DeployState deployState) { this(parent, name, false, index, deployState); @@ -36,7 +35,6 @@ public final class ApplicationContainer extends Container implements public ApplicationContainer(AbstractConfigProducer<?> parent, String name, boolean retired, int index, DeployState deployState) { super(parent, name, retired, index, deployState); this.isHostedVespa = deployState.isHosted(); - this.enableServerOcspStapling = deployState.featureFlags().enableServerOcspStapling(); addComponent(new SimpleComponent("com.yahoo.container.jdisc.messagebus.NetworkMultiplexerHolder")); addComponent(new SimpleComponent("com.yahoo.container.jdisc.messagebus.NetworkMultiplexerProvider")); @@ -68,12 +66,10 @@ public final class ApplicationContainer extends Container implements if (hasDocproc()) { b.append(ApplicationContainer.defaultHostedJVMArgs).append(' '); } - if (enableServerOcspStapling) { - b.append("-Djdk.tls.server.enableStatusRequestExtension=true ") - .append("-Djdk.tls.stapling.responseTimeout=2000 ") - .append("-Djdk.tls.stapling.cacheSize=256 ") - .append("-Djdk.tls.stapling.cacheLifetime=3600 "); - } + b.append("-Djdk.tls.server.enableStatusRequestExtension=true ") + .append("-Djdk.tls.stapling.responseTimeout=2000 ") + .append("-Djdk.tls.stapling.cacheSize=256 ") + .append("-Djdk.tls.stapling.cacheLifetime=3600 "); } String jvmArgs = super.getJvmOptions(); if (!jvmArgs.isBlank()) { diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/ApplicationContainerCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/container/ApplicationContainerCluster.java index 9785858b338..937d7cf58d3 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/ApplicationContainerCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/ApplicationContainerCluster.java @@ -78,7 +78,6 @@ public final class ApplicationContainerCluster extends ContainerCluster<Applicat public static final int heapSizePercentageOfTotalNodeMemory = 70; public static final int heapSizePercentageOfTotalNodeMemoryWhenCombinedCluster = 18; - private final Set<FileReference> applicationBundles = new LinkedHashSet<>(); private final Set<String> previousHosts; @@ -116,18 +115,10 @@ public final class ApplicationContainerCluster extends ContainerCluster<Applicat addMetricsHandlers(); addTestrunnerComponentsIfTester(deployState); - addPlatformBundlesForApplicationCluster(); transport_connections_per_target = deployState.featureFlags().mbusJavaRpcNumTargets(); transport_events_before_wakeup = deployState.featureFlags().mbusJavaEventsBeforeWakeup(); } - private void addPlatformBundlesForApplicationCluster() { - Set<String> bundles = Set.of( - "container-search-and-docproc", "container-search-gui", "docprocs", - "linguistics-components", "vespaclient-container-plugin"); - bundles.forEach(b -> addPlatformBundle(PlatformBundles.absoluteBundlePath(b))); - } - @Override protected void doPrepare(DeployState deployState) { addAndSendApplicationBundles(deployState); diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerCluster.java index 6b6192a04ef..2385b5b3812 100755 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerCluster.java @@ -70,6 +70,8 @@ import java.util.Objects; import java.util.Optional; import java.util.Set; import java.util.TreeSet; +import java.util.stream.Collectors; +import java.util.stream.Stream; /** * Parent class for all container cluster types. @@ -126,6 +128,10 @@ public abstract class ContainerCluster<CONTAINER extends Container> public static final BindingPattern VIP_HANDLER_BINDING = SystemBindingPattern.fromHttpPath("/status.html"); + public static final Set<Path> SEARCH_AND_DOCPROC_BUNDLES = Stream.of( + PlatformBundles.searchAndDocprocBundle, "container-search-gui", "docprocs", "linguistics-components") + .map(PlatformBundles::absoluteBundlePath).collect(Collectors.toSet()); + private final String name; protected List<CONTAINER> containers = new ArrayList<>(); @@ -388,6 +394,18 @@ public abstract class ContainerCluster<CONTAINER extends Container> return Collections.unmodifiableCollection(allComponents); } + /* + Add all search/docproc/feed related platform bundles. + This is only required for configured containers as the platform bundle set is not allowed to change between config generations. + For standalone container platform bundles can be added on features enabled as an update of application package requires restart. + */ + public void addAllPlatformBundles() { + ContainerDocumentApi.addVespaClientContainerBundle(this); + addSearchAndDocprocBundles(); + } + + public void addSearchAndDocprocBundles() { SEARCH_AND_DOCPROC_BUNDLES.forEach(this::addPlatformBundle); } + private void recursivelyFindAllComponents(Collection<Component<?, ?>> allComponents, AbstractConfigProducer<?> current) { for (AbstractConfigProducer<?> child: current.getChildren().values()) { if (child instanceof Component) diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/docproc/ContainerDocproc.java b/config-model/src/main/java/com/yahoo/vespa/model/container/docproc/ContainerDocproc.java index f2233680e9b..fee10b965aa 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/docproc/ContainerDocproc.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/docproc/ContainerDocproc.java @@ -27,29 +27,29 @@ public class ContainerDocproc extends ContainerSubsystem<DocprocChains> public final Options options; // Whether or not to prefer sending to a local node. - private boolean preferLocalNode = false; + private final boolean preferLocalNode = false; // The number of nodes to use per client. - private int numNodesPerClient = 0; + private final int numNodesPerClient = 0; - private Map<Pair<String, String>, String> fieldNameSchemaMap = new HashMap<>(); + private final Map<Pair<String, String>, String> fieldNameSchemaMap = new HashMap<>(); - public ContainerDocproc(ContainerCluster cluster, DocprocChains chains) { + public ContainerDocproc(ContainerCluster<?> cluster, DocprocChains chains) { this(cluster, chains, new Options( null, null, null, null, null, null)); } - public ContainerDocproc(ContainerCluster cluster, DocprocChains chains, Options options) { + public ContainerDocproc(ContainerCluster<?> cluster, DocprocChains chains, Options options) { this(cluster, chains, options, true); } private void addSource( - final ContainerCluster cluster, final String name, final SessionConfig.Type.Enum type) { + final ContainerCluster<?> cluster, final String name, final SessionConfig.Type.Enum type) { final MbusClient mbusClient = new MbusClient(name, type); mbusClient.addClientBindings(SystemBindingPattern.fromPattern("mbus://*/" + mbusClient.getSessionName())); cluster.addComponent(mbusClient); } - public ContainerDocproc(ContainerCluster cluster, DocprocChains chains, Options options, boolean addSourceClientProvider) { + public ContainerDocproc(ContainerCluster<?> cluster, DocprocChains chains, Options options, boolean addSourceClientProvider) { super(chains); assert (options != null) : "Null Options for " + this + " under cluster " + cluster.getName(); this.options = options; @@ -58,6 +58,7 @@ public class ContainerDocproc extends ContainerSubsystem<DocprocChains> addSource(cluster, "source", SessionConfig.Type.SOURCE); addSource(cluster, MbusRequestContext.internalNoThrottledSource, SessionConfig.Type.INTERNAL); } + cluster.addSearchAndDocprocBundles(); } public boolean isPreferLocalNode() { diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/search/ContainerSearch.java b/config-model/src/main/java/com/yahoo/vespa/model/container/search/ContainerSearch.java index 58151063956..3ac12381a1f 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/search/ContainerSearch.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/search/ContainerSearch.java @@ -59,6 +59,7 @@ public class ContainerSearch extends ContainerSubsystem<SearchChains> owningCluster.addComponent(Component.fromClassAndBundle(QUERY_PROFILE_REGISTRY_CLASS, searchAndDocprocBundle)); owningCluster.addComponent(Component.fromClassAndBundle(com.yahoo.search.schema.SchemaInfo.class.getName(), searchAndDocprocBundle)); owningCluster.addComponent(Component.fromClassAndBundle(SearchStatusExtension.class.getName(), searchAndDocprocBundle)); + cluster.addSearchAndDocprocBundles(); } public void connectSearchClusters(Map<String, SearchCluster> searchClusters) { diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java index 7cc1109f25f..1c47f1d7c9c 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java @@ -205,6 +205,8 @@ public class ContainerModelBuilder extends ConfigModelBuilder<ContainerModel> { addServerProviders(deployState, spec, cluster); + if (!standaloneBuilder) cluster.addAllPlatformBundles(); + // Must be added after nodes: addDeploymentSpecConfig(cluster, context, deployState.getDeployLogger()); addZooKeeper(cluster, spec); @@ -595,6 +597,7 @@ public class ContainerModelBuilder extends ConfigModelBuilder<ContainerModel> { Element processingElement = XML.getChild(spec, "processing"); if (processingElement == null) return; + cluster.addSearchAndDocprocBundles(); addIncludes(processingElement); cluster.setProcessingChains(new DomProcessingBuilder(null).build(deployState, cluster, processingElement), serverBindings(processingElement, ProcessingChains.defaultBindings).toArray(BindingPattern[]::new)); diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/Content.java b/config-model/src/main/java/com/yahoo/vespa/model/content/Content.java index d4595b3adfd..22923a724a6 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/content/Content.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/content/Content.java @@ -298,6 +298,7 @@ public class Content extends ConfigModel { content.ownedIndexingCluster = Optional.of(indexingCluster); indexingCluster.addDefaultHandlersWithVip(); + indexingCluster.addAllPlatformBundles(); addDocproc(indexingCluster); List<ApplicationContainer> nodes = new ArrayList<>(); diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/ContentNode.java b/config-model/src/main/java/com/yahoo/vespa/model/content/ContentNode.java index 6ff9329ce7e..c52bb6fa2de 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/content/ContentNode.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/content/ContentNode.java @@ -21,9 +21,6 @@ public abstract class ContentNode extends AbstractService private final int distributionKey; private final String rootDirectory; - private final boolean dispatch_on_encode; - private final boolean dispatch_on_decode; - private final int mbus_threads; private final int mbus_network_threads; private final int mbus_rpc_targets; private final int mbus_events_before_wakeup; @@ -34,9 +31,6 @@ public abstract class ContentNode extends AbstractService super(parent, "" + distributionKey); this.distributionKey = distributionKey; this.rootDirectory = rootDirectory; - dispatch_on_decode = featureFlags.mbusDispatchOnDecode(); - dispatch_on_encode = featureFlags.mbusDispatchOnEncode(); - mbus_threads = featureFlags.mbusThreads(); mbus_network_threads = featureFlags.mbusNetworkThreads(); mbus_rpc_targets = featureFlags.mbusCppRpcNumTargets(); mbus_events_before_wakeup = featureFlags.mbusCppEventsBeforeWakeup(); @@ -85,9 +79,6 @@ public abstract class ContentNode extends AbstractService public void getConfig(StorCommunicationmanagerConfig.Builder builder) { builder.mbusport(getRelativePort(0)); builder.rpcport(getRelativePort(1)); - builder.mbus.dispatch_on_decode(dispatch_on_decode); - builder.mbus.dispatch_on_encode(dispatch_on_encode); - builder.mbus.num_threads(mbus_threads); builder.mbus.num_network_threads(mbus_network_threads); builder.mbus.num_rpc_targets(mbus_rpc_targets); builder.mbus.events_before_wakeup(mbus_events_before_wakeup); diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/ContentCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/ContentCluster.java index 776d23e5227..7f262b4aee7 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/ContentCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/ContentCluster.java @@ -287,7 +287,7 @@ public class ContentCluster extends AbstractConfigProducer<AbstractConfigProduce if (context.properties().hostedVespa()) { clusterControllers = getDedicatedSharedControllers(contentElement, admin, context, deployState, clusterName); } - else if (admin.multitenant()) { // system tests: Put on logserver + else if (admin.multiTenant()) { // system tests: Put on logserver if (admin.getClusterControllers() == null) { // TODO: logserver == null only obtains in unit tests, disallow it List<HostResource> host = admin.getLogserver() == null ? List.of() : List.of(admin.getLogserver().getHostResource()); diff --git a/config-model/src/test/java/com/yahoo/vespa/model/builder/xml/dom/DomAdminV2BuilderTest.java b/config-model/src/test/java/com/yahoo/vespa/model/builder/xml/dom/DomAdminV2BuilderTest.java index 1a41d2689a2..093b0765530 100755 --- a/config-model/src/test/java/com/yahoo/vespa/model/builder/xml/dom/DomAdminV2BuilderTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/builder/xml/dom/DomAdminV2BuilderTest.java @@ -2,7 +2,7 @@ package com.yahoo.vespa.model.builder.xml.dom; import com.yahoo.cloud.config.log.LogdConfig; -import com.yahoo.config.model.ConfigModelContext; +import com.yahoo.config.model.ConfigModelContext.ApplicationType; import com.yahoo.config.model.api.ConfigServerSpec; import com.yahoo.config.model.builder.xml.test.DomBuilderTest; import com.yahoo.config.model.deploy.DeployState; @@ -16,13 +16,12 @@ import com.yahoo.vespa.model.admin.monitoring.Monitoring; import org.junit.Before; import org.junit.Test; import org.w3c.dom.Element; -import java.util.ArrayList; import java.util.Arrays; import java.util.List; +import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNotEquals; import static org.junit.Assert.assertNotNull; -import static org.junit.Assert.assertEquals; /** * @author hmusum @@ -36,16 +35,6 @@ public class DomAdminV2BuilderTest extends DomBuilderTest { root = new MockRoot("root"); } - // Supported for backwards compatibility - private Element servicesConfigserver() { - return XML.getDocument( - "<admin version=\"2.0\">" + - " <configserver hostalias=\"mockhost\"/>" + - " <adminserver hostalias=\"mockhost\"/>" + - "</admin>").getDocumentElement(); - - } - private Element servicesOverride() { return XML.getDocument( "<admin version=\"2.0\">" + @@ -131,19 +120,10 @@ public class DomAdminV2BuilderTest extends DomBuilderTest { } /** - * Tests that configserver works (deprecated, but allowed in admin 2.0) - */ - @Test - public void adminWithConfigserverElement() { - Admin admin = buildAdmin(servicesConfigserver()); - assertEquals(1, admin.getConfigservers().size()); - } - - /** * Tests that configservers/configserver works */ @Test - public void adminWithConfigserversElement() { + public void adminWithConfigServersElement() { Admin admin = buildAdmin(servicesConfigservers()); assertEquals(1, admin.getConfigservers().size()); } @@ -202,13 +182,12 @@ public class DomAdminV2BuilderTest extends DomBuilderTest { } private Admin buildAdmin(Element xml) { - return buildAdmin(xml, false, new ArrayList<>()); + return buildAdmin(xml, false, List.of()); } - private Admin buildAdmin(Element xml, boolean multitenant, List<ConfigServerSpec> configServerSpecs) { + private Admin buildAdmin(Element xml, boolean multitenant, List<ConfigServerSpec> configServers) { DeployState deployState = DeployState.createTestState(); - final DomAdminV2Builder domAdminBuilder = - new DomAdminV2Builder(ConfigModelContext.ApplicationType.DEFAULT, multitenant, configServerSpecs); + DomAdminV2Builder domAdminBuilder = new DomAdminV2Builder(ApplicationType.DEFAULT, multitenant, configServers); Admin admin = domAdminBuilder.build(deployState, root, xml); admin.addPerHostServices(root.hostSystem().getHosts(), deployState); return admin; diff --git a/config-model/src/test/java/com/yahoo/vespa/model/container/ContainerClusterTest.java b/config-model/src/test/java/com/yahoo/vespa/model/container/ContainerClusterTest.java index 198b5713876..b634356fcb6 100755 --- a/config-model/src/test/java/com/yahoo/vespa/model/container/ContainerClusterTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/container/ContainerClusterTest.java @@ -162,15 +162,15 @@ public class ContainerClusterTest { addContainer(root, cluster, "c1", "host-c1"); assertEquals(1, cluster.getContainers().size()); ApplicationContainer container = cluster.getContainers().get(0); - verifyJvmArgs(isHosted, hasDocProc, "", container.getJvmOptions()); + verifyJvmArgs(isHosted, hasDocProc, expectedJvmArgs(isHosted, ""), container.getJvmOptions()); container.setJvmOptions("initial"); - verifyJvmArgs(isHosted, hasDocProc, "initial", container.getJvmOptions()); + verifyJvmArgs(isHosted, hasDocProc, expectedJvmArgs(isHosted, "initial"), container.getJvmOptions()); container.prependJvmOptions("ignored"); - verifyJvmArgs(isHosted, hasDocProc, "ignored initial", container.getJvmOptions()); + verifyJvmArgs(isHosted, hasDocProc, expectedJvmArgs(isHosted, "ignored initial"), container.getJvmOptions()); container.appendJvmOptions("override"); - verifyJvmArgs(isHosted, hasDocProc, "ignored initial override", container.getJvmOptions()); + verifyJvmArgs(isHosted, hasDocProc, expectedJvmArgs(isHosted, "ignored initial override"), container.getJvmOptions()); container.setJvmOptions(null); - verifyJvmArgs(isHosted, hasDocProc, "", container.getJvmOptions()); + verifyJvmArgs(isHosted, hasDocProc, expectedJvmArgs(isHosted, ""), container.getJvmOptions()); } @Test @@ -510,4 +510,13 @@ public class ContainerClusterTest { return new ClusterInfoConfig(builder); } + private static String expectedJvmArgs(boolean isHosted, String extra) { + if (!isHosted) return extra; + return "-Djdk.tls.server.enableStatusRequestExtension=true " + + "-Djdk.tls.stapling.responseTimeout=2000 " + + "-Djdk.tls.stapling.cacheSize=256 " + + "-Djdk.tls.stapling.cacheLifetime=3600" + + (extra.isEmpty() ? "" : " " + extra); + } + } diff --git a/config-model/src/test/java/com/yahoo/vespa/model/content/DistributorTest.java b/config-model/src/test/java/com/yahoo/vespa/model/content/DistributorTest.java index 2031e74bd5b..015356d6088 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/content/DistributorTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/content/DistributorTest.java @@ -273,7 +273,6 @@ public class DistributorTest { cluster.getChildren().get("0").getConfig(builder); StorCommunicationmanagerConfig config = new StorCommunicationmanagerConfig(builder); - assertTrue(config.mbus().dispatch_on_encode()); assertEquals(14066, config.rpcport()); } @@ -290,9 +289,6 @@ public class DistributorTest { cluster.getChildren().get("0").getConfig(builder); StorCommunicationmanagerConfig config = new StorCommunicationmanagerConfig(builder); - assertTrue(config.mbus().dispatch_on_encode()); - assertTrue(config.mbus().dispatch_on_decode()); - assertEquals(4, config.mbus().num_threads()); assertEquals(1, config.mbus().num_network_threads()); } diff --git a/config-model/src/test/java/com/yahoo/vespa/model/content/StorageClusterTest.java b/config-model/src/test/java/com/yahoo/vespa/model/content/StorageClusterTest.java index b74fc763462..c494ba0394a 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/content/StorageClusterTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/content/StorageClusterTest.java @@ -101,9 +101,6 @@ public class StorageClusterTest { StorCommunicationmanagerConfig.Builder builder = new StorCommunicationmanagerConfig.Builder(); storage.getChildren().get("0").getConfig(builder); StorCommunicationmanagerConfig config = new StorCommunicationmanagerConfig(builder); - assertTrue(config.mbus().dispatch_on_encode()); - assertTrue(config.mbus().dispatch_on_decode()); - assertEquals(4, config.mbus().num_threads()); assertEquals(1, config.mbus().num_network_threads()); } @@ -154,9 +151,6 @@ public class StorageClusterTest { @Test public void verifyDefaultMbusConfig() { var confg = communicationmanagerConfigFromProperties(new TestProperties()); - assertTrue(confg.mbus().dispatch_on_decode()); - assertTrue(confg.mbus().dispatch_on_encode()); - assertEquals(4, confg.mbus().num_threads()); assertEquals(1, confg.mbus().num_network_threads()); assertEquals(1, confg.mbus().num_rpc_targets()); assertEquals(1, confg.mbus().events_before_wakeup()); @@ -167,17 +161,11 @@ public class StorageClusterTest { @Test public void verifyDefaultMbusConfigControl() { var confg = communicationmanagerConfigFromProperties(new TestProperties() - .setMbusDispatchOnDecode(false) - .setMbusDispatchOnEncode(false) - .setMbusThreads(3) .setMbusNetworkThreads(7) .setRpcNumTargets(11) .setRpcEventsBeforeWakeup(12) .setMbusCppRpcNumTargets(8) .setMbusCppEventsBeforeWakeup(9)); - assertFalse(confg.mbus().dispatch_on_decode()); - assertFalse(confg.mbus().dispatch_on_encode()); - assertEquals(3, confg.mbus().num_threads()); assertEquals(7, confg.mbus().num_network_threads()); assertEquals(8, confg.mbus().num_rpc_targets()); assertEquals(9, confg.mbus().events_before_wakeup()); diff --git a/config/src/main/java/com/yahoo/config/subscription/ConfigGetter.java b/config/src/main/java/com/yahoo/config/subscription/ConfigGetter.java index aabfd211fac..316edda1ee8 100755 --- a/config/src/main/java/com/yahoo/config/subscription/ConfigGetter.java +++ b/config/src/main/java/com/yahoo/config/subscription/ConfigGetter.java @@ -1,7 +1,6 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.config.subscription; - import com.yahoo.config.ConfigInstance; /** @@ -13,7 +12,9 @@ import com.yahoo.config.ConfigInstance; * command-line tools. * * @author gjoranv + * @deprecated Use config builders where possible */ +@Deprecated public class ConfigGetter<T extends ConfigInstance> { private final Class<T> clazz; diff --git a/config/src/test/java/com/yahoo/config/subscription/ConfigGetterTest.java b/config/src/test/java/com/yahoo/config/subscription/ConfigGetterTest.java index fb116729640..68b5d6a37d3 100644 --- a/config/src/test/java/com/yahoo/config/subscription/ConfigGetterTest.java +++ b/config/src/test/java/com/yahoo/config/subscription/ConfigGetterTest.java @@ -12,6 +12,7 @@ import static org.junit.Assert.assertTrue; * * @author gjoranv */ +@SuppressWarnings("deprecation") public class ConfigGetterTest { private final ConfigSourceSet sourceSet = new ConfigSourceSet("config-getter-test"); diff --git a/config/src/test/java/com/yahoo/config/subscription/FunctionTest.java b/config/src/test/java/com/yahoo/config/subscription/FunctionTest.java index 42c2c599899..8656c0e945f 100644 --- a/config/src/test/java/com/yahoo/config/subscription/FunctionTest.java +++ b/config/src/test/java/com/yahoo/config/subscription/FunctionTest.java @@ -2,20 +2,16 @@ package com.yahoo.config.subscription; import com.yahoo.foo.FunctionTestConfig; - import org.junit.Before; import org.junit.Test; - import java.io.BufferedReader; import java.io.File; import java.io.FileReader; import java.io.IOException; -import static org.hamcrest.CoreMatchers.is; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertNull; -import static org.junit.Assert.assertThat; import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; @@ -28,6 +24,7 @@ import static org.junit.Assert.fail; * * @author gjoranv */ +@SuppressWarnings("deprecation") public class FunctionTest { public static final String PATH = "src/test/resources/configs/function-test/"; diff --git a/config/src/test/java/com/yahoo/config/subscription/NamespaceTest.java b/config/src/test/java/com/yahoo/config/subscription/NamespaceTest.java index 963f3de5e43..2e7beb0f0f4 100644 --- a/config/src/test/java/com/yahoo/config/subscription/NamespaceTest.java +++ b/config/src/test/java/com/yahoo/config/subscription/NamespaceTest.java @@ -12,6 +12,7 @@ import static org.junit.Assert.assertEquals; public class NamespaceTest { @Test + @SuppressWarnings("deprecation") public void verifyConfigClassWithExplicitNamespace() { NamespaceConfig config = new ConfigGetter<>(NamespaceConfig.class).getConfig("raw: a 0\n"); assertEquals(0, config.a()); diff --git a/config/src/test/java/com/yahoo/config/subscription/UnicodeTest.java b/config/src/test/java/com/yahoo/config/subscription/UnicodeTest.java index 8864f114d87..c1d98ac7e3e 100644 --- a/config/src/test/java/com/yahoo/config/subscription/UnicodeTest.java +++ b/config/src/test/java/com/yahoo/config/subscription/UnicodeTest.java @@ -22,6 +22,7 @@ public class UnicodeTest { * received correctly from the server */ @Test + @SuppressWarnings("deprecation") public void testUnicodeConfigReading() { ConfigGetter<UnicodeConfig> getter = new ConfigGetter<>(UnicodeConfig.class); UnicodeConfig config = getter.getConfig("file:src/test/resources/configs/unicode/unicode.cfg"); diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java index 177f7c5b823..068323f7784 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java @@ -197,7 +197,6 @@ public class ModelContextImpl implements ModelContext { private final boolean useV8GeoPositions; private final int maxCompactBuffers; private final List<String> ignoredHttpUserAgents; - private final boolean enableServerOcspStapling; private final String mergeThrottlingPolicy; private final double persistenceThrottlingWsDecrementFactor; private final double persistenceThrottlingWsBackoff; @@ -258,7 +257,6 @@ public class ModelContextImpl implements ModelContext { this.useV8GeoPositions = flagValue(source, appId, version, Flags.USE_V8_GEO_POSITIONS); this.maxCompactBuffers = flagValue(source, appId, version, Flags.MAX_COMPACT_BUFFERS); this.ignoredHttpUserAgents = flagValue(source, appId, version, PermanentFlags.IGNORED_HTTP_USER_AGENTS); - this.enableServerOcspStapling = flagValue(source, appId, version, Flags.ENABLE_SERVER_OCSP_STAPLING); this.mergeThrottlingPolicy = flagValue(source, appId, version, Flags.MERGE_THROTTLING_POLICY); this.persistenceThrottlingWsDecrementFactor = flagValue(source, appId, version, Flags.PERSISTENCE_THROTTLING_WS_DECREMENT_FACTOR); this.persistenceThrottlingWsBackoff = flagValue(source, appId, version, Flags.PERSISTENCE_THROTTLING_WS_BACKOFF); @@ -317,7 +315,6 @@ public class ModelContextImpl implements ModelContext { @Override public boolean useV8GeoPositions() { return useV8GeoPositions; } @Override public int maxCompactBuffers() { return maxCompactBuffers; } @Override public List<String> ignoredHttpUserAgents() { return ignoredHttpUserAgents; } - @Override public boolean enableServerOcspStapling() { return enableServerOcspStapling; } @Override public String mergeThrottlingPolicy() { return mergeThrottlingPolicy; } @Override public double persistenceThrottlingWsDecrementFactor() { return persistenceThrottlingWsDecrementFactor; } @Override public double persistenceThrottlingWsBackoff() { return persistenceThrottlingWsBackoff; } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionRepository.java b/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionRepository.java index a144940e443..d803488cb0a 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionRepository.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionRepository.java @@ -44,13 +44,11 @@ import com.yahoo.vespa.curator.Curator; import com.yahoo.vespa.defaults.Defaults; import com.yahoo.vespa.flags.FlagSource; import com.yahoo.vespa.flags.Flags; -import com.yahoo.vespa.flags.StringFlag; import com.yahoo.yolean.Exceptions; import org.apache.curator.framework.CuratorFramework; import org.apache.curator.framework.recipes.cache.ChildData; import org.apache.curator.framework.recipes.cache.PathChildrenCacheEvent; import org.apache.zookeeper.KeeperException; - import java.io.File; import java.io.FilenameFilter; import java.io.IOException; @@ -83,6 +81,7 @@ import java.util.logging.Logger; import java.util.stream.Collectors; import static com.yahoo.vespa.curator.Curator.CompletionWaiter; +import static com.yahoo.vespa.flags.FetchVector.Dimension.APPLICATION_ID; import static java.nio.file.Files.readAttributes; /** @@ -128,7 +127,6 @@ public class SessionRepository { private final ModelFactoryRegistry modelFactoryRegistry; private final ConfigDefinitionRepo configDefinitionRepo; private final int maxNodeSize; - private final StringFlag failDeploymentForFilesWithUnknownExtension; public SessionRepository(TenantName tenantName, TenantApplications applicationRepo, @@ -172,7 +170,6 @@ public class SessionRepository { this.modelFactoryRegistry = modelFactoryRegistry; this.configDefinitionRepo = configDefinitionRepo; this.maxNodeSize = maxNodeSize; - this.failDeploymentForFilesWithUnknownExtension = Flags.APPLICATION_FILES_WITH_UNKNOWN_EXTENSION.bindTo(flagSource); loadSessions(); // Needs to be done before creating cache below this.directoryCache = curator.createDirectoryCache(sessionsPath.getAbsolute(), false, false, zkCacheExecutor); @@ -684,7 +681,10 @@ public class SessionRepository { try { app.validateFileExtensions(); } catch (IllegalArgumentException e) { - switch (failDeploymentForFilesWithUnknownExtension.value()) { + String flag = Flags.APPLICATION_FILES_WITH_UNKNOWN_EXTENSION.bindTo(flagSource) + .with(APPLICATION_ID, applicationId.serializedForm()) + .value(); + switch (flag) { case "FAIL": throw e; case "LOG": diff --git a/container-core/src/main/java/com/yahoo/container/jdisc/state/MetricsPacketsHandler.java b/container-core/src/main/java/com/yahoo/container/jdisc/state/MetricsPacketsHandler.java index b949edefb31..8fa658bf7fc 100644 --- a/container-core/src/main/java/com/yahoo/container/jdisc/state/MetricsPacketsHandler.java +++ b/container-core/src/main/java/com/yahoo/container/jdisc/state/MetricsPacketsHandler.java @@ -22,9 +22,9 @@ import java.nio.ByteBuffer; import java.nio.charset.StandardCharsets; import java.util.ArrayList; import java.util.Collections; -import java.util.Iterator; import java.util.List; import java.util.Map; +import java.util.Set; import java.util.concurrent.TimeUnit; import static com.yahoo.container.jdisc.state.JsonUtil.sanitizeDouble; @@ -61,6 +61,7 @@ public class MetricsPacketsHandler extends AbstractRequestHandler { private final Timer timer; private final SnapshotProvider snapshotProvider; private final String applicationName; + private final String hostDimension; @Inject public MetricsPacketsHandler(StateMonitor monitor, @@ -71,6 +72,7 @@ public class MetricsPacketsHandler extends AbstractRequestHandler { this.timer = timer; snapshotProvider = getSnapshotProviderOrThrow(snapshotProviders); applicationName = config.application(); + hostDimension = config.hostname(); } @@ -173,16 +175,16 @@ public class MetricsPacketsHandler extends AbstractRequestHandler { } private void addDimensions(MetricDimensions metricDimensions, ObjectNode packet) { - if (metricDimensions == null) return; - - Iterator<Map.Entry<String, String>> dimensionsIterator = metricDimensions.iterator(); - if (dimensionsIterator.hasNext()) { - ObjectNode jsonDim = jsonMapper.createObjectNode(); - packet.set(DIMENSIONS_KEY, jsonDim); - for (Map.Entry<String, String> dimensionEntry : metricDimensions) { - jsonDim.put(dimensionEntry.getKey(), dimensionEntry.getValue()); - } + if (metricDimensions == null && hostDimension.isEmpty()) return; + + ObjectNode jsonDim = jsonMapper.createObjectNode(); + packet.set(DIMENSIONS_KEY, jsonDim); + Iterable<Map.Entry<String, String>> dimensionIterator = metricDimensions == null ? Set.of() : metricDimensions; + for (Map.Entry<String, String> dimensionEntry : dimensionIterator) { + jsonDim.put(dimensionEntry.getKey(), dimensionEntry.getValue()); } + if (!hostDimension.isEmpty() && !jsonDim.has("host")) + jsonDim.put("host", hostDimension); } private void addMetrics(MetricSet metricSet, ObjectNode packet) { diff --git a/container-core/src/main/resources/configdefinitions/container.jdisc.state.metrics-packets-handler.def b/container-core/src/main/resources/configdefinitions/container.jdisc.state.metrics-packets-handler.def index 9ec81b7db1b..ab0362c3995 100644 --- a/container-core/src/main/resources/configdefinitions/container.jdisc.state.metrics-packets-handler.def +++ b/container-core/src/main/resources/configdefinitions/container.jdisc.state.metrics-packets-handler.def @@ -4,3 +4,5 @@ namespace=container.jdisc.state # The name of the application that is reporting metrics. application string +# Optional hostname to add as dimension +hostname string default=""
\ No newline at end of file diff --git a/container-core/src/test/java/com/yahoo/container/di/componentgraph/core/ComponentGraphTest.java b/container-core/src/test/java/com/yahoo/container/di/componentgraph/core/ComponentGraphTest.java index 4d44281658c..f8f567e1890 100644 --- a/container-core/src/test/java/com/yahoo/container/di/componentgraph/core/ComponentGraphTest.java +++ b/container-core/src/test/java/com/yahoo/container/di/componentgraph/core/ComponentGraphTest.java @@ -50,6 +50,7 @@ public class ComponentGraphTest { super(); } + @SuppressWarnings("deprecation") public <T extends ConfigInstance> ConfigMap add(Class<T> clazz, String configId) { ConfigKey<T> key = new ConfigKey<>(clazz, configId); put(key, ConfigGetter.getConfig(key.getConfigClass(), key.getConfigId())); diff --git a/container-core/src/test/java/com/yahoo/container/di/componentgraph/core/ReuseComponentsTest.java b/container-core/src/test/java/com/yahoo/container/di/componentgraph/core/ReuseComponentsTest.java index 29452f7babe..2215fcdf4dd 100644 --- a/container-core/src/test/java/com/yahoo/container/di/componentgraph/core/ReuseComponentsTest.java +++ b/container-core/src/test/java/com/yahoo/container/di/componentgraph/core/ReuseComponentsTest.java @@ -62,6 +62,7 @@ public class ReuseComponentsTest { SimpleComponent throwsException = getComponent(newGraph, SimpleComponent.class); } + @SuppressWarnings("deprecation") @Test public void require_that_component_is_not_reused_when_config_is_changed() { Class<ComponentTakingConfig> componentClass = ComponentTakingConfig.class; @@ -80,6 +81,7 @@ public class ReuseComponentsTest { assertNotSame(instance2, instance); } + @SuppressWarnings("deprecation") @Test public void require_that_component_is_not_reused_when_injected_component_is_changed() { Function<String, ComponentGraph> buildGraph = config -> { @@ -144,6 +146,7 @@ public class ReuseComponentsTest { assertNotSame(newSimpleComponentRegistry, oldSimpleComponentRegistry); } + @SuppressWarnings("deprecation") @Test public void require_that_injected_component_is_reused_even_when_dependent_component_is_changed() { Function<String, ComponentGraph> buildGraph = config -> { diff --git a/container-core/src/test/java/com/yahoo/container/jdisc/state/MetricsPacketsHandlerTest.java b/container-core/src/test/java/com/yahoo/container/jdisc/state/MetricsPacketsHandlerTest.java index 99a31640429..6c05af95289 100644 --- a/container-core/src/test/java/com/yahoo/container/jdisc/state/MetricsPacketsHandlerTest.java +++ b/container-core/src/test/java/com/yahoo/container/jdisc/state/MetricsPacketsHandlerTest.java @@ -27,13 +27,14 @@ import static org.junit.Assert.assertTrue; public class MetricsPacketsHandlerTest extends StateHandlerTestBase { private static final String APPLICATION_NAME = "state-handler-test-base"; + private static final String HOST_DIMENSION = "some-hostname"; private static MetricsPacketsHandler metricsPacketsHandler; @Before public void setupHandler() { metricsPacketsHandlerConfig = new MetricsPacketsHandlerConfig(new MetricsPacketsHandlerConfig.Builder() - .application(APPLICATION_NAME)); + .application(APPLICATION_NAME).hostname(HOST_DIMENSION)); metricsPacketsHandler = new MetricsPacketsHandler(monitor, timer, snapshotProviderRegistry, metricsPacketsHandlerConfig); testDriver = new RequestHandlerTestDriver(metricsPacketsHandler); } @@ -138,6 +139,26 @@ public class MetricsPacketsHandlerTest extends StateHandlerTestBase { List<JsonNode> packets = incrementTimeAndGetJsonPackets(); assertEquals(3, packets.size()); } + + @Test + public void host_dimension_only_created_if_absent() throws Exception { + var context1 = StateMetricContext.newInstance(Map.of("dim1", "value1", "host", "foo.bar")); + var context2 = StateMetricContext.newInstance(Map.of("dim2", "value2")); + var snapshot = new MetricSnapshot(); + snapshot.add(context1, "counter1", 1); + snapshot.add(context2, "counter2", 2); + snapshotProvider.setSnapshot(snapshot); + + var packets = incrementTimeAndGetJsonPackets(); + assertEquals(3, packets.size()); + + packets.forEach(packet -> { + if (!packet.has(DIMENSIONS_KEY)) return; + var dimensions = packet.get(DIMENSIONS_KEY); + if (dimensions.has("dim1")) assertDimension(packet, "host", "foo.bar"); + if (dimensions.has("dim2")) assertDimension(packet, "host", HOST_DIMENSION); + }); + } private List<JsonNode> incrementTimeAndGetJsonPackets() throws Exception { advanceToNextSnapshot(); @@ -163,6 +184,13 @@ public class MetricsPacketsHandlerTest extends StateHandlerTestBase { assertEquals(expected, counterMetrics.get(metricName).asLong()); } + private void assertDimension(JsonNode metricsPacket, String dimensionName, String expectedDimensionValue) { + assertTrue(metricsPacket.has(DIMENSIONS_KEY)); + var dimensions = metricsPacket.get(DIMENSIONS_KEY); + assertTrue(dimensions.has(dimensionName)); + assertEquals(expectedDimensionValue, dimensions.get(dimensionName).asText()); + } + private void createSnapshotWithCountMetric(String name, Number value, MetricDimensions context) { var snapshot = new MetricSnapshot(); snapshot.add(context, name, value); diff --git a/container-messagebus/src/main/java/com/yahoo/messagebus/shared/SharedMessageBus.java b/container-messagebus/src/main/java/com/yahoo/messagebus/shared/SharedMessageBus.java index e3042310ad0..cf50eeb2c8d 100644 --- a/container-messagebus/src/main/java/com/yahoo/messagebus/shared/SharedMessageBus.java +++ b/container-messagebus/src/main/java/com/yahoo/messagebus/shared/SharedMessageBus.java @@ -1,12 +1,9 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.messagebus.shared; +import com.yahoo.cloud.config.SlobroksConfig; import com.yahoo.config.subscription.ConfigGetter; import com.yahoo.jdisc.AbstractResource; - -import java.util.Objects; -import java.util.logging.Level; - import com.yahoo.messagebus.DestinationSessionParams; import com.yahoo.messagebus.IntermediateSessionParams; import com.yahoo.messagebus.MessageBus; @@ -15,8 +12,8 @@ import com.yahoo.messagebus.SourceSessionParams; import com.yahoo.messagebus.network.Network; import com.yahoo.messagebus.network.rpc.RPCNetwork; import com.yahoo.messagebus.network.rpc.RPCNetworkParams; -import com.yahoo.cloud.config.SlobroksConfig; - +import java.util.Objects; +import java.util.logging.Level; import java.util.logging.Logger; /** @@ -57,6 +54,7 @@ public class SharedMessageBus extends AbstractResource { return new SharedMessageBus(new MessageBus(newNetwork(netParams), mbusParams)); } + @SuppressWarnings("deprecation") private static Network newNetwork(RPCNetworkParams params) { SlobroksConfig cfg = params.getSlobroksConfig(); if (cfg == null) { diff --git a/container-search/src/main/java/com/yahoo/search/query/profile/config/QueryProfileConfigurer.java b/container-search/src/main/java/com/yahoo/search/query/profile/config/QueryProfileConfigurer.java index 31278af9579..5b30e3c383d 100644 --- a/container-search/src/main/java/com/yahoo/search/query/profile/config/QueryProfileConfigurer.java +++ b/container-search/src/main/java/com/yahoo/search/query/profile/config/QueryProfileConfigurer.java @@ -21,6 +21,7 @@ import java.util.Set; */ public class QueryProfileConfigurer { + @SuppressWarnings("deprecation") public static QueryProfileRegistry createFromConfigId(String configId) { return createFromConfig(ConfigGetter.getConfig(QueryProfilesConfig.class, configId)); } diff --git a/container-search/src/test/java/com/yahoo/prelude/IndexFactsFactory.java b/container-search/src/test/java/com/yahoo/prelude/IndexFactsFactory.java index 88cc066b665..482c9e3c6ba 100644 --- a/container-search/src/test/java/com/yahoo/prelude/IndexFactsFactory.java +++ b/container-search/src/test/java/com/yahoo/prelude/IndexFactsFactory.java @@ -23,6 +23,7 @@ public abstract class IndexFactsFactory { } + @SuppressWarnings("deprecation") private static <T extends ConfigInstance> T resolveConfig(Class<T> configClass, String configId) { if (configId == null) return null; return ConfigGetter.getConfig(configClass, configId); diff --git a/container-search/src/test/java/com/yahoo/prelude/query/parser/test/ParsingTester.java b/container-search/src/test/java/com/yahoo/prelude/query/parser/test/ParsingTester.java index 229416f7a85..42bc1c22529 100644 --- a/container-search/src/test/java/com/yahoo/prelude/query/parser/test/ParsingTester.java +++ b/container-search/src/test/java/com/yahoo/prelude/query/parser/test/ParsingTester.java @@ -62,6 +62,7 @@ public class ParsingTester { * Returns an unfrozen version of the IndexFacts this will use. * This can be used to add new indexes and passing the resulting IndexFacts to the constructor of this. */ + @SuppressWarnings("deprecation") public static IndexFacts createIndexFacts() { String indexInfoConfigID = "file:src/test/java/com/yahoo/prelude/query/parser/test/parseindexinfo.cfg"; ConfigGetter<IndexInfoConfig> getter = new ConfigGetter<>(IndexInfoConfig.class); diff --git a/container-search/src/test/java/com/yahoo/prelude/querytransform/test/StemmingSearcherTestCase.java b/container-search/src/test/java/com/yahoo/prelude/querytransform/test/StemmingSearcherTestCase.java index 55ce7ed4e99..0d7b0caa65b 100644 --- a/container-search/src/test/java/com/yahoo/prelude/querytransform/test/StemmingSearcherTestCase.java +++ b/container-search/src/test/java/com/yahoo/prelude/querytransform/test/StemmingSearcherTestCase.java @@ -96,6 +96,7 @@ public class StemmingSearcherTestCase { "WEAKAND(100) notnoun:tower notnoun:tower notnoun:tow"); } + @SuppressWarnings("deprecation") @Test public void testEmptyIndexInfo() { String indexInfoConfigID = "file:src/test/java/com/yahoo/prelude/querytransform/test/emptyindexinfo.cfg"; diff --git a/container-search/src/test/java/com/yahoo/prelude/searcher/test/QuotingSearcherTestCase.java b/container-search/src/test/java/com/yahoo/prelude/searcher/test/QuotingSearcherTestCase.java index ef3526f2fb1..0b926f5e9b7 100644 --- a/container-search/src/test/java/com/yahoo/prelude/searcher/test/QuotingSearcherTestCase.java +++ b/container-search/src/test/java/com/yahoo/prelude/searcher/test/QuotingSearcherTestCase.java @@ -4,21 +4,18 @@ package com.yahoo.prelude.searcher.test; import com.yahoo.component.ComponentId; import com.yahoo.component.chain.Chain; import com.yahoo.config.subscription.ConfigGetter; -import com.yahoo.language.simple.SimpleLinguistics; +import com.yahoo.prelude.fastsearch.FastHit; +import com.yahoo.prelude.hitfield.HitField; import com.yahoo.prelude.searcher.QrQuotetableConfig; -import com.yahoo.search.rendering.RendererRegistry; -import com.yahoo.search.result.Hit; -import com.yahoo.search.result.Relevance; +import com.yahoo.prelude.searcher.QuotingSearcher; import com.yahoo.search.Query; import com.yahoo.search.Result; -import com.yahoo.prelude.fastsearch.FastHit; -import com.yahoo.prelude.hitfield.HitField; import com.yahoo.search.Searcher; -import com.yahoo.prelude.searcher.QuotingSearcher; +import com.yahoo.search.result.Hit; +import com.yahoo.search.result.Relevance; import com.yahoo.search.searchchain.Execution; import com.yahoo.search.searchchain.testutil.DocumentSourceSearcher; import org.junit.Test; - import java.util.ArrayList; import java.util.HashMap; import java.util.List; @@ -34,6 +31,7 @@ import static org.junit.Assert.assertTrue; */ public class QuotingSearcherTestCase { + @SuppressWarnings("deprecation") public static QuotingSearcher createQuotingSearcher(String configId) { QrQuotetableConfig config = new ConfigGetter<>(QrQuotetableConfig.class).getConfig(configId); return new QuotingSearcher(new ComponentId("QuotingSearcher"), config); diff --git a/container-search/src/test/java/com/yahoo/prelude/searcher/test/ValidateSortingSearcherTestCase.java b/container-search/src/test/java/com/yahoo/prelude/searcher/test/ValidateSortingSearcherTestCase.java index 84b8ef32871..6b8c268f4ca 100644 --- a/container-search/src/test/java/com/yahoo/prelude/searcher/test/ValidateSortingSearcherTestCase.java +++ b/container-search/src/test/java/com/yahoo/prelude/searcher/test/ValidateSortingSearcherTestCase.java @@ -2,24 +2,23 @@ package com.yahoo.prelude.searcher.test; import com.yahoo.component.chain.Chain; -import com.yahoo.language.simple.SimpleLinguistics; -import com.yahoo.search.Searcher; -import com.yahoo.search.rendering.RendererRegistry; -import com.yahoo.search.searchchain.Execution; -import com.yahoo.vespa.config.search.AttributesConfig; -import com.yahoo.search.config.ClusterConfig; import com.yahoo.config.subscription.ConfigGetter; import com.yahoo.container.QrSearchersConfig; import com.yahoo.prelude.searcher.ValidateSortingSearcher; import com.yahoo.search.Query; import com.yahoo.search.Result; +import com.yahoo.search.Searcher; +import com.yahoo.search.config.ClusterConfig; +import com.yahoo.search.searchchain.Execution; import com.yahoo.search.test.QueryTestCase; +import com.yahoo.vespa.config.search.AttributesConfig; import org.junit.Test; - import java.util.ArrayList; import java.util.List; -import static org.junit.Assert.*; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertNull; /** * Check sorting validation behaves OK. @@ -30,6 +29,7 @@ public class ValidateSortingSearcherTestCase { private final ValidateSortingSearcher searcher; + @SuppressWarnings("deprecation") public ValidateSortingSearcherTestCase() { QrSearchersConfig.Builder qrsCfg = new QrSearchersConfig.Builder(); qrsCfg.searchcluster(new QrSearchersConfig.Searchcluster.Builder().name("giraffes")); diff --git a/container-search/src/test/java/com/yahoo/prelude/semantics/test/ConfigurationTestCase.java b/container-search/src/test/java/com/yahoo/prelude/semantics/test/ConfigurationTestCase.java index 0515417f515..7d9ea07339f 100644 --- a/container-search/src/test/java/com/yahoo/prelude/semantics/test/ConfigurationTestCase.java +++ b/container-search/src/test/java/com/yahoo/prelude/semantics/test/ConfigurationTestCase.java @@ -4,18 +4,16 @@ package com.yahoo.prelude.semantics.test; import com.yahoo.component.chain.Chain; import com.yahoo.config.subscription.ConfigGetter; import com.yahoo.language.simple.SimpleLinguistics; -import com.yahoo.prelude.semantics.SemanticRulesConfig; -import com.yahoo.search.Query; import com.yahoo.prelude.semantics.RuleBase; import com.yahoo.prelude.semantics.RuleBaseException; +import com.yahoo.prelude.semantics.SemanticRulesConfig; import com.yahoo.prelude.semantics.SemanticSearcher; +import com.yahoo.search.Query; import com.yahoo.search.Result; import com.yahoo.search.Searcher; -import com.yahoo.search.rendering.RendererRegistry; import com.yahoo.search.searchchain.Execution; import com.yahoo.search.test.QueryTestCase; import org.junit.Test; - import java.util.ArrayList; import java.util.List; @@ -28,6 +26,7 @@ import static org.junit.Assert.fail; * * @author bratseth */ +@SuppressWarnings("deprecation") public class ConfigurationTestCase { private static final String root="src/test/java/com/yahoo/prelude/semantics/test/rulebases/"; diff --git a/container-search/src/test/java/com/yahoo/prelude/test/IndexFactsTestCase.java b/container-search/src/test/java/com/yahoo/prelude/test/IndexFactsTestCase.java index 0f7c9526533..6a54492f5e5 100644 --- a/container-search/src/test/java/com/yahoo/prelude/test/IndexFactsTestCase.java +++ b/container-search/src/test/java/com/yahoo/prelude/test/IndexFactsTestCase.java @@ -1,39 +1,29 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.prelude.test; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collection; -import java.util.Collections; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.TreeMap; - import com.google.common.collect.ImmutableList; import com.yahoo.config.subscription.ConfigGetter; -import com.yahoo.container.QrSearchersConfig; -import com.yahoo.search.config.IndexInfoConfig; -import com.yahoo.search.config.IndexInfoConfig.Indexinfo; -import com.yahoo.search.config.IndexInfoConfig.Indexinfo.Alias; -import com.yahoo.search.config.IndexInfoConfig.Indexinfo.Command; import com.yahoo.language.process.StemMode; import com.yahoo.prelude.Index; import com.yahoo.prelude.IndexFacts; import com.yahoo.prelude.IndexModel; import com.yahoo.prelude.SearchDefinition; import com.yahoo.search.Query; +import com.yahoo.search.config.IndexInfoConfig; import com.yahoo.search.searchchain.Execution; import org.junit.Test; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertNotNull; -import static org.junit.Assert.assertNotSame; import static org.junit.Assert.assertNull; -import static org.junit.Assert.assertSame; import static org.junit.Assert.assertTrue; -import static org.junit.Assert.fail; /** * Tests using synthetic index names for IndexFacts class. @@ -45,6 +35,7 @@ public class IndexFactsTestCase { private static final String INDEXFACTS_TESTING = "file:src/test/java/com/yahoo/prelude/test/indexfactstesting.cfg"; + @SuppressWarnings("deprecation") private IndexFacts createIndexFacts() { ConfigGetter<IndexInfoConfig> getter = new ConfigGetter<>(IndexInfoConfig.class); IndexInfoConfig config = getter.getConfig(INDEXFACTS_TESTING); diff --git a/container-search/src/test/java/com/yahoo/search/query/rewrite/test/QueryRewriteSearcherTestUtils.java b/container-search/src/test/java/com/yahoo/search/query/rewrite/test/QueryRewriteSearcherTestUtils.java index da005871539..c12a3fb0e14 100644 --- a/container-search/src/test/java/com/yahoo/search/query/rewrite/test/QueryRewriteSearcherTestUtils.java +++ b/container-search/src/test/java/com/yahoo/search/query/rewrite/test/QueryRewriteSearcherTestUtils.java @@ -40,6 +40,7 @@ public class QueryRewriteSearcherTestUtils { * * @param configPath path for the searcher config */ + @SuppressWarnings("deprecation") public static RewritesConfig createConfigObj(String configPath) { return new ConfigGetter<>(RewritesConfig.class).getConfig(configPath); } diff --git a/container-search/src/test/java/com/yahoo/search/searchers/ValidateFuzzySearcherTestCase.java b/container-search/src/test/java/com/yahoo/search/searchers/ValidateFuzzySearcherTestCase.java index 587b40dfd03..577963ef2b3 100644 --- a/container-search/src/test/java/com/yahoo/search/searchers/ValidateFuzzySearcherTestCase.java +++ b/container-search/src/test/java/com/yahoo/search/searchers/ValidateFuzzySearcherTestCase.java @@ -1,7 +1,6 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.search.searchers; -import com.yahoo.config.subscription.ConfigGetter; import com.yahoo.prelude.IndexFacts; import com.yahoo.prelude.IndexModel; import com.yahoo.prelude.SearchDefinition; @@ -13,10 +12,9 @@ import com.yahoo.search.query.parser.ParserEnvironment; import com.yahoo.search.result.ErrorMessage; import com.yahoo.search.searchchain.Execution; import com.yahoo.search.yql.YqlParser; -import com.yahoo.vespa.config.search.AttributesConfig.Attribute; import com.yahoo.vespa.config.search.AttributesConfig; +import com.yahoo.vespa.config.search.AttributesConfig.Attribute; import org.junit.Test; - import java.util.ArrayList; import java.util.List; import java.util.Set; @@ -33,35 +31,25 @@ public class ValidateFuzzySearcherTestCase { List<String> attributes; public ValidateFuzzySearcherTestCase() { - int i = 0; attributes = new ArrayList<>(); - StringBuilder attributeConfig = new StringBuilder(); + AttributesConfig.Builder configBuilder = new AttributesConfig.Builder(); + List<AttributesConfig.Attribute.Builder> attributesList = new ArrayList<>(); for (Attribute.Datatype.Enum attr: Attribute.Datatype.Enum.values()) { for (Attribute.Collectiontype.Enum ctype: Attribute.Collectiontype.Enum.values()) { + AttributesConfig.Attribute.Builder attributesBuilder = new AttributesConfig.Attribute.Builder(); String attributeName = attr.name().toLowerCase() + "_" + ctype.name().toLowerCase(); + attributesBuilder.name(attributeName); + attributesBuilder.datatype(attr); + attributesBuilder.collectiontype(ctype); + attributesList.add(attributesBuilder); - attributeConfig.append("attribute[" + i + "].name "); - attributeConfig.append(attributeName); - attributeConfig.append("\n"); - - attributeConfig.append("attribute[" + i + "].datatype "); - attributeConfig.append(attr.name()); - attributeConfig.append("\n"); - - attributeConfig.append("attribute[" + i + "].collectiontype "); - attributeConfig.append(ctype.name()); - attributeConfig.append("\n"); - - i += 1; attributes.add(attributeName); } } + configBuilder.attribute(attributesList); + AttributesConfig config = configBuilder.build(); - searcher = new ValidateFuzzySearcher(ConfigGetter.getConfig( - AttributesConfig.class, - "raw: " + - "attribute[" + attributes.size() + "]\n" + - attributeConfig)); + searcher = new ValidateFuzzySearcher(config); } private String makeQuery(String attribute, String query, int maxEditDistance, int prefixLength) { diff --git a/container-search/src/test/java/com/yahoo/search/searchers/ValidateNearestNeighborTestCase.java b/container-search/src/test/java/com/yahoo/search/searchers/ValidateNearestNeighborTestCase.java index 9a760bfb0cb..c2f4ee31aa8 100644 --- a/container-search/src/test/java/com/yahoo/search/searchers/ValidateNearestNeighborTestCase.java +++ b/container-search/src/test/java/com/yahoo/search/searchers/ValidateNearestNeighborTestCase.java @@ -2,28 +2,22 @@ package com.yahoo.search.searchers; import com.yahoo.config.subscription.ConfigGetter; -import com.yahoo.config.subscription.FileSource; -import com.yahoo.config.subscription.RawSource; import com.yahoo.prelude.IndexFacts; import com.yahoo.prelude.IndexModel; import com.yahoo.prelude.SearchDefinition; import com.yahoo.search.Query; +import com.yahoo.search.Result; +import com.yahoo.search.query.QueryTree; import com.yahoo.search.query.parser.Parsable; import com.yahoo.search.query.parser.ParserEnvironment; -import com.yahoo.search.query.QueryTree; -import com.yahoo.search.Result; import com.yahoo.search.result.ErrorMessage; import com.yahoo.search.searchchain.Execution; import com.yahoo.search.yql.YqlParser; import com.yahoo.tensor.Tensor; import com.yahoo.tensor.TensorType; import com.yahoo.vespa.config.search.AttributesConfig; - -import com.yahoo.vespa.config.search.RankProfilesConfig; import org.junit.Test; -import java.io.File; - import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNull; @@ -34,6 +28,7 @@ public class ValidateNearestNeighborTestCase { ValidateNearestNeighborSearcher searcher; + @SuppressWarnings("deprecation") public ValidateNearestNeighborTestCase() { searcher = new ValidateNearestNeighborSearcher( ConfigGetter.getConfig(AttributesConfig.class, diff --git a/container-search/src/test/java/com/yahoo/search/searchers/test/ValidateMatchPhaseSearcherTestCase.java b/container-search/src/test/java/com/yahoo/search/searchers/test/ValidateMatchPhaseSearcherTestCase.java index 01e360858c1..1b3689a94b0 100644 --- a/container-search/src/test/java/com/yahoo/search/searchers/test/ValidateMatchPhaseSearcherTestCase.java +++ b/container-search/src/test/java/com/yahoo/search/searchers/test/ValidateMatchPhaseSearcherTestCase.java @@ -3,7 +3,6 @@ package com.yahoo.search.searchers.test; import com.yahoo.component.chain.Chain; import com.yahoo.config.subscription.ConfigGetter; -import com.yahoo.config.subscription.RawSource; import com.yahoo.search.Searcher; import com.yahoo.search.searchchain.Execution; import com.yahoo.search.searchers.ValidateMatchPhaseSearcher; @@ -15,7 +14,7 @@ import org.junit.Test; import java.util.ArrayList; import java.util.List; -import static org.junit.Assert.*; +import static org.junit.Assert.assertEquals; /** * @author baldersheim @@ -24,6 +23,7 @@ public class ValidateMatchPhaseSearcherTestCase { private final ValidateMatchPhaseSearcher searcher; + @SuppressWarnings("deprecation") public ValidateMatchPhaseSearcherTestCase() { searcher = new ValidateMatchPhaseSearcher( ConfigGetter.getConfig(AttributesConfig.class, diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/Node.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/Node.java index a381bcd5bed..3355f765f42 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/Node.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/Node.java @@ -41,6 +41,7 @@ public class Node { private final Version wantedVersion; private final Version currentOsVersion; private final Version wantedOsVersion; + private final boolean deferOsUpgrade; private final DockerImage currentDockerImage; private final DockerImage wantedDockerImage; private final ServiceState serviceState; @@ -76,7 +77,7 @@ public class Node { private Node(String id, HostName hostname, Optional<HostName> parentHostname, State state, NodeType type, NodeResources resources, Optional<ApplicationId> owner, Version currentVersion, Version wantedVersion, - Version currentOsVersion, Version wantedOsVersion, Optional<Instant> currentFirmwareCheck, + Version currentOsVersion, Version wantedOsVersion, boolean deferOsUpgrade, Optional<Instant> currentFirmwareCheck, Optional<Instant> wantedFirmwareCheck, ServiceState serviceState, Optional<Instant> suspendedSince, long restartGeneration, long wantedRestartGeneration, long rebootGeneration, long wantedRebootGeneration, int cost, int failCount, Optional<String> flavor, String clusterId, @@ -97,6 +98,7 @@ public class Node { this.wantedVersion = Objects.requireNonNull(wantedVersion, "wantedVersion must be non-null"); this.currentOsVersion = Objects.requireNonNull(currentOsVersion, "currentOsVersion must be non-null"); this.wantedOsVersion = Objects.requireNonNull(wantedOsVersion, "wantedOsVersion must be non-null"); + this.deferOsUpgrade = deferOsUpgrade; this.currentFirmwareCheck = Objects.requireNonNull(currentFirmwareCheck, "currentFirmwareCheck must be non-null"); this.wantedFirmwareCheck = Objects.requireNonNull(wantedFirmwareCheck, "wantedFirmwareCheck must be non-null"); this.serviceState = Objects.requireNonNull(serviceState, "serviceState must be non-null"); @@ -184,6 +186,11 @@ public class Node { return wantedOsVersion; } + /** Returns whether the node is currently deferring any OS upgrade */ + public boolean deferOsUpgrade() { + return deferOsUpgrade; + } + /** The container image of this is currently running */ public DockerImage currentDockerImage() { return currentDockerImage; @@ -455,6 +462,7 @@ public class Node { private Version wantedVersion = Version.emptyVersion; private Version currentOsVersion = Version.emptyVersion; private Version wantedOsVersion = Version.emptyVersion; + private boolean deferOsUpgrade = false; private DockerImage currentDockerImage = DockerImage.EMPTY; private DockerImage wantedDockerImage = DockerImage.EMPTY; private Optional<Instant> currentFirmwareCheck = Optional.empty(); @@ -502,6 +510,7 @@ public class Node { this.wantedVersion = node.wantedVersion; this.currentOsVersion = node.currentOsVersion; this.wantedOsVersion = node.wantedOsVersion; + this.deferOsUpgrade = node.deferOsUpgrade; this.currentDockerImage = node.currentDockerImage; this.wantedDockerImage = node.wantedDockerImage; this.serviceState = node.serviceState; @@ -599,6 +608,11 @@ public class Node { return this; } + public Builder deferOsUpgrade(boolean deferOsUpgrade) { + this.deferOsUpgrade = deferOsUpgrade; + return this; + } + public Builder currentDockerImage(DockerImage currentDockerImage) { this.currentDockerImage = currentDockerImage; return this; @@ -761,7 +775,7 @@ public class Node { public Node build() { return new Node(id, hostname, parentHostname, state, type, resources, owner, currentVersion, wantedVersion, - currentOsVersion, wantedOsVersion, currentFirmwareCheck, wantedFirmwareCheck, serviceState, + currentOsVersion, wantedOsVersion, deferOsUpgrade, currentFirmwareCheck, wantedFirmwareCheck, serviceState, suspendedSince, restartGeneration, wantedRestartGeneration, rebootGeneration, wantedRebootGeneration, cost, failCount, flavor, clusterId, clusterType, group, index, retired, wantToRetire, wantToDeprovision, wantToRebuild, down, reservedTo, exclusiveTo, wantedDockerImage, diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/deployment/JobType.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/deployment/JobType.java index 09ddcf4fa5e..b6905a97b5f 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/deployment/JobType.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/deployment/JobType.java @@ -161,7 +161,7 @@ public final class JobType implements Comparable<JobType> { /** A serialized form of this: {@code <environment>.<region>[.test]}; the inverse of {@link #ofSerialized(String)} */ public String serialized() { - return zone.environment().value() + "." + zone.region().value() + (isProductionTest ? ".test" : ""); + return zone().value() + (isProductionTest ? ".test" : ""); } public String jobName() { diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/noderepository/NodeRepositoryNode.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/noderepository/NodeRepositoryNode.java index 8964e1b5127..d8a804ecd2c 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/noderepository/NodeRepositoryNode.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/noderepository/NodeRepositoryNode.java @@ -63,6 +63,8 @@ public class NodeRepositoryNode { private String currentOsVersion; @JsonProperty("wantedOsVersion") private String wantedOsVersion; + @JsonProperty("deferOsUpgrade") + private Boolean deferOsUpgrade; @JsonProperty("currentFirmwareCheck") private Long currentFirmwareCheck; @JsonProperty("wantedFirmwareCheck") @@ -250,6 +252,14 @@ public class NodeRepositoryNode { this.wantedVespaVersion = wantedVespaVersion; } + public Boolean getDeferOsUpgrade() { + return deferOsUpgrade; + } + + public void setDeferOsUpgrade(Boolean deferOsUpgrade) { + this.deferOsUpgrade = deferOsUpgrade; + } + public Integer getFailCount() { return failCount; } @@ -441,12 +451,13 @@ public class NodeRepositoryNode { // --- end + @Override public String toString() { return "NodeRepositoryNode{" + "url='" + url + '\'' + ", id='" + id + '\'' + - ", state=" + state + + ", state='" + state + '\'' + ", hostname='" + hostname + '\'' + ", ipAddresses=" + ipAddresses + ", additionalIpAddresses=" + additionalIpAddresses + @@ -464,11 +475,12 @@ public class NodeRepositoryNode { ", wantedVespaVersion='" + wantedVespaVersion + '\'' + ", currentOsVersion='" + currentOsVersion + '\'' + ", wantedOsVersion='" + wantedOsVersion + '\'' + + ", deferOsUpgrade=" + deferOsUpgrade + ", currentFirmwareCheck=" + currentFirmwareCheck + ", wantedFirmwareCheck=" + wantedFirmwareCheck + ", failCount=" + failCount + - ", environment=" + environment + - ", type=" + type + + ", environment='" + environment + '\'' + + ", type='" + type + '\'' + ", wantedDockerImage='" + wantedDockerImage + '\'' + ", currentDockerImage='" + currentDockerImage + '\'' + ", parentHostname='" + parentHostname + '\'' + diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTrigger.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTrigger.java index 7ceeda08d3a..d83f552ab25 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTrigger.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTrigger.java @@ -228,10 +228,9 @@ public class DeploymentTrigger { Instance instance = application.require(applicationId.instance()); JobId job = new JobId(instance.id(), jobType); JobStatus jobStatus = jobs.jobStatus(new JobId(applicationId, jobType)); - Versions versions = jobStatus.lastTriggered() - .orElseThrow(() -> new IllegalArgumentException(job + " has never been triggered")) - .versions(); - trigger(deploymentJob(instance, versions, jobType, jobStatus, clock.instant()), reason); + Run last = jobStatus.lastTriggered() + .orElseThrow(() -> new IllegalArgumentException(job + " has never been triggered")); + trigger(deploymentJob(instance, last.versions(), last.id().type(), jobStatus.isNodeAllocationFailure(), clock.instant()), reason); return job; } @@ -259,7 +258,12 @@ public class DeploymentTrigger { .collect(toMap(Map.Entry::getKey, Map.Entry::getValue)); jobs.forEach((jobId, versionsList) -> { - trigger(deploymentJob(application.require(job.application().instance()), versionsList.get(0).versions(), jobId.type(), status.jobs().get(jobId).get(), clock.instant()), reason); + trigger(deploymentJob(application.require(job.application().instance()), + versionsList.get(0).versions(), + jobId.type(), + status.jobs().get(jobId).get().isNodeAllocationFailure(), + clock.instant()), + reason); }); return List.copyOf(jobs.keySet()); } @@ -388,7 +392,7 @@ public class DeploymentTrigger { jobs.add(deploymentJob(status.application().require(jobId.application().instance()), job.versions(), job.type(), - status.instanceJobs(jobId.application().instance()).get(jobId.type()), + status.instanceJobs(jobId.application().instance()).get(jobId.type()).isNodeAllocationFailure(), job.readyAt().get())); }); return Collections.unmodifiableList(jobs); @@ -475,8 +479,8 @@ public class DeploymentTrigger { // ---------- Version and job helpers ---------- - private Job deploymentJob(Instance instance, Versions versions, JobType jobType, JobStatus jobStatus, Instant availableSince) { - return new Job(instance, versions, jobType, availableSince, jobStatus.isNodeAllocationFailure(), instance.change().revision().isPresent()); + private Job deploymentJob(Instance instance, Versions versions, JobType jobType, boolean isNodeAllocationFailure, Instant availableSince) { + return new Job(instance, versions, jobType, availableSince, isNodeAllocationFailure, instance.change().revision().isPresent()); } // ---------- Data containers ---------- diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java index 19b2afb3af9..881107fa0f9 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java @@ -158,7 +158,7 @@ public class JobController { /** Stores the given log entries for the given run and step. */ public void log(RunId id, Step step, List<LogEntry> entries) { locked(id, __ -> { - logs.append(id.application(), id.type(), step, entries); + logs.append(id.application(), id.type(), step, entries, true); return __; }); } @@ -211,7 +211,7 @@ public class JobController { if (log.isEmpty()) return run; - logs.append(id.application(), id.type(), Step.copyVespaLogs, log); + logs.append(id.application(), id.type(), Step.copyVespaLogs, log, false); return run.with(log.get(log.size() - 1).at()); }); } @@ -230,7 +230,7 @@ public class JobController { if (entries.isEmpty()) return run; - logs.append(id.application(), id.type(), step.get(), entries); + logs.append(id.application(), id.type(), step.get(), entries, false); return run.with(entries.stream().mapToLong(LogEntry::id).max().getAsLong()); }); } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintenance.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintenance.java index 4aeecdcd4ff..9793cded918 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintenance.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintenance.java @@ -70,7 +70,7 @@ public class ControllerMaintenance extends AbstractComponent { maintainers.add(new ArchiveAccessMaintainer(controller, metric, intervals.archiveAccessMaintainer)); maintainers.add(new TenantRoleMaintainer(controller, intervals.tenantRoleMaintainer)); maintainers.add(new ChangeRequestMaintainer(controller, intervals.changeRequestMaintainer)); - maintainers.add(new VcmrMaintainer(controller, intervals.vcmrMaintainer)); + maintainers.add(new VcmrMaintainer(controller, intervals.vcmrMaintainer, metric)); maintainers.add(new CloudTrialExpirer(controller, intervals.defaultInterval)); maintainers.add(new RetriggerMaintainer(controller, intervals.retriggerMaintainer)); maintainers.add(new UserManagementMaintainer(controller, intervals.userManagementMaintainer, controller.serviceRegistry().roleMaintainer())); diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/OsUpgradeScheduler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/OsUpgradeScheduler.java index 3bd1c7bb358..5e4d6d71ff6 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/OsUpgradeScheduler.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/OsUpgradeScheduler.java @@ -9,8 +9,10 @@ import com.yahoo.vespa.hosted.controller.api.integration.deployment.ArtifactRepo import com.yahoo.vespa.hosted.controller.api.integration.deployment.OsRelease; import com.yahoo.vespa.hosted.controller.versions.OsVersionTarget; +import java.time.DayOfWeek; import java.time.Duration; import java.time.Instant; +import java.time.LocalDate; import java.time.ZoneOffset; import java.time.format.DateTimeFormatter; import java.util.Objects; @@ -84,14 +86,11 @@ public class OsUpgradeScheduler extends ControllerMaintainer { } /** OS release based on a tag */ - private static class TaggedRelease implements Release { + private record TaggedRelease(SystemName system, ArtifactRepository artifactRepository) implements Release { - private final SystemName system; - private final ArtifactRepository artifactRepository; - - private TaggedRelease(SystemName system, ArtifactRepository artifactRepository) { - this.system = Objects.requireNonNull(system); - this.artifactRepository = Objects.requireNonNull(artifactRepository); + public TaggedRelease { + Objects.requireNonNull(system); + Objects.requireNonNull(artifactRepository); } @Override @@ -119,41 +118,30 @@ public class OsUpgradeScheduler extends ControllerMaintainer { } /** OS release based on calendar-versioning */ - private static class CalendarVersionedRelease implements Release { + record CalendarVersionedRelease(SystemName system) implements Release { - /** The time to wait before scheduling upgrade to next version */ - private static final Duration SCHEDULING_INTERVAL = Duration.ofDays(45); + /** A fixed point in time which the release schedule is calculated from */ + private static final Instant START_OF_SCHEDULE = LocalDate.of(2022, 1, 1) + .atStartOfDay() + .toInstant(ZoneOffset.UTC); - /** - * The interval at which new versions become available. We use this to avoid scheduling upgrades to a version - * that has not been released yet. Example: Version N is the latest one and target is set to N+1. If N+1 does - * not exist the zone will not converge until N+1 has been released and we may end up triggering multiple - * rounds of upgrades. - */ - private static final Duration AVAILABILITY_INTERVAL = Duration.ofDays(7); + /** The time that should elapse between versions */ + private static final Duration SCHEDULING_STEP = Duration.ofDays(45); - private static final DateTimeFormatter CALENDAR_VERSION_PATTERN = DateTimeFormatter.ofPattern("yyyyMMdd"); + /** The day of week new releases are published */ + private static final DayOfWeek RELEASE_DAY = DayOfWeek.MONDAY; - private final SystemName system; + private static final DateTimeFormatter CALENDAR_VERSION_PATTERN = DateTimeFormatter.ofPattern("yyyyMMdd"); - public CalendarVersionedRelease(SystemName system) { - this.system = Objects.requireNonNull(system); + public CalendarVersionedRelease { + Objects.requireNonNull(system); } @Override public Version version(OsVersionTarget currentTarget, Instant now) { - Instant scheduledAt = currentTarget.scheduledAt(); Version currentVersion = currentTarget.osVersion().version(); - if (scheduledAt.isBefore(now.minus(SCHEDULING_INTERVAL))) { - String calendarVersion = now.minus(AVAILABILITY_INTERVAL) - .atZone(ZoneOffset.UTC) - .format(CALENDAR_VERSION_PATTERN); - return new Version(currentVersion.getMajor(), - currentVersion.getMinor(), - currentVersion.getMicro(), - calendarVersion); - } - return currentVersion; // New version should not be scheduled yet + Version wantedVersion = asVersion(dateOfWantedVersion(now), currentVersion); + return wantedVersion.isAfter(currentVersion) ? wantedVersion : currentVersion; } @Override @@ -161,6 +149,32 @@ public class OsUpgradeScheduler extends ControllerMaintainer { return system.isCd() ? Duration.ZERO : Duration.ofDays(14); } + /** + * Calculate the date of the wanted version relative to now. A given zone will choose the oldest release + * available which is not older than this date. + */ + static LocalDate dateOfWantedVersion(Instant now) { + Instant candidate = START_OF_SCHEDULE; + while (!candidate.plus(SCHEDULING_STEP).isAfter(now)) { + candidate = candidate.plus(SCHEDULING_STEP); + } + LocalDate date = LocalDate.ofInstant(candidate, ZoneOffset.UTC); + return releaseDayOf(date); + } + + private static LocalDate releaseDayOf(LocalDate date) { + int releaseDayDelta = RELEASE_DAY.getValue() - date.getDayOfWeek().getValue(); + return date.plusDays(releaseDayDelta); + } + + private static Version asVersion(LocalDate dateOfVersion, Version currentVersion) { + String calendarVersion = dateOfVersion.format(CALENDAR_VERSION_PATTERN); + return new Version(currentVersion.getMajor(), + currentVersion.getMinor(), + currentVersion.getMicro(), + calendarVersion); + } + } } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/OsUpgrader.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/OsUpgrader.java index 0c146179f34..8155476f139 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/OsUpgrader.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/OsUpgrader.java @@ -62,7 +62,7 @@ public class OsUpgrader extends InfrastructureUpgrader<OsVersionTarget> { protected boolean expectUpgradeOf(Node node, SystemApplication application, ZoneApi zone) { return cloud.equals(zone.getCloudName()) && // Cloud is managed by this upgrader application.shouldUpgradeOs() && // Application should upgrade in this cloud - canUpgrade(node); // Node is in an upgradable state + canUpgrade(node); } @Override @@ -101,9 +101,9 @@ public class OsUpgrader extends InfrastructureUpgrader<OsVersionTarget> { return !controller().zoneRegistry().systemZone().getVirtualId().equals(zone.getVirtualId()); // Do not spend budget on controller zone } - /** Returns whether node is in a state where it can be upgraded */ + /** Returns whether node currently allows upgrades */ public static boolean canUpgrade(Node node) { - return upgradableNodeStates.contains(node.state()); + return !node.deferOsUpgrade() && upgradableNodeStates.contains(node.state()); } private static String name(CloudName cloud) { diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/VcmrMaintainer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/VcmrMaintainer.java index 551f803f368..daba7e74f34 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/VcmrMaintainer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/VcmrMaintainer.java @@ -5,6 +5,7 @@ import com.yahoo.config.provision.Environment; import com.yahoo.config.provision.NodeType; import com.yahoo.config.provision.SystemName; import com.yahoo.config.provision.zone.ZoneId; +import com.yahoo.jdisc.Metric; import com.yahoo.text.Text; import com.yahoo.vespa.hosted.controller.Controller; import com.yahoo.vespa.hosted.controller.api.integration.configserver.Node; @@ -46,26 +47,28 @@ public class VcmrMaintainer extends ControllerMaintainer { private static final Logger LOG = Logger.getLogger(VcmrMaintainer.class.getName()); private static final int DAYS_TO_RETIRE = 2; private static final Duration ALLOWED_POSTPONEMENT_TIME = Duration.ofDays(7); + protected static final String TRACKED_CMRS_METRIC = "cmr.tracked"; private final CuratorDb curator; private final NodeRepository nodeRepository; private final ChangeRequestClient changeRequestClient; private final SystemName system; + private final Metric metric; - public VcmrMaintainer(Controller controller, Duration interval) { + public VcmrMaintainer(Controller controller, Duration interval, Metric metric) { super(controller, interval, null, SystemName.allOf(Predicate.not(SystemName::isPublic))); this.curator = controller.curator(); this.nodeRepository = controller.serviceRegistry().configServer().nodeRepository(); this.changeRequestClient = controller.serviceRegistry().changeRequestClient(); this.system = controller.system(); + this.metric = metric; } @Override protected double maintain() { var changeRequests = curator.readChangeRequests() .stream() - .filter(shouldUpdate()) - .collect(Collectors.toList()); + .filter(shouldUpdate()).toList(); var nodesByZone = nodesByZone(); @@ -86,6 +89,7 @@ public class VcmrMaintainer extends ControllerMaintainer { }); } }); + updateMetrics(); return 1.0; } @@ -357,4 +361,15 @@ public class VcmrMaintainer extends ControllerMaintainer { return time; } + private void updateMetrics() { + var cmrsByStatus = curator.readChangeRequests() + .stream() + .collect(Collectors.groupingBy(VespaChangeRequest::getStatus)); + + for (var status : Status.values()) { + var count = cmrsByStatus.getOrDefault(status, List.of()).size(); + metric.set(TRACKED_CMRS_METRIC, count, metric.createContext(Map.of("status", status.name()))); + } + } + } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/BufferedLogStore.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/BufferedLogStore.java index 9721026c628..ecb9db8195f 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/BufferedLogStore.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/BufferedLogStore.java @@ -49,7 +49,7 @@ public class BufferedLogStore { } /** Appends to the log of the given, active run, reassigning IDs as counted here, and converting to Vespa log levels. */ - public void append(ApplicationId id, JobType type, Step step, List<LogEntry> entries) { + public void append(ApplicationId id, JobType type, Step step, List<LogEntry> entries, boolean forceLog) { if (entries.isEmpty()) return; @@ -58,7 +58,7 @@ public class BufferedLogStore { long lastEntryId = buffer.readLastLogEntryId(id, type).orElse(-1L); long lastChunkId = buffer.getLogChunkIds(id, type).max().orElse(0); long numberOfChunks = Math.max(1, buffer.getLogChunkIds(id, type).count()); - if (numberOfChunks > maxLogSize / chunkSize) + if (numberOfChunks > maxLogSize / chunkSize && ! forceLog) return; // Max size exceeded — store no more. byte[] emptyChunk = "[]".getBytes(); @@ -72,8 +72,12 @@ public class BufferedLogStore { buffer.writeLastLogEntryId(id, type, lastEntryId); buffer.writeLog(id, type, lastChunkId, logSerializer.toJson(log)); lastChunkId = lastEntryId + 1; - if (++numberOfChunks > maxLogSize / chunkSize) { - log = Map.of(step, List.of(new LogEntry(++lastEntryId, entry.at(), LogEntry.Type.warning, "Max log size of " + (maxLogSize >> 20) + "Mb exceeded; further entries are discarded."))); + if (++numberOfChunks > maxLogSize / chunkSize && ! forceLog) { + log = Map.of(step, List.of(new LogEntry(++lastEntryId, + entry.at(), + LogEntry.Type.warning, + "Max log size of " + (maxLogSize >> 20) + + "Mb exceeded; further user entries are discarded."))); break; } log = new HashMap<>(); diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java index cfb00db7b63..56eaf2f3a2e 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java @@ -75,6 +75,7 @@ import com.yahoo.vespa.hosted.controller.api.integration.deployment.RunId; import com.yahoo.vespa.hosted.controller.api.integration.deployment.SourceRevision; import com.yahoo.vespa.hosted.controller.api.integration.noderepository.RestartFilter; import com.yahoo.vespa.hosted.controller.api.integration.secrets.TenantSecretStore; +import com.yahoo.vespa.hosted.controller.api.integration.zone.ZoneRegistry; import com.yahoo.vespa.hosted.controller.api.role.Role; import com.yahoo.vespa.hosted.controller.api.role.RoleDefinition; import com.yahoo.vespa.hosted.controller.api.role.SecurityContext; @@ -1473,6 +1474,15 @@ public class ApplicationApiHandler extends AuditLoggingRequestHandler { private HttpResponse trigger(ApplicationId id, JobType type, HttpRequest request) { + // JobType.fromJobName doesn't properly initiate test jobs. Triggering these without context isn't _really_ + // necessary, but triggering a test in the default cloud is better than failing with a weird error. + ZoneRegistry zones = controller.zoneRegistry(); + type = switch (type.environment()) { + case test -> JobType.systemTest(zones, zones.systemZone().getCloudName()); + case staging -> JobType.stagingTest(zones, zones.systemZone().getCloudName()); + default -> type; + }; + Inspector requestObject = toSlime(request.getData()).get(); boolean requireTests = ! requestObject.field("skipTests").asBool(); boolean reTrigger = requestObject.field("reTrigger").asBool(); diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/JobControllerApiHandlerHelper.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/JobControllerApiHandlerHelper.java index 9c016eccd27..25953c16bf0 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/JobControllerApiHandlerHelper.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/JobControllerApiHandlerHelper.java @@ -115,6 +115,7 @@ class JobControllerApiHandlerHelper { Run run = jobController.run(runId); detailsObject.setBool("active", ! run.hasEnded()); detailsObject.setString("status", nameOf(run.status())); + run.reason().ifPresent(reason -> detailsObject.setString("reason", reason)); try { jobController.updateTestLog(runId); jobController.updateVespaLog(runId); @@ -421,6 +422,7 @@ class JobControllerApiHandlerHelper { runObject.setLong("start", run.start().toEpochMilli()); run.end().ifPresent(end -> runObject.setLong("end", end.toEpochMilli())); runObject.setString("status", run.status().name()); + run.reason().ifPresent(reason -> runObject.setString("reason", reason)); toSlime(runObject.setObject("versions"), run.versions(), application); Cursor runStepsArray = runObject.setArray("steps"); run.steps().forEach((step, info) -> { diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTriggerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTriggerTest.java index 673cbf9708b..08fc6df37fb 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTriggerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTriggerTest.java @@ -2153,6 +2153,7 @@ public class DeploymentTriggerTest { Version version2 = new Version("8"); tester.controllerTester().flagSource().withListFlag(PermanentFlags.INCOMPATIBLE_VERSIONS.id(), List.of("8"), String.class); + // App deploys on version1. tester.controllerTester().upgradeSystem(version1); DeploymentContext app = tester.newDeploymentContext() .submit(new ApplicationPackageBuilder().region("us-east-3") @@ -2160,10 +2161,12 @@ public class DeploymentTriggerTest { .build()) .deploy(); + // System upgrades to version2, but the app is not upgraded. tester.controllerTester().upgradeSystem(version2); tester.upgrader().run(); assertEquals(Change.empty(), app.instance().change()); + // App compiles against version2, and upgrades. app.submit(new ApplicationPackageBuilder().region("us-east-3") .compileVersion(version2) .build()); @@ -2171,6 +2174,18 @@ public class DeploymentTriggerTest { assertEquals(version2, tester.jobs().last(app.instanceId(), productionUsEast3).get().versions().targetPlatform()); assertEquals(version2, app.application().revisions().get(tester.jobs().last(app.instanceId(), productionUsEast3).get().versions().targetRevision()).compileVersion().get()); + + // App specifies version1 in deployment spec, compiles against version1, pins to version1, and then downgrades. + app.submit(new ApplicationPackageBuilder().region("us-east-3") + .majorVersion(7) + .compileVersion(version1) + .build()); + tester.deploymentTrigger().forceChange(app.instanceId(), app.instance().change().withPin()); + app.deploy(); + assertEquals(version1, tester.jobs().last(app.instanceId(), productionUsEast3).get().versions().targetPlatform()); + assertEquals(version1, app.application().revisions().get(tester.jobs().last(app.instanceId(), productionUsEast3).get().versions().targetRevision()).compileVersion().get()); + + // A new app, compiled against version1, is deployed on version1. DeploymentContext newApp = tester.newDeploymentContext("new", "app", "default") .submit(new ApplicationPackageBuilder().region("us-east-3") .compileVersion(version1) diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/OsUpgradeSchedulerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/OsUpgradeSchedulerTest.java index 300aa86b5ea..478bb943eba 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/OsUpgradeSchedulerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/OsUpgradeSchedulerTest.java @@ -13,7 +13,10 @@ import org.junit.Test; import java.time.Duration; import java.time.Instant; +import java.time.LocalDate; +import java.time.ZoneOffset; import java.util.List; +import java.util.Map; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; @@ -27,7 +30,7 @@ public class OsUpgradeSchedulerTest { public void schedule_calendar_versioned_release() { ControllerTester tester = new ControllerTester(); OsUpgradeScheduler scheduler = new OsUpgradeScheduler(tester.controller(), Duration.ofDays(1)); - Instant t0 = Instant.parse("2021-01-23T00:00:00.00Z"); // Outside trigger period + Instant t0 = Instant.parse("2022-01-16T00:00:00.00Z"); // Outside trigger period tester.clock().setInstant(t0); CloudName cloud = CloudName.from("cloud"); @@ -38,8 +41,8 @@ public class OsUpgradeSchedulerTest { scheduler.maintain(); assertTrue("No target set", tester.controller().osVersionTarget(cloud).isEmpty()); - // Target is set - Version version0 = Version.fromString("7.0.0.20210123190005"); + // Target is set manually + Version version0 = Version.fromString("7.0.0.20220101"); tester.controller().upgradeOsIn(cloud, version0, Duration.ofDays(1), false); // Target remains unchanged as it hasn't expired yet @@ -49,8 +52,8 @@ public class OsUpgradeSchedulerTest { assertEquals(version0, tester.controller().osVersionTarget(cloud).get().osVersion().version()); } - // Just over 45 days pass, and a new target replaces the expired one - Version version1 = Version.fromString("7.0.0.20210302"); + // Enough days pass that the next release is triggered + Version version1 = Version.fromString("7.0.0.20220214"); tester.clock().advance(Duration.ofDays(15).plus(Duration.ofSeconds(1))); scheduler.maintain(); assertEquals("Target is unchanged because we're outside trigger period", version0, @@ -60,7 +63,7 @@ public class OsUpgradeSchedulerTest { assertEquals("New target set", version1, tester.controller().osVersionTarget(cloud).get().osVersion().version()); - // A few days pass and target remains unchanged + // A few more days pass and target remains unchanged tester.clock().advance(Duration.ofDays(2)); scheduler.maintain(); assertEquals(version1, tester.controller().osVersionTarget(cloud).get().osVersion().version()); @@ -112,6 +115,24 @@ public class OsUpgradeSchedulerTest { scheduleUpgradeAfter(Duration.ofDays(1), version1, tester); } + @Test + public void schedule_of_calender_versioned_releases() { + Map<String, String> tests = Map.of("2022-01-01", "2021-12-27", + "2022-02-14", "2021-12-27", + "2022-02-15", "2022-02-14", + "2022-03-31", "2022-02-14", + "2022-04-01", "2022-03-28", + "2022-05-15", "2022-03-28", + "2022-05-16", "2022-05-16", + "2022-06-29", "2022-05-16", + "2022-06-30", "2022-06-27"); + tests.forEach((now, expected) -> { + Instant instant = LocalDate.parse(now).atStartOfDay().toInstant(ZoneOffset.UTC); + LocalDate dateOfWantedVersion = OsUpgradeScheduler.CalendarVersionedRelease.dateOfWantedVersion(instant); + assertEquals("scheduled wanted version at " + now, LocalDate.parse(expected), dateOfWantedVersion); + }); + } + private void scheduleUpgradeAfter(Duration duration, Version version, ControllerTester tester) { tester.clock().advance(duration); new OsUpgradeScheduler(tester.controller(), Duration.ofDays(1)).maintain(); diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/OsUpgraderTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/OsUpgraderTest.java index 8adf55155a5..3a5b4a90baa 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/OsUpgraderTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/OsUpgraderTest.java @@ -19,7 +19,9 @@ import java.time.Duration; import java.util.Collection; import java.util.List; import java.util.function.Function; +import java.util.function.UnaryOperator; import java.util.stream.Collectors; +import java.util.stream.Stream; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; @@ -52,18 +54,19 @@ public class OsUpgraderTest { OsUpgrader osUpgrader = osUpgrader(upgradePolicy, cloud1, false); // Bootstrap system - List<ZoneId> nonControllerZones = List.of(zone1, zone2, zone3, zone4, zone5).stream() - .map(ZoneApi::getVirtualId) - .collect(Collectors.toList()); + List<ZoneId> nonControllerZones = Stream.of(zone1, zone2, zone3, zone4, zone5) + .map(ZoneApi::getVirtualId) + .collect(Collectors.toList()); tester.configServer().bootstrap(nonControllerZones, List.of(SystemApplication.tenantHost)); tester.configServer().addNodes(List.of(zone0.getVirtualId()), List.of(SystemApplication.controllerHost)); // Add system application that exists in a real system, but isn't eligible for OS upgrades tester.configServer().addNodes(nonControllerZones, List.of(SystemApplication.configServer)); - // Fail a few nodes. Failed nodes should not affect versions + // Change state of a few nodes. These should not affect convergence failNodeIn(zone1, SystemApplication.tenantHost); failNodeIn(zone3, SystemApplication.tenantHost); + Node nodeDeferringOsUpgrade = deferOsUpgradeIn(zone2, SystemApplication.tenantHost); // New OS version released Version version1 = Version.fromString("7.1"); @@ -91,7 +94,7 @@ public class OsUpgraderTest { completeUpgrade(version1, SystemApplication.tenantHost, zone1); statusUpdater.maintain(); assertEquals(5, nodesOn(version1).size()); - assertEquals(11, nodesOn(Version.emptyVersion).size()); + assertEquals(10, nodesOn(Version.emptyVersion).size()); // zone 2 and 3: begins upgrading osUpgrader.maintain(); @@ -102,6 +105,10 @@ public class OsUpgraderTest { // zone 2 and 3: completes upgrade completeUpgrade(version1, SystemApplication.tenantHost, zone2, zone3); + assertEquals("Current version is unchanged for node deferring OS upgrade", Version.emptyVersion, + nodeRepository().list(zone2.getVirtualId(), NodeFilter.all().hostnames(nodeDeferringOsUpgrade.hostname())) + .get(0) + .currentOsVersion()); // zone 4: begins upgrading osUpgrader.maintain(); @@ -271,13 +278,23 @@ public class OsUpgraderTest { .collect(Collectors.toList()); } - private void failNodeIn(ZoneApi zone, SystemApplication application) { + private Node failNodeIn(ZoneApi zone, SystemApplication application) { + return patchOneNodeIn(zone, application, (node) -> Node.builder(node).state(Node.State.failed).build()); + } + + private Node deferOsUpgradeIn(ZoneApi zone, SystemApplication application) { + return patchOneNodeIn(zone, application, (node) -> Node.builder(node).deferOsUpgrade(true).build()); + } + + private Node patchOneNodeIn(ZoneApi zone, SystemApplication application, UnaryOperator<Node> patcher) { List<Node> nodes = nodeRepository().list(zone.getVirtualId(), NodeFilter.all().applications(application.id())); if (nodes.isEmpty()) { throw new IllegalArgumentException("No nodes allocated to " + application.id()); } Node node = nodes.get(0); - nodeRepository().putNodes(zone.getVirtualId(), Node.builder(node).state(Node.State.failed).build()); + Node newNode = patcher.apply(node); + nodeRepository().putNodes(zone.getVirtualId(), newNode); + return newNode; } /** Simulate OS upgrade of nodes allocated to application. In a real system this is done by the node itself */ diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/VcmrMaintainerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/VcmrMaintainerTest.java index bfbd3836ce7..321ec3ad8ea 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/VcmrMaintainerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/VcmrMaintainerTest.java @@ -14,6 +14,7 @@ import com.yahoo.vespa.hosted.controller.api.integration.vcmr.HostAction.State; import com.yahoo.vespa.hosted.controller.api.integration.vcmr.VcmrReport; import com.yahoo.vespa.hosted.controller.api.integration.vcmr.VespaChangeRequest; import com.yahoo.vespa.hosted.controller.api.integration.vcmr.VespaChangeRequest.Status; +import com.yahoo.vespa.hosted.controller.integration.MetricsMock; import com.yahoo.vespa.hosted.controller.integration.NodeRepositoryMock; import org.junit.Before; import org.junit.Test; @@ -25,6 +26,7 @@ import java.time.ZonedDateTime; import java.time.temporal.TemporalAdjusters; import java.util.List; +import static com.yahoo.vespa.hosted.controller.maintenance.VcmrMaintainer.TRACKED_CMRS_METRIC; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; @@ -37,6 +39,7 @@ public class VcmrMaintainerTest { private ControllerTester tester; private VcmrMaintainer maintainer; private NodeRepositoryMock nodeRepo; + private MetricsMock metrics; private final ZoneId zoneId = ZoneId.from("prod.us-east-3"); private final ZoneId zone2 = ZoneId.from("prod.us-west-1"); private final HostName host1 = HostName.of("host1"); @@ -47,7 +50,8 @@ public class VcmrMaintainerTest { @Before public void setup() { tester = new ControllerTester(); - maintainer = new VcmrMaintainer(tester.controller(), Duration.ofMinutes(1)); + metrics = new MetricsMock(); + maintainer = new VcmrMaintainer(tester.controller(), Duration.ofMinutes(1), metrics); nodeRepo = tester.serviceRegistry().configServer().nodeRepository().allowPatching(true); } @@ -244,6 +248,8 @@ public class VcmrMaintainerTest { assertEquals(State.OUT_OF_SYNC, action.getState()); assertEquals(Status.OUT_OF_SYNC, writtenChangeRequest.getStatus()); + assertEquals(1, metrics.getMetric(context -> "OUT_OF_SYNC".equals(context.get("status")), TRACKED_CMRS_METRIC).get()); + assertEquals(0, metrics.getMetric(context -> "REQUIRES_OPERATOR_ACTION".equals(context.get("status")), TRACKED_CMRS_METRIC).get()); } @Test diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/BufferedLogStoreTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/BufferedLogStoreTest.java index 0f7f97d333a..87280c0c1a3 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/BufferedLogStoreTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/BufferedLogStoreTest.java @@ -50,19 +50,19 @@ public class BufferedLogStoreTest { assertEquals(Optional.empty(), logs.readFinished(id, -1)); assertEquals(RunLog.empty(), logs.readActive(id.application(), id.type(), -1)); - logs.append(id.application(), id.type(), Step.deployReal, List.of(entry)); + logs.append(id.application(), id.type(), Step.deployReal, List.of(entry), false); assertEquals(List.of(entry0), logs.readActive(id.application(), id.type(), -1).get(Step.deployReal)); assertEquals(RunLog.empty(), logs.readActive(id.application(), id.type(), 0)); - logs.append(id.application(), id.type(), Step.deployReal, List.of(entry)); + logs.append(id.application(), id.type(), Step.deployReal, List.of(entry), false); assertEquals(List.of(entry0, entry1), logs.readActive(id.application(), id.type(), -1).get(Step.deployReal)); assertEquals(List.of(entry1), logs.readActive(id.application(), id.type(), 0).get(Step.deployReal)); assertEquals(RunLog.empty(), logs.readActive(id.application(), id.type(), 1)); - logs.append(id.application(), id.type(), Step.deployReal, List.of(entry, entry, entry)); + logs.append(id.application(), id.type(), Step.deployReal, List.of(entry, entry, entry), false); assertEquals(List.of(entry0, entry1, entry2, entry3, entry4), logs.readActive(id.application(), id.type(), -1).get(Step.deployReal)); assertEquals(List.of(entry1, entry2, entry3, entry4), @@ -105,17 +105,28 @@ public class BufferedLogStoreTest { logged.remove(logged.size() - 1); logged.remove(logged.size() - 1); logged.remove(logged.size() - 1); - logged.add(new LogEntry(2 * maxChunks, entry.at(), LogEntry.Type.warning, "Max log size of " + ((chunkSize * maxChunks) >> 20) + "Mb exceeded; further entries are discarded.")); + logged.add(new LogEntry(2 * maxChunks, entry.at(), LogEntry.Type.warning, "Max log size of " + ((chunkSize * maxChunks) >> 20) + "Mb exceeded; further user entries are discarded.")); - logs.append(id.application(), id.type(), Step.deployReal, monsterLog); + logs.append(id.application(), id.type(), Step.deployReal, monsterLog, false); assertEquals(logged.size(), logs.readActive(id.application(), id.type(), -1).get(Step.deployReal).size()); assertEquals(logged, logs.readActive(id.application(), id.type(), -1).get(Step.deployReal)); + // An additional, forced entry is appended. + LogEntry forced = new LogEntry(logged.size(), entry.at(), entry.type(), entry.message()); + logs.append(id.application(), id.type(), Step.deployReal, List.of(forced), true); + logged.add(forced); + assertEquals(logged.size(), + logs.readActive(id.application(), id.type(), -1).get(Step.deployReal).size()); + assertEquals(logged, + logs.readActive(id.application(), id.type(), -1).get(Step.deployReal)); + logged.remove(logged.size() - 1); + + // Flushing the buffer clears it again, and makes it ready for reuse. logs.flush(id); for (int i = 0; i < 2 * maxChunks + 3; i++) - logs.append(id.application(), id.type(), Step.deployReal, List.of(entry)); + logs.append(id.application(), id.type(), Step.deployReal, List.of(entry), false); assertEquals(logged.size(), logs.readActive(id.application(), id.type(), -1).get(Step.deployReal).size()); assertEquals(logged, diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/deployment-overview.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/deployment-overview.json index 696f1ef0ba3..38e9d8c823e 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/deployment-overview.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/deployment-overview.json @@ -428,6 +428,7 @@ "url": "http://localhost:8080/application/v4/tenant/tenant1/application/application1/instance/instance1/job/production-us-west-1/run/1", "start": 1600000000000, "status": "running", + "reason": "triggered by user.myuser", "versions": { "targetPlatform": "6.1.0", "targetApplication": { diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/jobs.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/jobs.json index 12430b67539..2477e8df56e 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/jobs.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/jobs.json @@ -9,6 +9,7 @@ "start": 1600000000000, "end": 1600000000000, "status": "success", + "reason": "triggered by user.myuser", "versions": { "targetPlatform": "6.1.0", "targetApplication": { diff --git a/docprocs/src/test/java/com/yahoo/docprocs/indexing/IndexingProcessorTestCase.java b/docprocs/src/test/java/com/yahoo/docprocs/indexing/IndexingProcessorTestCase.java index 76f4578ac87..c6ba63b7924 100644 --- a/docprocs/src/test/java/com/yahoo/docprocs/indexing/IndexingProcessorTestCase.java +++ b/docprocs/src/test/java/com/yahoo/docprocs/indexing/IndexingProcessorTestCase.java @@ -125,6 +125,7 @@ public class IndexingProcessorTestCase { return lst.get(0); } + @SuppressWarnings("deprecation") private static IndexingProcessor newProcessor(String configId) { return new IndexingProcessor(new DocumentTypeManager(ConfigGetter.getConfig(DocumentmanagerConfig.class, configId)), ConfigGetter.getConfig(IlscriptsConfig.class, configId), diff --git a/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/FileDownloader.java b/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/FileDownloader.java index dc87ae2a0b4..63ae8faacfe 100644 --- a/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/FileDownloader.java +++ b/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/FileDownloader.java @@ -31,6 +31,7 @@ public class FileDownloader implements AutoCloseable { private static final Logger log = Logger.getLogger(FileDownloader.class.getName()); private static final Duration defaultSleepBetweenRetries = Duration.ofSeconds(5); public static final File defaultDownloadDirectory = new File(Defaults.getDefaults().underVespaHome("var/db/vespa/filedistribution")); + private static final boolean forceDownload = Boolean.parseBoolean(System.getenv("VESPA_CONFIG_PROXY_FORCE_DOWNLOAD_OF_FILE_REFERENCES")); private final ConnectionPool connectionPool; private final Supervisor supervisor; @@ -65,6 +66,8 @@ public class FileDownloader implements AutoCloseable { sleepBetweenRetries, downloadDirectory, acceptedCompressionTypes); + if (forceDownload) + log.log(Level.INFO, "Force download of file references (download even if file reference exists on disk)"); } public Optional<File> getFile(FileReferenceDownload fileReferenceDownload) { @@ -103,6 +106,8 @@ public class FileDownloader implements AutoCloseable { } private static Optional<File> getFileFromFileSystem(FileReference fileReference, File downloadDirectory) { + if (forceDownload) return Optional.empty(); + File[] files = new File(downloadDirectory, fileReference.value()).listFiles(); if (files == null) return Optional.empty(); if (files.length == 0) return Optional.empty(); diff --git a/flags/src/main/java/com/yahoo/vespa/flags/Flags.java b/flags/src/main/java/com/yahoo/vespa/flags/Flags.java index 00ca552e2c8..04c7dcf0595 100644 --- a/flags/src/main/java/com/yahoo/vespa/flags/Flags.java +++ b/flags/src/main/java/com/yahoo/vespa/flags/Flags.java @@ -393,9 +393,9 @@ public class Flags { ZONE_ID, APPLICATION_ID); public static final UnboundStringFlag APPLICATION_FILES_WITH_UNKNOWN_EXTENSION = defineStringFlag( - "fail-deployment-for-files-with-unknown-extension", "NOOP", + "fail-deployment-for-files-with-unknown-extension", "LOG", List.of("hmusum"), "2022-04-27", "2022-07-27", - "Whether to log, fail or do nothing for deployments when app has a file with unknown extension (valid values LOG, FAIL, NOOP)", + "Whether to log, fail or do nothing for deployments when app has a file with unknown extension (valid values LOG, FAIL, NOOP)", "Takes effect at redeployment", ZONE_ID, APPLICATION_ID); @@ -420,13 +420,6 @@ public class Flags { "Takes effect on redeployment", APPLICATION_ID); - public static final UnboundStringFlag FILE_DISTRIBUTION_COMPRESSION_ALGORITHM = defineStringFlag( - "file-distribution-compression-algorithm", "gzip", - List.of("hmusum"), "2022-05-24", "2022-07-24", - "Which algorithm to use for compressing file references when distributing files. Valid values: none, gzip", - "Takes effect immediately", - APPLICATION_ID); - public static final UnboundListFlag<String> FILE_DISTRIBUTION_ACCEPTED_COMPRESSION_TYPES = defineListFlag( "file-distribution-accepted-compression-types", List.of("gzip"), String.class, List.of("hmusum"), "2022-07-05", "2022-09-05", @@ -456,12 +449,19 @@ public class Flags { ZONE_ID, APPLICATION_ID); public static final UnboundBooleanFlag FIX_IPV6_GATEWAY = defineFeatureFlag( - "fix-ipv6-gateway", false, + "fix-ipv6-gateway", true, List.of("mpolden"), "2022-07-04", "2022-09-01", "Fix a misconfigured IPv6 gateway automatically", "Takes effect on first host admin resume", HOSTNAME); + public static final UnboundBooleanFlag SEPARATE_METRIC_CHECK_CONFIG = defineFeatureFlag( + "separate-metric-check-config", false, + List.of("olaa"), "2022-07-04", "2022-09-01", + "Determines whether one metrics config check should be written per Vespa node", + "Takes effect on next tick", + HOSTNAME); + /** WARNING: public for testing: All flags should be defined in {@link Flags}. */ public static UnboundBooleanFlag defineFeatureFlag(String flagId, boolean defaultValue, List<String> owners, String createdAt, String expiresAt, String description, diff --git a/messagebus/src/vespa/messagebus/network/rpcnetworkparams.cpp b/messagebus/src/vespa/messagebus/network/rpcnetworkparams.cpp index 76dcb81919f..fc060b48fc2 100644 --- a/messagebus/src/vespa/messagebus/network/rpcnetworkparams.cpp +++ b/messagebus/src/vespa/messagebus/network/rpcnetworkparams.cpp @@ -15,7 +15,6 @@ RPCNetworkParams::RPCNetworkParams(config::ConfigUri configUri) : _listenPort(0), _maxInputBufferSize(256_Ki), _maxOutputBufferSize(256_Ki), - _numThreads(4), _numNetworkThreads(1), _numRpcTargets(1), _events_before_wakeup(1), diff --git a/messagebus/src/vespa/messagebus/network/rpcnetworkparams.h b/messagebus/src/vespa/messagebus/network/rpcnetworkparams.h index 4837d58b42c..4a4d92ba797 100644 --- a/messagebus/src/vespa/messagebus/network/rpcnetworkparams.h +++ b/messagebus/src/vespa/messagebus/network/rpcnetworkparams.h @@ -19,7 +19,6 @@ private: int _listenPort; uint32_t _maxInputBufferSize; uint32_t _maxOutputBufferSize; - uint32_t _numThreads; uint32_t _numNetworkThreads; uint32_t _numRpcTargets; uint32_t _events_before_wakeup; @@ -111,19 +110,6 @@ public: return *this; } - /** - * Sets number of threads for the thread pool. - * - * @param numThreads number of threads for thread pool - * @return This, to allow chaining. - */ - RPCNetworkParams &setNumThreads(uint32_t numThreads) { - _numThreads = numThreads; - return *this; - } - - uint32_t getNumThreads() const { return _numThreads; } - RPCNetworkParams &setTcpNoDelay(bool tcpNoDelay) { _tcpNoDelay = tcpNoDelay; return *this; diff --git a/model-evaluation/src/test/java/ai/vespa/models/evaluation/ModelTester.java b/model-evaluation/src/test/java/ai/vespa/models/evaluation/ModelTester.java index a36215e005f..ab2f53db863 100644 --- a/model-evaluation/src/test/java/ai/vespa/models/evaluation/ModelTester.java +++ b/model-evaluation/src/test/java/ai/vespa/models/evaluation/ModelTester.java @@ -2,7 +2,6 @@ package ai.vespa.models.evaluation; import com.yahoo.config.subscription.ConfigGetter; -import com.yahoo.config.subscription.FileSource; import com.yahoo.filedistribution.fileacquirer.MockFileAcquirer; import com.yahoo.path.Path; import com.yahoo.searchlib.rankingexpression.ExpressionFunction; @@ -10,7 +9,6 @@ import com.yahoo.vespa.config.search.RankProfilesConfig; import com.yahoo.vespa.config.search.core.OnnxModelsConfig; import com.yahoo.vespa.config.search.core.RankingConstantsConfig; import com.yahoo.vespa.config.search.core.RankingExpressionsConfig; - import java.util.Map; import static org.junit.Assert.assertEquals; @@ -31,8 +29,8 @@ public class ModelTester { public Map<String, Model> models() { return models; } + @SuppressWarnings("deprecation") private static Map<String, Model> createModels(String path) { - RankProfilesConfig config = ConfigGetter.getConfig(RankProfilesConfig.class, fileConfigId(path, "rank-profiles.cfg")); RankingConstantsConfig constantsConfig = ConfigGetter.getConfig(RankingConstantsConfig.class, fileConfigId(path, "ranking-constants.cfg")); RankingExpressionsConfig expressionsConfig = ConfigGetter.getConfig(RankingExpressionsConfig.class, fileConfigId(path, "ranking-expressions.cfg")); diff --git a/model-evaluation/src/test/java/ai/vespa/models/evaluation/ModelsEvaluatorTest.java b/model-evaluation/src/test/java/ai/vespa/models/evaluation/ModelsEvaluatorTest.java index 540c534925e..c4e859bec9f 100644 --- a/model-evaluation/src/test/java/ai/vespa/models/evaluation/ModelsEvaluatorTest.java +++ b/model-evaluation/src/test/java/ai/vespa/models/evaluation/ModelsEvaluatorTest.java @@ -2,7 +2,6 @@ package ai.vespa.models.evaluation; import com.yahoo.config.subscription.ConfigGetter; -import com.yahoo.config.subscription.FileSource; import com.yahoo.filedistribution.fileacquirer.MockFileAcquirer; import com.yahoo.path.Path; import com.yahoo.searchlib.rankingexpression.ExpressionFunction; @@ -15,7 +14,6 @@ import com.yahoo.vespa.config.search.core.RankingConstantsConfig; import com.yahoo.vespa.config.search.core.RankingExpressionsConfig; import com.yahoo.yolean.Exceptions; import org.junit.Test; - import java.util.ArrayList; import java.util.List; @@ -128,6 +126,7 @@ public class ModelsEvaluatorTest { // TODO: Test argument-less function // TODO: Test with nested functions + @SuppressWarnings("deprecation") private ModelsEvaluator createModels() { RankProfilesConfig config = ConfigGetter.getConfig(RankProfilesConfig.class, fileConfigId("rank-profiles.cfg")); RankingConstantsConfig constantsConfig = ConfigGetter.getConfig(RankingConstantsConfig.class, fileConfigId("ranking-constants.cfg")); diff --git a/model-evaluation/src/test/java/ai/vespa/models/evaluation/OnnxEvaluatorTest.java b/model-evaluation/src/test/java/ai/vespa/models/evaluation/OnnxEvaluatorTest.java index 27d1c08ea39..992dae22aaf 100644 --- a/model-evaluation/src/test/java/ai/vespa/models/evaluation/OnnxEvaluatorTest.java +++ b/model-evaluation/src/test/java/ai/vespa/models/evaluation/OnnxEvaluatorTest.java @@ -51,6 +51,7 @@ public class OnnxEvaluatorTest { assertEquals(function.evaluate(), Tensor.from("tensor<float>(d0[2],d1[1]):[0.63931,0.67574]")); } + @SuppressWarnings("deprecation") private ModelsEvaluator createModels() { RankProfilesConfig config = ConfigGetter.getConfig(RankProfilesConfig.class, fileConfigId("rank-profiles.cfg")); RankingConstantsConfig constantsConfig = ConfigGetter.getConfig(RankingConstantsConfig.class, fileConfigId("ranking-constants.cfg")); diff --git a/model-evaluation/src/test/java/ai/vespa/models/handler/ModelsEvaluationHandlerTest.java b/model-evaluation/src/test/java/ai/vespa/models/handler/ModelsEvaluationHandlerTest.java index 7790f8a60d0..0de8ce5f061 100644 --- a/model-evaluation/src/test/java/ai/vespa/models/handler/ModelsEvaluationHandlerTest.java +++ b/model-evaluation/src/test/java/ai/vespa/models/handler/ModelsEvaluationHandlerTest.java @@ -257,6 +257,7 @@ public class ModelsEvaluationHandlerTest { handler.assertResponse(url, properties, 200, expected); } + @SuppressWarnings("deprecation") static private ModelsEvaluator createModels() { RankProfilesConfig config = ConfigGetter.getConfig(RankProfilesConfig.class, fileConfigId("rank-profiles.cfg")); RankingConstantsConfig constantsConfig = ConfigGetter.getConfig(RankingConstantsConfig.class, fileConfigId("ranking-constants.cfg")); diff --git a/model-evaluation/src/test/java/ai/vespa/models/handler/OnnxEvaluationHandlerTest.java b/model-evaluation/src/test/java/ai/vespa/models/handler/OnnxEvaluationHandlerTest.java index f065435ec15..8ab282668da 100644 --- a/model-evaluation/src/test/java/ai/vespa/models/handler/OnnxEvaluationHandlerTest.java +++ b/model-evaluation/src/test/java/ai/vespa/models/handler/OnnxEvaluationHandlerTest.java @@ -117,6 +117,7 @@ public class OnnxEvaluationHandlerTest { handler.assertResponse(url, properties, 200, expected); } + @SuppressWarnings("deprecation") static private ModelsEvaluator createModels() { RankProfilesConfig config = ConfigGetter.getConfig(RankProfilesConfig.class, fileConfigId("rank-profiles.cfg")); RankingConstantsConfig constantsConfig = ConfigGetter.getConfig(RankingConstantsConfig.class, fileConfigId("ranking-constants.cfg")); diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/History.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/History.java index ac804f99cd3..c2d4506a28c 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/History.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/History.java @@ -4,6 +4,7 @@ package com.yahoo.vespa.hosted.provision.node; import com.google.common.collect.ImmutableMap; import com.yahoo.vespa.hosted.provision.Node; +import java.time.Duration; import java.time.Instant; import java.util.ArrayList; import java.util.Collection; @@ -50,6 +51,12 @@ public class History { return builder.build(); } + /** Returns the age of this node as best as we can determine: The time since the first event registered for it */ + public Duration age(Instant now) { + Instant oldestEventTime = events.values().stream().map(event -> event.at()).sorted().findFirst().orElse(now); + return Duration.between(oldestEventTime, now); + } + /** Returns the last event of given type, if it is present in this history */ public Optional<Event> event(Event.Type type) { return Optional.ofNullable(events.get(type)); } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/filter/NodeListFilter.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/filter/NodeListFilter.java index 8578e3eb5ec..2b790ff7392 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/filter/NodeListFilter.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/filter/NodeListFilter.java @@ -30,4 +30,5 @@ public class NodeListFilter { public static Predicate<Node> from(List<Node> nodes) { return makePredicate(nodes); } + } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/os/DelegatingOsUpgrader.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/os/DelegatingOsUpgrader.java index 30fd2713017..4178d4a6328 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/os/DelegatingOsUpgrader.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/os/DelegatingOsUpgrader.java @@ -7,6 +7,7 @@ import com.yahoo.vespa.hosted.provision.NodeList; import com.yahoo.vespa.hosted.provision.NodeRepository; import com.yahoo.vespa.hosted.provision.node.filter.NodeListFilter; +import java.time.Instant; import java.util.Objects; import java.util.Optional; import java.util.logging.Logger; @@ -39,8 +40,10 @@ public class DelegatingOsUpgrader implements OsUpgrader { public void upgradeTo(OsVersionTarget target) { NodeList activeNodes = nodeRepository.nodes().list(Node.State.active).nodeType(target.nodeType()); int numberToUpgrade = Math.max(0, maxActiveUpgrades - activeNodes.changingOsVersionTo(target.version()).size()); + Instant now = nodeRepository.clock().instant(); NodeList nodesToUpgrade = activeNodes.not().changingOsVersionTo(target.version()) .osVersionIsBefore(target.version()) + .matching(node -> canUpgradeAt(now, node)) .byIncreasingOsVersion() .first(numberToUpgrade); if (nodesToUpgrade.size() == 0) return; diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/os/OsUpgrader.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/os/OsUpgrader.java index 5310ef339ed..4140de76368 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/os/OsUpgrader.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/os/OsUpgrader.java @@ -2,6 +2,9 @@ package com.yahoo.vespa.hosted.provision.os; import com.yahoo.config.provision.NodeType; +import com.yahoo.vespa.hosted.provision.Node; + +import java.time.Instant; /** * Interface for an OS upgrader. @@ -16,4 +19,9 @@ public interface OsUpgrader { /** Disable OS upgrade for all nodes of given type */ void disableUpgrade(NodeType type); + /** Returns whether node can upgrade at given instant */ + default boolean canUpgradeAt(Instant instant, Node node) { + return true; + } + } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/os/OsVersions.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/os/OsVersions.java index 7c6d1cb69db..440046ab818 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/os/OsVersions.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/os/OsVersions.java @@ -84,7 +84,7 @@ public class OsVersions { Version target = Optional.ofNullable(change.targets().get(nodeType)) .map(OsVersionTarget::version) .orElse(Version.emptyVersion); - chooseUpgrader(nodeType, target).disableUpgrade(nodeType); + chooseUpgrader(nodeType, Optional.of(target)).disableUpgrade(nodeType); return change.withoutTarget(nodeType); }); } @@ -120,7 +120,7 @@ public class OsVersions { try (Lock lock = db.lockOsVersionChange()) { OsVersionTarget target = readChange().targets().get(nodeType); if (target == null) return; // No target set for this type - OsUpgrader upgrader = chooseUpgrader(nodeType, target.version()); + OsUpgrader upgrader = chooseUpgrader(nodeType, Optional.of(target.version())); if (resume) { upgrader.upgradeTo(target); } else { @@ -129,17 +129,23 @@ public class OsVersions { } } + /** Returns whether node can be upgraded now */ + public boolean canUpgrade(Node node) { + return chooseUpgrader(node.type(), Optional.empty()).canUpgradeAt(nodeRepository.clock().instant(), node); + } + /** Returns the upgrader to use when upgrading given node type to target */ - private OsUpgrader chooseUpgrader(NodeType nodeType, Version target) { + private OsUpgrader chooseUpgrader(NodeType nodeType, Optional<Version> target) { if (reprovisionToUpgradeOs) { return new RetiringOsUpgrader(nodeRepository); } // Require rebuild if we have any nodes of this type on a major version lower than target - boolean rebuildRequired = nodeRepository.nodes().list(Node.State.active).nodeType(nodeType).stream() + boolean rebuildRequired = target.isPresent() && + nodeRepository.nodes().list(Node.State.active).nodeType(nodeType).stream() .map(Node::status) .map(Status::osVersion) .anyMatch(osVersion -> osVersion.current().isPresent() && - osVersion.current().get().getMajor() < target.getMajor()); + osVersion.current().get().getMajor() < target.get().getMajor()); if (rebuildRequired) { return new RebuildingOsUpgrader(nodeRepository); } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/os/RebuildingOsUpgrader.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/os/RebuildingOsUpgrader.java index efc377e6cc3..f96effe9e10 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/os/RebuildingOsUpgrader.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/os/RebuildingOsUpgrader.java @@ -47,7 +47,7 @@ public class RebuildingOsUpgrader implements OsUpgrader { public void upgradeTo(OsVersionTarget target) { NodeList allNodes = nodeRepository.nodes().list(); Instant now = nodeRepository.clock().instant(); - rebuildableHosts(target, allNodes).forEach(host -> rebuild(host, target.version(), now)); + rebuildableHosts(target, allNodes, now).forEach(host -> rebuild(host, target.version(), now)); } @Override @@ -62,7 +62,7 @@ public class RebuildingOsUpgrader implements OsUpgrader { return Math.max(0, limit - hostsOfType.rebuilding().size()); } - private List<Node> rebuildableHosts(OsVersionTarget target, NodeList allNodes) { + private List<Node> rebuildableHosts(OsVersionTarget target, NodeList allNodes, Instant now) { NodeList hostsOfTargetType = allNodes.nodeType(target.nodeType()); int rebuildLimit = rebuildLimit(target.nodeType(), hostsOfTargetType); @@ -76,6 +76,7 @@ public class RebuildingOsUpgrader implements OsUpgrader { NodeList candidates = hostsOfTargetType.state(Node.State.active) .not().rebuilding() .osVersionIsBefore(target.version()) + .matching(node -> canUpgradeAt(now, node)) .byIncreasingOsVersion(); for (Node host : candidates) { if (hostsToRebuild.size() == rebuildLimit) break; diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/os/RetiringOsUpgrader.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/os/RetiringOsUpgrader.java index d923c78a929..79b7441cc34 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/os/RetiringOsUpgrader.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/os/RetiringOsUpgrader.java @@ -26,6 +26,9 @@ public class RetiringOsUpgrader implements OsUpgrader { private static final Logger LOG = Logger.getLogger(RetiringOsUpgrader.class.getName()); + /** The duration this leaves new nodes alone before scheduling any upgrade */ + static final Duration GRACE_PERIOD = Duration.ofDays(30); + protected final NodeRepository nodeRepository; public RetiringOsUpgrader(NodeRepository nodeRepository) { @@ -33,21 +36,27 @@ public class RetiringOsUpgrader implements OsUpgrader { } @Override - public final void upgradeTo(OsVersionTarget target) { + public void upgradeTo(OsVersionTarget target) { NodeList allNodes = nodeRepository.nodes().list(); Instant now = nodeRepository.clock().instant(); NodeList candidates = candidates(now, target, allNodes); candidates.not().deprovisioning() + .matching(node -> canUpgradeAt(now, node)) .byIncreasingOsVersion() .first(1) .forEach(node -> deprovision(node, target.version(), now)); } @Override - public final void disableUpgrade(NodeType type) { + public void disableUpgrade(NodeType type) { // No action needed in this implementation. } + @Override + public boolean canUpgradeAt(Instant instant, Node node) { + return node.history().age(instant).compareTo(GRACE_PERIOD) > 0; + } + /** Returns nodes that are candidates for upgrade */ private NodeList candidates(Instant instant, OsVersionTarget target, NodeList allNodes) { NodeList activeNodes = allNodes.state(Node.State.active).nodeType(target.nodeType()); diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/NodesResponse.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/NodesResponse.java index 3659166c9da..efd76187bc6 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/NodesResponse.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/NodesResponse.java @@ -161,7 +161,10 @@ class NodesResponse extends SlimeJsonResponse { object.setLong("rebootGeneration", node.status().reboot().wanted()); object.setLong("currentRebootGeneration", node.status().reboot().current()); node.status().osVersion().current().ifPresent(version -> object.setString("currentOsVersion", version.toFullString())); - node.status().osVersion().wanted().ifPresent(version -> object.setString("wantedOsVersion", version.toFullString())); + node.status().osVersion().wanted().ifPresent(version -> { + object.setString("wantedOsVersion", version.toFullString()); + object.setBool("deferOsUpgrade", !nodeRepository.osVersions().canUpgrade(node)); + }); node.status().firmwareVerifiedAt().ifPresent(instant -> object.setLong("currentFirmwareCheck", instant.toEpochMilli())); if (node.type().isHost()) nodeRepository.firmwareChecks().requiredAfter().ifPresent(after -> object.setLong("wantedFirmwareCheck", after.toEpochMilli())); diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/os/OsVersionsTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/os/OsVersionsTest.java index 97a8ac0d655..af2a215dae0 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/os/OsVersionsTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/os/OsVersionsTest.java @@ -172,6 +172,7 @@ public class OsVersionsTest { Supplier<NodeList> hostNodes = () -> tester.nodeRepository().nodes().list() .hosts() .not().state(Node.State.deprovisioned); + tester.clock().advance(RetiringOsUpgrader.GRACE_PERIOD.plusDays(1)); // Target is set and upgrade started var version1 = Version.fromString("7.1"); @@ -233,6 +234,7 @@ public class OsVersionsTest { Supplier<NodeList> hostNodes = () -> tester.nodeRepository().nodes().list() .nodeType(NodeType.confighost) .not().state(Node.State.deprovisioned); + tester.clock().advance(RetiringOsUpgrader.GRACE_PERIOD.plusDays(1)); // Target is set with zero budget and upgrade started var version1 = Version.fromString("7.1"); diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/NodesV2ApiTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/NodesV2ApiTest.java index 6b1853b3893..19af4d00e54 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/NodesV2ApiTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/NodesV2ApiTest.java @@ -770,8 +770,9 @@ public class NodesV2ApiTest { Request.Method.PATCH), "{\"message\":\"Set osVersion to 7.5.2, upgradeBudget to PT0S for nodes of type host\"}"); + var nodeRepository = (NodeRepository) tester.container().components().getComponent(MockNodeRepository.class.getName()); + // Activate target - var nodeRepository = (NodeRepository)tester.container().components().getComponent(MockNodeRepository.class.getName()); var osUpgradeActivator = new OsUpgradeActivator(nodeRepository, Duration.ofDays(1), new TestMetric()); osUpgradeActivator.run(); diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/docker-node1-os-upgrade-complete.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/docker-node1-os-upgrade-complete.json index b885f7bd7fc..287db73faf6 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/docker-node1-os-upgrade-complete.json +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/docker-node1-os-upgrade-complete.json @@ -30,6 +30,7 @@ "currentRebootGeneration": 0, "currentOsVersion": "7.5.2", "wantedOsVersion": "7.5.2", + "deferOsUpgrade": false, "failCount": 0, "wantToRetire": false, "preferToRetire": false, diff --git a/persistence/src/vespa/persistence/conformancetest/conformancetest.cpp b/persistence/src/vespa/persistence/conformancetest/conformancetest.cpp index bf1c828e2e6..e03403f601d 100644 --- a/persistence/src/vespa/persistence/conformancetest/conformancetest.cpp +++ b/persistence/src/vespa/persistence/conformancetest/conformancetest.cpp @@ -765,11 +765,11 @@ TEST_F(ConformanceTest, testRemoveMulti) docs.push_back(testDocMan.createRandomDocumentAtLocation(0x01, i)); } - std::vector<PersistenceProvider::TimeStampAndDocumentId> ids; + std::vector<spi::IdAndTimestamp> ids; for (size_t i(0); i < docs.size(); i++) { spi->put(bucket1, Timestamp(i), docs[i]); if (i & 0x1) { - ids.emplace_back(Timestamp(i), docs[i]->getId()); + ids.emplace_back(docs[i]->getId(), Timestamp(i)); } } diff --git a/persistence/src/vespa/persistence/dummyimpl/dummypersistence.cpp b/persistence/src/vespa/persistence/dummyimpl/dummypersistence.cpp index 81bfdf7f9a3..6eabadc2f86 100644 --- a/persistence/src/vespa/persistence/dummyimpl/dummypersistence.cpp +++ b/persistence/src/vespa/persistence/dummyimpl/dummypersistence.cpp @@ -476,16 +476,16 @@ DummyPersistence::updateAsync(const Bucket& bucket, Timestamp ts, DocumentUpdate } void -DummyPersistence::removeAsync(const Bucket& b, std::vector<TimeStampAndDocumentId> ids, OperationComplete::UP onComplete) +DummyPersistence::removeAsync(const Bucket& b, std::vector<spi::IdAndTimestamp> ids, OperationComplete::UP onComplete) { DUMMYPERSISTENCE_VERIFY_INITIALIZED; assert(b.getBucketSpace() == FixedBucketSpaces::default_space()); BucketContentGuard::UP bc(acquireBucketWithLock(b)); uint32_t numRemoves(0); - for (const TimeStampAndDocumentId & stampedId : ids) { - const DocumentId & id = stampedId.second; - Timestamp t = stampedId.first; + for (const spi::IdAndTimestamp & stampedId : ids) { + const DocumentId & id = stampedId.id; + Timestamp t = stampedId.timestamp; LOG(debug, "remove(%s, %" PRIu64 ", %s)", b.toString().c_str(), uint64_t(t), id.toString().c_str()); while (!bc) { diff --git a/persistence/src/vespa/persistence/dummyimpl/dummypersistence.h b/persistence/src/vespa/persistence/dummyimpl/dummypersistence.h index e015185e5b0..56602b3ab00 100644 --- a/persistence/src/vespa/persistence/dummyimpl/dummypersistence.h +++ b/persistence/src/vespa/persistence/dummyimpl/dummypersistence.h @@ -160,7 +160,7 @@ public: BucketInfoResult getBucketInfo(const Bucket&) const override; GetResult get(const Bucket&, const document::FieldSet&, const DocumentId&, Context&) const override; void putAsync(const Bucket&, Timestamp, DocumentSP, OperationComplete::UP) override; - void removeAsync(const Bucket& b, std::vector<TimeStampAndDocumentId> ids, OperationComplete::UP) override; + void removeAsync(const Bucket& b, std::vector<spi::IdAndTimestamp> ids, OperationComplete::UP) override; void updateAsync(const Bucket&, Timestamp, DocumentUpdateSP, OperationComplete::UP) override; CreateIteratorResult diff --git a/persistence/src/vespa/persistence/spi/CMakeLists.txt b/persistence/src/vespa/persistence/spi/CMakeLists.txt index e4bae1c7551..bc94020ae95 100644 --- a/persistence/src/vespa/persistence/spi/CMakeLists.txt +++ b/persistence/src/vespa/persistence/spi/CMakeLists.txt @@ -10,6 +10,7 @@ vespa_add_library(persistence_spi OBJECT context.cpp docentry.cpp exceptions.cpp + id_and_timestamp.cpp persistenceprovider.cpp read_consistency.cpp resource_usage.cpp diff --git a/persistence/src/vespa/persistence/spi/abstractpersistenceprovider.cpp b/persistence/src/vespa/persistence/spi/abstractpersistenceprovider.cpp index f301a9c5428..04d06235f59 100644 --- a/persistence/src/vespa/persistence/spi/abstractpersistenceprovider.cpp +++ b/persistence/src/vespa/persistence/spi/abstractpersistenceprovider.cpp @@ -10,8 +10,8 @@ void AbstractPersistenceProvider::removeIfFoundAsync(const Bucket& b, Timestamp timestamp, const DocumentId& id, OperationComplete::UP onComplete) { - std::vector<TimeStampAndDocumentId> ids; - ids.emplace_back(timestamp, id); + std::vector<IdAndTimestamp> ids; + ids.emplace_back(id, timestamp); removeAsync(b, std::move(ids), std::move(onComplete)); } diff --git a/persistence/src/vespa/persistence/spi/id_and_timestamp.cpp b/persistence/src/vespa/persistence/spi/id_and_timestamp.cpp new file mode 100644 index 00000000000..fba45990744 --- /dev/null +++ b/persistence/src/vespa/persistence/spi/id_and_timestamp.cpp @@ -0,0 +1,17 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include "id_and_timestamp.h" + +namespace storage::spi { + +IdAndTimestamp::IdAndTimestamp() : id(), timestamp(0) {} +IdAndTimestamp::IdAndTimestamp(document::DocumentId id_, Timestamp timestamp_) noexcept + : id(std::move(id_)), + timestamp(timestamp_) +{} + +IdAndTimestamp::IdAndTimestamp(const IdAndTimestamp&) = default; +IdAndTimestamp& IdAndTimestamp::operator=(const IdAndTimestamp&) = default; +IdAndTimestamp::IdAndTimestamp(IdAndTimestamp&&) noexcept = default; +IdAndTimestamp& IdAndTimestamp::operator=(IdAndTimestamp&&) noexcept = default; + +} diff --git a/persistence/src/vespa/persistence/spi/id_and_timestamp.h b/persistence/src/vespa/persistence/spi/id_and_timestamp.h new file mode 100644 index 00000000000..d8cdba3d063 --- /dev/null +++ b/persistence/src/vespa/persistence/spi/id_and_timestamp.h @@ -0,0 +1,38 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#pragma once + +#include "types.h" +#include <vespa/document/base/documentid.h> + +namespace storage::spi { + +/** + * Convenience wrapper for referencing a document ID at a particular timestamp. + * + * Prefer this instead of a std::pair due to named fields and a pre-provided hash function. + */ +struct IdAndTimestamp { + document::DocumentId id; + Timestamp timestamp; + + IdAndTimestamp(); + IdAndTimestamp(document::DocumentId id_, Timestamp timestamp_) noexcept; + + IdAndTimestamp(const IdAndTimestamp&); + IdAndTimestamp& operator=(const IdAndTimestamp&); + IdAndTimestamp(IdAndTimestamp&&) noexcept; + IdAndTimestamp& operator=(IdAndTimestamp&&) noexcept; + + bool operator==(const IdAndTimestamp& rhs) const noexcept { + return ((id == rhs.id) && (timestamp == rhs.timestamp)); + } + + struct hash { + size_t operator()(const IdAndTimestamp& id_ts) const noexcept { + const size_t h = document::GlobalId::hash()(id_ts.id.getGlobalId()); + return h ^ (id_ts.timestamp + 0x9e3779b9U + (h << 6U) + (h >> 2U)); // Basically boost::hash_combine + } + }; +}; + +} diff --git a/persistence/src/vespa/persistence/spi/persistenceprovider.cpp b/persistence/src/vespa/persistence/spi/persistenceprovider.cpp index 03cefb8df89..911b3753b1f 100644 --- a/persistence/src/vespa/persistence/spi/persistenceprovider.cpp +++ b/persistence/src/vespa/persistence/spi/persistenceprovider.cpp @@ -44,8 +44,8 @@ RemoveResult PersistenceProvider::remove(const Bucket& bucket, Timestamp timestamp, const DocumentId & docId) { auto catcher = std::make_unique<CatchResult>(); auto future = catcher->future_result(); - std::vector<TimeStampAndDocumentId> ids; - ids.emplace_back(timestamp, docId); + std::vector<IdAndTimestamp> ids; + ids.emplace_back(docId, timestamp); removeAsync(bucket, std::move(ids), std::move(catcher)); return dynamic_cast<const RemoveResult &>(*future.get()); } diff --git a/persistence/src/vespa/persistence/spi/persistenceprovider.h b/persistence/src/vespa/persistence/spi/persistenceprovider.h index a90d39e8334..d3e1465e528 100644 --- a/persistence/src/vespa/persistence/spi/persistenceprovider.h +++ b/persistence/src/vespa/persistence/spi/persistenceprovider.h @@ -4,6 +4,7 @@ #include "bucket.h" #include "bucketinfo.h" #include "context.h" +#include "id_and_timestamp.h" #include "result.h" #include "selection.h" #include "clusterstate.h" @@ -170,7 +171,7 @@ struct PersistenceProvider * @param timestamp The timestamp for the new bucket entry. * @param id The ID to remove */ - virtual void removeAsync(const Bucket&, std::vector<TimeStampAndDocumentId> ids, OperationComplete::UP) = 0; + virtual void removeAsync(const Bucket&, std::vector<IdAndTimestamp> ids, OperationComplete::UP) = 0; /** * @see remove() diff --git a/screwdriver/build-vespa.sh b/screwdriver/build-vespa.sh index d14a46697ab..6a93474c620 100755 --- a/screwdriver/build-vespa.sh +++ b/screwdriver/build-vespa.sh @@ -53,28 +53,6 @@ case $SHOULD_BUILD in esac if [[ $SHOULD_BUILD == systemtest ]]; then - dnf module enable -y ruby:3.0 - dnf install -y \ - gcc-toolset-11-annobin \ - gcc-toolset-11-annobin-plugin-gcc \ - gcc-toolset-11-binutils \ - gcc-toolset-11-gcc-c++ \ - gcc-toolset-11-libatomic-devel \ - libxml2-devel \ - ruby \ - ruby-devel \ - rubygems-devel \ - rubygem-bigdecimal \ - rubygem-builder \ - rubygem-concurrent-ruby \ - rubygem-parallel \ - rubygem-rexml \ - rubygem-test-unit \ - zstd - - source /opt/rh/gcc-toolset-11/enable - gem install ffi libxml-ruby - cd $HOME git clone https://github.com/vespa-engine/system-test export SYSTEM_TEST_DIR=$(pwd)/system-test diff --git a/searchcore/src/vespa/searchcore/bmcluster/spi_bm_feed_handler.cpp b/searchcore/src/vespa/searchcore/bmcluster/spi_bm_feed_handler.cpp index 69013e8d7c5..dcdba3b0715 100644 --- a/searchcore/src/vespa/searchcore/bmcluster/spi_bm_feed_handler.cpp +++ b/searchcore/src/vespa/searchcore/bmcluster/spi_bm_feed_handler.cpp @@ -134,8 +134,8 @@ SpiBmFeedHandler::remove(const document::Bucket& bucket, const DocumentId& docum auto provider = get_provider(bucket); if (provider) { Bucket spi_bucket(bucket); - std::vector<storage::spi::PersistenceProvider::TimeStampAndDocumentId> ids; - ids.emplace_back(Timestamp(timestamp), document_id); + std::vector<storage::spi::IdAndTimestamp> ids; + ids.emplace_back(document_id, Timestamp(timestamp)); provider->removeAsync(spi_bucket, std::move(ids), std::make_unique<MyOperationComplete>(provider, _errors, spi_bucket, tracker)); } else { ++_errors; diff --git a/searchcore/src/vespa/searchcore/proton/persistenceengine/persistenceengine.cpp b/searchcore/src/vespa/searchcore/proton/persistenceengine/persistenceengine.cpp index 5a0bcb1cd41..81a7244ba1d 100644 --- a/searchcore/src/vespa/searchcore/proton/persistenceengine/persistenceengine.cpp +++ b/searchcore/src/vespa/searchcore/proton/persistenceengine/persistenceengine.cpp @@ -370,21 +370,21 @@ PersistenceEngine::putAsync(const Bucket &bucket, Timestamp ts, storage::spi::Do } void -PersistenceEngine::removeAsync(const Bucket& b, std::vector<TimeStampAndDocumentId> ids, OperationComplete::UP onComplete) +PersistenceEngine::removeAsync(const Bucket& b, std::vector<storage::spi::IdAndTimestamp> ids, OperationComplete::UP onComplete) { if (ids.size() == 1) { - removeAsyncSingle(b, ids[0].first, ids[0].second, std::move(onComplete)); + removeAsyncSingle(b, ids[0].timestamp, ids[0].id, std::move(onComplete)); } else { removeAsyncMulti(b, std::move(ids), std::move(onComplete)); } } void -PersistenceEngine::removeAsyncMulti(const Bucket& b, std::vector<TimeStampAndDocumentId> ids, OperationComplete::UP onComplete) { +PersistenceEngine::removeAsyncMulti(const Bucket& b, std::vector<storage::spi::IdAndTimestamp> ids, OperationComplete::UP onComplete) { ReadGuard rguard(_rwMutex); //TODO Group per document type/handler and handle in one go. - for (const TimeStampAndDocumentId & stampedId : ids) { - const document::DocumentId & id = stampedId.second; + for (const auto & stampedId : ids) { + const document::DocumentId & id = stampedId.id; if (!id.hasDocType()) { return onComplete->onComplete( std::make_unique<RemoveResult>(Result::ErrorType::PERMANENT_ERROR, @@ -399,11 +399,11 @@ PersistenceEngine::removeAsyncMulti(const Bucket& b, std::vector<TimeStampAndDoc } } auto transportContext = std::make_shared<AsyncRemoveTransportContext>(ids.size(), std::move(onComplete)); - for (const TimeStampAndDocumentId & stampedId : ids) { - const document::DocumentId & id = stampedId.second; + for (const auto & stampedId : ids) { + const document::DocumentId & id = stampedId.id; DocTypeName docType(id.getDocType()); IPersistenceHandler *handler = getHandler(rguard, b.getBucketSpace(), docType); - handler->handleRemove(feedtoken::make(transportContext), b, stampedId.first, id); + handler->handleRemove(feedtoken::make(transportContext), b, stampedId.timestamp, id); } } diff --git a/searchcore/src/vespa/searchcore/proton/persistenceengine/persistenceengine.h b/searchcore/src/vespa/searchcore/proton/persistenceengine/persistenceengine.h index a8886e19def..c16cc6e6a83 100644 --- a/searchcore/src/vespa/searchcore/proton/persistenceengine/persistenceengine.h +++ b/searchcore/src/vespa/searchcore/proton/persistenceengine/persistenceengine.h @@ -89,7 +89,7 @@ private: ClusterState::SP savedClusterState(BucketSpace bucketSpace) const; std::shared_ptr<BucketExecutor> get_bucket_executor() noexcept { return _bucket_executor.lock(); } void removeAsyncSingle(const Bucket&, Timestamp, const document::DocumentId &id, OperationComplete::UP); - void removeAsyncMulti(const Bucket&, std::vector<TimeStampAndDocumentId> ids, OperationComplete::UP); + void removeAsyncMulti(const Bucket&, std::vector<storage::spi::IdAndTimestamp> ids, OperationComplete::UP); public: typedef std::unique_ptr<PersistenceEngine> UP; @@ -107,7 +107,7 @@ public: void setActiveStateAsync(const Bucket&, BucketInfo::ActiveState, OperationComplete::UP) override; BucketInfoResult getBucketInfo(const Bucket&) const override; void putAsync(const Bucket &, Timestamp, storage::spi::DocumentSP, OperationComplete::UP) override; - void removeAsync(const Bucket&, std::vector<TimeStampAndDocumentId> ids, OperationComplete::UP) override; + void removeAsync(const Bucket&, std::vector<storage::spi::IdAndTimestamp> ids, OperationComplete::UP) override; void updateAsync(const Bucket&, Timestamp, storage::spi::DocumentUpdateSP, OperationComplete::UP) override; GetResult get(const Bucket&, const document::FieldSet&, const document::DocumentId&, Context&) const override; CreateIteratorResult diff --git a/searchsummary/src/tests/docsummary/attribute_combiner/attribute_combiner_test.cpp b/searchsummary/src/tests/docsummary/attribute_combiner/attribute_combiner_test.cpp index d5a908fc8bd..7265dd89be4 100644 --- a/searchsummary/src/tests/docsummary/attribute_combiner/attribute_combiner_test.cpp +++ b/searchsummary/src/tests/docsummary/attribute_combiner/attribute_combiner_test.cpp @@ -5,7 +5,7 @@ #include <vespa/searchlib/common/matching_elements.h> #include <vespa/searchlib/common/matching_elements_fields.h> #include <vespa/searchlib/util/slime_output_raw_buf_adapter.h> -#include <vespa/searchsummary/docsummary/docsumfieldwriter.h> +#include <vespa/searchsummary/docsummary/docsum_field_writer.h> #include <vespa/searchsummary/docsummary/docsumstate.h> #include <vespa/searchsummary/docsummary/docsum_field_writer_state.h> #include <vespa/searchsummary/docsummary/attribute_combiner_dfw.h> @@ -24,7 +24,7 @@ using search::docsummary::AttributeCombinerDFW; using search::docsummary::GetDocsumsState; using search::docsummary::GetDocsumsStateCallback; using search::docsummary::IDocsumEnvironment; -using search::docsummary::IDocsumFieldWriter; +using search::docsummary::DocsumFieldWriter; using search::docsummary::test::MockAttributeManager; using search::docsummary::test::MockStateCallback; using search::docsummary::test::SlimeValue; @@ -34,7 +34,7 @@ namespace { struct AttributeCombinerTest : public ::testing::Test { MockAttributeManager attrs; - std::unique_ptr<IDocsumFieldWriter> writer; + std::unique_ptr<DocsumFieldWriter> writer; MockStateCallback callback; GetDocsumsState state; std::shared_ptr<search::MatchingElementsFields> _matching_elems_fields; diff --git a/searchsummary/src/tests/docsummary/attributedfw/attributedfw_test.cpp b/searchsummary/src/tests/docsummary/attributedfw/attributedfw_test.cpp index 67d505582d8..42443cf1058 100644 --- a/searchsummary/src/tests/docsummary/attributedfw/attributedfw_test.cpp +++ b/searchsummary/src/tests/docsummary/attributedfw/attributedfw_test.cpp @@ -16,7 +16,7 @@ using search::attribute::BasicType; using search::attribute::CollectionType; using search::docsummary::AttributeDFWFactory; using search::docsummary::GetDocsumsState; -using search::docsummary::IDocsumFieldWriter; +using search::docsummary::DocsumFieldWriter; using search::docsummary::test::MockAttributeManager; using search::docsummary::test::MockStateCallback; using search::docsummary::test::SlimeValue; @@ -26,7 +26,7 @@ using ElementVector = std::vector<uint32_t>; class AttributeDFWTest : public ::testing::Test { protected: MockAttributeManager _attrs; - std::unique_ptr<IDocsumFieldWriter> _writer; + std::unique_ptr<DocsumFieldWriter> _writer; MockStateCallback _callback; GetDocsumsState _state; std::shared_ptr<search::MatchingElementsFields> _matching_elems_fields; diff --git a/searchsummary/src/tests/docsummary/matched_elements_filter/matched_elements_filter_test.cpp b/searchsummary/src/tests/docsummary/matched_elements_filter/matched_elements_filter_test.cpp index 675af283ee8..82aa9ceba92 100644 --- a/searchsummary/src/tests/docsummary/matched_elements_filter/matched_elements_filter_test.cpp +++ b/searchsummary/src/tests/docsummary/matched_elements_filter/matched_elements_filter_test.cpp @@ -16,8 +16,10 @@ #include <vespa/searchlib/common/matching_elements_fields.h> #include <vespa/searchlib/util/slime_output_raw_buf_adapter.h> #include <vespa/searchsummary/docsummary/docsum_store_document.h> +#include <vespa/searchsummary/docsummary/docsumstorevalue.h> #include <vespa/searchsummary/docsummary/docsumstate.h> #include <vespa/searchsummary/docsummary/idocsumenvironment.h> +#include <vespa/searchsummary/docsummary/general_result.h> #include <vespa/searchsummary/docsummary/matched_elements_filter_dfw.h> #include <vespa/searchsummary/docsummary/resultconfig.h> #include <vespa/searchsummary/docsummary/resultpacker.h> @@ -215,7 +217,7 @@ public: { } ~MatchedElementsFilterTest(); - std::unique_ptr<IDocsumFieldWriter> make_field_writer(const std::string& input_field_name) { + std::unique_ptr<DocsumFieldWriter> make_field_writer(const std::string& input_field_name) { int input_field_enum = _doc_store.get_config().GetFieldNameEnum().Lookup(input_field_name.c_str()); return MatchedElementsFilterDFW::create(input_field_name, input_field_enum, _attr_ctx, _fields); diff --git a/searchsummary/src/tests/docsummary/positionsdfw_test.cpp b/searchsummary/src/tests/docsummary/positionsdfw_test.cpp index 11f0d0eb6d6..60584b26e31 100644 --- a/searchsummary/src/tests/docsummary/positionsdfw_test.cpp +++ b/searchsummary/src/tests/docsummary/positionsdfw_test.cpp @@ -4,7 +4,7 @@ #include <vespa/searchlib/attribute/extendableattributes.h> #include <vespa/searchlib/attribute/iattributemanager.h> #include <vespa/searchlib/common/matching_elements.h> -#include <vespa/searchsummary/docsummary/docsumfieldwriter.h> +#include <vespa/searchsummary/docsummary/docsum_field_writer.h> #include <vespa/searchsummary/docsummary/positionsdfw.h> #include <vespa/searchsummary/docsummary/idocsumenvironment.h> #include <vespa/searchsummary/docsummary/docsumstate.h> diff --git a/searchsummary/src/vespa/searchsummary/docsummary/CMakeLists.txt b/searchsummary/src/vespa/searchsummary/docsummary/CMakeLists.txt index 947fe9deeab..e3272fb36de 100644 --- a/searchsummary/src/vespa/searchsummary/docsummary/CMakeLists.txt +++ b/searchsummary/src/vespa/searchsummary/docsummary/CMakeLists.txt @@ -6,13 +6,15 @@ vespa_add_library(searchsummary_docsummary OBJECT attribute_field_writer.cpp attributedfw.cpp check_undefined_value_visitor.cpp + copy_dfw.cpp docsumconfig.cpp - docsumfieldwriter.cpp - docsumstate.cpp + docsum_field_writer.cpp docsum_store_document.cpp + docsumstate.cpp docsumstorevalue.cpp docsumwriter.cpp dynamicteaserdfw.cpp + empty_dfw.cpp general_result.cpp geoposdfw.cpp getdocsumargs.cpp @@ -22,10 +24,12 @@ vespa_add_library(searchsummary_docsummary OBJECT matched_elements_filter_dfw.cpp positionsdfw.cpp rankfeaturesdfw.cpp + res_type_utils.cpp resultclass.cpp resultconfig.cpp resultpacker.cpp searchdatatype.cpp + simple_dfw.cpp struct_fields_resolver.cpp struct_map_attribute_combiner_dfw.cpp summaryfeaturesdfw.cpp diff --git a/searchsummary/src/vespa/searchsummary/docsummary/array_attribute_combiner_dfw.cpp b/searchsummary/src/vespa/searchsummary/docsummary/array_attribute_combiner_dfw.cpp index ff5c2c5e05b..f308795a1bc 100644 --- a/searchsummary/src/vespa/searchsummary/docsummary/array_attribute_combiner_dfw.cpp +++ b/searchsummary/src/vespa/searchsummary/docsummary/array_attribute_combiner_dfw.cpp @@ -9,6 +9,7 @@ #include <vespa/searchlib/common/matching_elements.h> #include <vespa/searchlib/common/matching_elements_fields.h> #include <vespa/vespalib/data/slime/cursor.h> +#include <vespa/vespalib/data/slime/inserter.h> #include <vespa/vespalib/util/stash.h> #include <algorithm> #include <cassert> diff --git a/searchsummary/src/vespa/searchsummary/docsummary/array_attribute_combiner_dfw.h b/searchsummary/src/vespa/searchsummary/docsummary/array_attribute_combiner_dfw.h index 18b4fd34e66..e5bed876b63 100644 --- a/searchsummary/src/vespa/searchsummary/docsummary/array_attribute_combiner_dfw.h +++ b/searchsummary/src/vespa/searchsummary/docsummary/array_attribute_combiner_dfw.h @@ -3,6 +3,7 @@ #pragma once #include "attribute_combiner_dfw.h" +#include <vector> namespace search::attribute { class IAttributeContext; } diff --git a/searchsummary/src/vespa/searchsummary/docsummary/attribute_combiner_dfw.cpp b/searchsummary/src/vespa/searchsummary/docsummary/attribute_combiner_dfw.cpp index 79c11b20479..bf5578f38d6 100644 --- a/searchsummary/src/vespa/searchsummary/docsummary/attribute_combiner_dfw.cpp +++ b/searchsummary/src/vespa/searchsummary/docsummary/attribute_combiner_dfw.cpp @@ -18,7 +18,7 @@ namespace search::docsummary { AttributeCombinerDFW::AttributeCombinerDFW(const vespalib::string &fieldName, bool filter_elements, std::shared_ptr<MatchingElementsFields> matching_elems_fields) - : ISimpleDFW(), + : SimpleDFW(), _stateIndex(0), _filter_elements(filter_elements), _fieldName(fieldName), @@ -41,13 +41,13 @@ AttributeCombinerDFW::setFieldWriterStateIndex(uint32_t fieldWriterStateIndex) return true; } -std::unique_ptr<IDocsumFieldWriter> +std::unique_ptr<DocsumFieldWriter> AttributeCombinerDFW::create(const vespalib::string &fieldName, IAttributeContext &attrCtx, bool filter_elements, std::shared_ptr<MatchingElementsFields> matching_elems_fields) { StructFieldsResolver structFields(fieldName, attrCtx, true); if (structFields.has_error()) { - return std::unique_ptr<IDocsumFieldWriter>(); + return std::unique_ptr<DocsumFieldWriter>(); } else if (structFields.is_map_of_struct()) { return std::make_unique<StructMapAttributeCombinerDFW>(fieldName, structFields, filter_elements, std::move(matching_elems_fields)); } diff --git a/searchsummary/src/vespa/searchsummary/docsummary/attribute_combiner_dfw.h b/searchsummary/src/vespa/searchsummary/docsummary/attribute_combiner_dfw.h index c1742595745..39f2d498c5b 100644 --- a/searchsummary/src/vespa/searchsummary/docsummary/attribute_combiner_dfw.h +++ b/searchsummary/src/vespa/searchsummary/docsummary/attribute_combiner_dfw.h @@ -2,7 +2,8 @@ #pragma once -#include "docsumfieldwriter.h" +#include "simple_dfw.h" +#include <memory> namespace search { class MatchingElements; @@ -21,7 +22,7 @@ class DynamicDocsumWriter; * This class reads values from multiple struct field attributes and * inserts them as an array of struct or a map of struct. */ -class AttributeCombinerDFW : public ISimpleDFW +class AttributeCombinerDFW : public SimpleDFW { protected: uint32_t _stateIndex; @@ -36,8 +37,8 @@ public: ~AttributeCombinerDFW() override; bool IsGenerated() const override; bool setFieldWriterStateIndex(uint32_t fieldWriterStateIndex) override; - static std::unique_ptr<IDocsumFieldWriter> create(const vespalib::string &fieldName, search::attribute::IAttributeContext &attrCtx, - bool filter_elements, std::shared_ptr<MatchingElementsFields> matching_elems_fields); + static std::unique_ptr<DocsumFieldWriter> create(const vespalib::string &fieldName, search::attribute::IAttributeContext &attrCtx, + bool filter_elements, std::shared_ptr<MatchingElementsFields> matching_elems_fields); void insertField(uint32_t docid, GetDocsumsState *state, ResType type, vespalib::slime::Inserter &target) override; }; diff --git a/searchsummary/src/vespa/searchsummary/docsummary/attributedfw.cpp b/searchsummary/src/vespa/searchsummary/docsummary/attributedfw.cpp index d5fdee096b1..e7b6acec646 100644 --- a/searchsummary/src/vespa/searchsummary/docsummary/attributedfw.cpp +++ b/searchsummary/src/vespa/searchsummary/docsummary/attributedfw.cpp @@ -2,6 +2,7 @@ #include "attributedfw.h" #include "docsumwriter.h" +#include "docsumstate.h" #include "docsum_field_writer_state.h" #include <vespa/eval/eval/value.h> #include <vespa/eval/eval/value_codec.h> @@ -38,6 +39,24 @@ AttrDFW::AttrDFW(const vespalib::string & attrName) : { } +const attribute::IAttributeVector& +AttrDFW::get_attribute(const GetDocsumsState& s) const +{ + return *s.getAttribute(getIndex()); +} + +const vespalib::string & +AttrDFW::getAttributeName() const +{ + return _attrName; +} + +bool +AttrDFW::IsGenerated() const +{ + return true; +} + namespace { class SingleAttrDFW : public AttrDFW @@ -333,7 +352,7 @@ MultiAttrDFW::insertField(uint32_t docid, GetDocsumsState *state, ResType, vespa field_writer_state->insertField(docid, target); } -std::unique_ptr<IDocsumFieldWriter> +std::unique_ptr<DocsumFieldWriter> create_multi_writer(const IAttributeVector& attr, bool filter_elements, std::shared_ptr<MatchingElementsFields> matching_elems_fields) { auto type = attr.getBasicType(); @@ -355,7 +374,7 @@ create_multi_writer(const IAttributeVector& attr, bool filter_elements, std::sha } -std::unique_ptr<IDocsumFieldWriter> +std::unique_ptr<DocsumFieldWriter> AttributeDFWFactory::create(IAttributeManager& attr_mgr, const vespalib::string& attr_name, bool filter_elements, @@ -365,7 +384,7 @@ AttributeDFWFactory::create(IAttributeManager& attr_mgr, const auto* attr = ctx->getAttribute(attr_name); if (attr == nullptr) { Issue::report("No valid attribute vector found: '%s'", attr_name.c_str()); - return std::unique_ptr<IDocsumFieldWriter>(); + return std::unique_ptr<DocsumFieldWriter>(); } if (attr->hasMultiValue()) { return create_multi_writer(*attr, filter_elements, std::move(matching_elems_fields)); diff --git a/searchsummary/src/vespa/searchsummary/docsummary/attributedfw.h b/searchsummary/src/vespa/searchsummary/docsummary/attributedfw.h index 35f67fd5446..26351bdf501 100644 --- a/searchsummary/src/vespa/searchsummary/docsummary/attributedfw.h +++ b/searchsummary/src/vespa/searchsummary/docsummary/attributedfw.h @@ -2,38 +2,38 @@ #pragma once -#include "docsumfieldwriter.h" -#include "docsumstate.h" +#include "simple_dfw.h" +#include <memory> -namespace search { class MatchingElementsFields; } +namespace search { +class IAttributeManager; +class MatchingElementsFields; +} namespace search::attribute { class IAttributeVector; } namespace search::docsummary { /** - * Factory to create an IDocsumFieldWriter to write an attribute vector to slime. + * Factory to create an DocsumFieldWriter to write an attribute vector to slime. */ class AttributeDFWFactory { public: - static std::unique_ptr<IDocsumFieldWriter> create(IAttributeManager& attr_mgr, - const vespalib::string& attr_name, - bool filter_elements = false, - std::shared_ptr<MatchingElementsFields> matching_elems_fields - = std::shared_ptr<MatchingElementsFields>()); + static std::unique_ptr<DocsumFieldWriter> create(IAttributeManager& attr_mgr, + const vespalib::string& attr_name, + bool filter_elements = false, + std::shared_ptr<MatchingElementsFields> matching_elems_fields = std::shared_ptr<MatchingElementsFields>()); }; -class AttrDFW : public ISimpleDFW +class AttrDFW : public SimpleDFW { private: vespalib::string _attrName; protected: - const attribute::IAttributeVector& get_attribute(const GetDocsumsState& s) const { - return *s.getAttribute(getIndex()); - } - const vespalib::string & getAttributeName() const override { return _attrName; } + const attribute::IAttributeVector& get_attribute(const GetDocsumsState& s) const; + const vespalib::string & getAttributeName() const override; public: AttrDFW(const vespalib::string & attrName); - bool IsGenerated() const override { return true; } + bool IsGenerated() const override; }; } diff --git a/searchsummary/src/vespa/searchsummary/docsummary/docsumfieldwriter.cpp b/searchsummary/src/vespa/searchsummary/docsummary/copy_dfw.cpp index 0f0ac9ddf72..836273ce3d8 100644 --- a/searchsummary/src/vespa/searchsummary/docsummary/docsumfieldwriter.cpp +++ b/searchsummary/src/vespa/searchsummary/docsummary/copy_dfw.cpp @@ -1,51 +1,16 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include "docsumfieldwriter.h" +#include "copy_dfw.h" +#include "general_result.h" #include "i_docsum_store_document.h" -#include "idocsumenvironment.h" -#include "docsumstate.h" -#include "summaryfieldconverter.h" -#include <vespa/searchlib/common/documentlocations.h> -#include <vespa/searchlib/common/location.h> -#include <vespa/searchlib/parsequery/stackdumpiterator.h> +#include "resultconfig.h" #include <vespa/vespalib/data/slime/slime.h> #include <vespa/log/log.h> -LOG_SETUP(".searchlib.docsummary.docsumfieldwriter"); +LOG_SETUP(".searchlib.docsummary.copy_dfw"); namespace search::docsummary { -using search::attribute::IAttributeContext; -using search::attribute::IAttributeVector; -using search::attribute::BasicType; -using search::common::Location; - -//-------------------------------------------------------------------------- - -const vespalib::string IDocsumFieldWriter::_empty(""); - -bool -IDocsumFieldWriter::setFieldWriterStateIndex(uint32_t) -{ - return false; // Don't need any field writer state by default -} - -//-------------------------------------------------------------------------- - -EmptyDFW::EmptyDFW() = default; - -EmptyDFW::~EmptyDFW() = default; - -void -EmptyDFW::insertField(uint32_t, GetDocsumsState *, ResType, vespalib::slime::Inserter &target) -{ - // insert explicitly-empty field? - // target.insertNix(); - (void)target; -} - -//-------------------------------------------------------------------------- - CopyDFW::CopyDFW() : _inputFieldEnumValue(static_cast<uint32_t>(-1)), _input_field_name() diff --git a/searchsummary/src/vespa/searchsummary/docsummary/copy_dfw.h b/searchsummary/src/vespa/searchsummary/docsummary/copy_dfw.h new file mode 100644 index 00000000000..dab7417f60a --- /dev/null +++ b/searchsummary/src/vespa/searchsummary/docsummary/copy_dfw.h @@ -0,0 +1,31 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#pragma once + +#include "docsum_field_writer.h" + +namespace search::docsummary { + +class ResultConfig; + +/* + * Class for writing document summaries with content from another field. + */ +class CopyDFW : public DocsumFieldWriter +{ +private: + uint32_t _inputFieldEnumValue; + vespalib::string _input_field_name; + +public: + CopyDFW(); + ~CopyDFW() override; + + bool Init(const ResultConfig & config, const char *inputField); + + bool IsGenerated() const override { return false; } + void insertField(uint32_t docid, GeneralResult *gres, GetDocsumsState *state, ResType type, + vespalib::slime::Inserter &target) override; +}; + +} diff --git a/searchsummary/src/vespa/searchsummary/docsummary/docsum_blob_entry_filter.h b/searchsummary/src/vespa/searchsummary/docsummary/docsum_blob_entry_filter.h new file mode 100644 index 00000000000..1d006386d35 --- /dev/null +++ b/searchsummary/src/vespa/searchsummary/docsummary/docsum_blob_entry_filter.h @@ -0,0 +1,29 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#pragma once + +#include "res_type.h" +#include <bitset> + +namespace search::docsummary { + +/* + * Class containing the set of result types not stored in docsum blobs. + * This is used for gradual migration towards elimination of docsum blobs. + */ +class DocsumBlobEntryFilter { + std::bitset<14> _skip_types; + +public: + DocsumBlobEntryFilter() + : _skip_types() + { + } + bool skip(ResType type) const noexcept { return _skip_types.test(type); } + DocsumBlobEntryFilter &add_skip(ResType type) { + _skip_types.set(type); + return *this; + } +}; + +} diff --git a/searchsummary/src/vespa/searchsummary/docsummary/docsum_field_writer.cpp b/searchsummary/src/vespa/searchsummary/docsummary/docsum_field_writer.cpp new file mode 100644 index 00000000000..c698f0603c6 --- /dev/null +++ b/searchsummary/src/vespa/searchsummary/docsummary/docsum_field_writer.cpp @@ -0,0 +1,27 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include "docsum_field_writer.h" + +namespace search::docsummary { + +const vespalib::string DocsumFieldWriter::_empty(""); + +const vespalib::string& +DocsumFieldWriter::getAttributeName() const +{ + return _empty; +} + +bool +DocsumFieldWriter::isDefaultValue(uint32_t, const GetDocsumsState*) const +{ + return false; +} + +bool +DocsumFieldWriter::setFieldWriterStateIndex(uint32_t) +{ + return false; // Don't need any field writer state by default +} + +} diff --git a/searchsummary/src/vespa/searchsummary/docsummary/docsum_field_writer.h b/searchsummary/src/vespa/searchsummary/docsummary/docsum_field_writer.h new file mode 100644 index 00000000000..764f3507380 --- /dev/null +++ b/searchsummary/src/vespa/searchsummary/docsummary/docsum_field_writer.h @@ -0,0 +1,41 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#pragma once + +#include "res_type_utils.h" +#include <vespa/vespalib/stllike/string.h> + +namespace vespalib::slime { struct Inserter; } + +namespace search::docsummary { + +class GeneralResult; +class GetDocsumsState; + +/* + * Abstract class for writing document summaries. + */ +class DocsumFieldWriter +{ +public: + DocsumFieldWriter() + : _index(0) + { + } + virtual ~DocsumFieldWriter() = default; + static bool IsRuntimeCompatible(ResType a, ResType b) { + return ResTypeUtils::IsRuntimeCompatible(a, b); + } + virtual bool IsGenerated() const = 0; + virtual void insertField(uint32_t docid, GeneralResult *gres, GetDocsumsState *state, ResType type, vespalib::slime::Inserter &target) = 0; + virtual const vespalib::string & getAttributeName() const; + virtual bool isDefaultValue(uint32_t docid, const GetDocsumsState * state) const; + void setIndex(size_t v) { _index = v; } + size_t getIndex() const { return _index; } + virtual bool setFieldWriterStateIndex(uint32_t fieldWriterStateIndex); +private: + size_t _index; + static const vespalib::string _empty; +}; + +} diff --git a/searchsummary/src/vespa/searchsummary/docsummary/docsumconfig.cpp b/searchsummary/src/vespa/searchsummary/docsummary/docsumconfig.cpp index 24642c418fd..376d4f90204 100644 --- a/searchsummary/src/vespa/searchsummary/docsummary/docsumconfig.cpp +++ b/searchsummary/src/vespa/searchsummary/docsummary/docsumconfig.cpp @@ -1,8 +1,10 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include "attribute_combiner_dfw.h" #include "docsumconfig.h" +#include "attribute_combiner_dfw.h" +#include "copy_dfw.h" #include "docsumwriter.h" +#include "empty_dfw.h" #include "geoposdfw.h" #include "idocsumenvironment.h" #include "juniperdfw.h" @@ -10,6 +12,7 @@ #include "positionsdfw.h" #include "rankfeaturesdfw.h" #include "textextractordfw.h" +#include "summaryfeaturesdfw.h" #include <vespa/searchlib/common/matching_elements_fields.h> #include <vespa/vespalib/util/stringfmt.h> #include <vespa/vespalib/util/exceptions.h> @@ -24,12 +27,12 @@ DynamicDocsumConfig::getResultConfig() const { return *_writer->GetResultConfig(); } -IDocsumFieldWriter::UP +std::unique_ptr<DocsumFieldWriter> DynamicDocsumConfig::createFieldWriter(const string & fieldName, const string & overrideName, const string & argument, bool & rc, std::shared_ptr<MatchingElementsFields> matching_elems_fields) { const ResultConfig & resultConfig = getResultConfig(); rc = false; - IDocsumFieldWriter::UP fieldWriter; + std::unique_ptr<DocsumFieldWriter> fieldWriter; if (overrideName == "dynamicteaser") { if ( ! argument.empty() ) { const char *langFieldName = "something unused"; @@ -127,7 +130,7 @@ DynamicDocsumConfig::configure(const vespa::config::search::SummarymapConfig &cf for (size_t i = 0; i < cfg.override.size(); ++i) { const vespa::config::search::SummarymapConfig::Override & o = cfg.override[i]; bool rc(false); - IDocsumFieldWriter::UP fieldWriter = createFieldWriter(o.field, o.command, o.arguments, rc, matching_elems_fields); + std::unique_ptr<DocsumFieldWriter> fieldWriter = createFieldWriter(o.field, o.command, o.arguments, rc, matching_elems_fields); if (rc && fieldWriter) { rc = _writer->Override(o.field.c_str(), fieldWriter.release()); // OBJECT HAND-OVER } diff --git a/searchsummary/src/vespa/searchsummary/docsummary/docsumconfig.h b/searchsummary/src/vespa/searchsummary/docsummary/docsumconfig.h index 70c8d524527..b86313dfbd4 100644 --- a/searchsummary/src/vespa/searchsummary/docsummary/docsumconfig.h +++ b/searchsummary/src/vespa/searchsummary/docsummary/docsumconfig.h @@ -8,9 +8,9 @@ namespace search { class MatchingElementsFields; } namespace search::docsummary { class IDocsumEnvironment; +class DocsumFieldWriter; class DynamicDocsumWriter; class ResultConfig; -class IDocsumFieldWriter; class DynamicDocsumConfig { @@ -27,7 +27,7 @@ protected: const IDocsumEnvironment * getEnvironment() const { return _env; } const ResultConfig & getResultConfig() const; - virtual std::unique_ptr<IDocsumFieldWriter> + virtual std::unique_ptr<DocsumFieldWriter> createFieldWriter(const string & fieldName, const string & overrideName, const string & argument, bool & rc, std::shared_ptr<MatchingElementsFields> matching_elems_fields); private: diff --git a/searchsummary/src/vespa/searchsummary/docsummary/docsumfieldwriter.h b/searchsummary/src/vespa/searchsummary/docsummary/docsumfieldwriter.h deleted file mode 100644 index bc135404de1..00000000000 --- a/searchsummary/src/vespa/searchsummary/docsummary/docsumfieldwriter.h +++ /dev/null @@ -1,86 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#pragma once - -#include "general_result.h" -#include "resultconfig.h" -#include <vespa/searchlib/util/rawbuf.h> -#include <vespa/vespalib/data/slime/inserter.h> - -namespace search { class IAttributeManager; } - -namespace search::docsummary { - -class GetDocsumsState; - -class IDocsumFieldWriter -{ -public: - using UP = std::unique_ptr<IDocsumFieldWriter>; - IDocsumFieldWriter() : _index(0) { } - virtual ~IDocsumFieldWriter() = default; - - static bool IsRuntimeCompatible(ResType a, ResType b) { - return ResultConfig::IsRuntimeCompatible(a, b); - } - - virtual bool IsGenerated() const = 0; - virtual void insertField(uint32_t docid, GeneralResult *gres, GetDocsumsState *state, ResType type, - vespalib::slime::Inserter &target) = 0; - virtual const vespalib::string & getAttributeName() const { return _empty; } - virtual bool isDefaultValue(uint32_t docid, const GetDocsumsState * state) const { - (void) docid; - (void) state; - return false; - } - void setIndex(size_t v) { _index = v; } - size_t getIndex() const { return _index; } - virtual bool setFieldWriterStateIndex(uint32_t fieldWriterStateIndex); -private: - size_t _index; - static const vespalib::string _empty; -}; - -class ISimpleDFW : public IDocsumFieldWriter -{ -public: - virtual void insertField(uint32_t docid, GetDocsumsState *state, ResType type, vespalib::slime::Inserter &target) = 0; - void insertField(uint32_t docid, GeneralResult *, GetDocsumsState *state, ResType type, - vespalib::slime::Inserter &target) override - { - insertField(docid, state, type, target); - } -}; - -//-------------------------------------------------------------------------- - -class EmptyDFW : public ISimpleDFW -{ -public: - EmptyDFW(); - ~EmptyDFW() override; - - bool IsGenerated() const override { return true; } - void insertField(uint32_t docid, GetDocsumsState *state, ResType type, vespalib::slime::Inserter &target) override; -}; - -//-------------------------------------------------------------------------- - -class CopyDFW : public IDocsumFieldWriter -{ -private: - uint32_t _inputFieldEnumValue; - vespalib::string _input_field_name; - -public: - CopyDFW(); - ~CopyDFW() override; - - bool Init(const ResultConfig & config, const char *inputField); - - bool IsGenerated() const override { return false; } - void insertField(uint32_t docid, GeneralResult *gres, GetDocsumsState *state, ResType type, - vespalib::slime::Inserter &target) override; -}; - -} diff --git a/searchsummary/src/vespa/searchsummary/docsummary/docsumwriter.cpp b/searchsummary/src/vespa/searchsummary/docsummary/docsumwriter.cpp index c6e5a224e21..1492ce2b435 100644 --- a/searchsummary/src/vespa/searchsummary/docsummary/docsumwriter.cpp +++ b/searchsummary/src/vespa/searchsummary/docsummary/docsumwriter.cpp @@ -168,7 +168,7 @@ DynamicDocsumWriter::insertDocsum(const ResolveClassInfo & rci, uint32_t docid, vespalib::slime::Cursor & docsum = topInserter.insertObject(); for (uint32_t i = 0; i < rci.outputClass->GetNumEntries(); ++i) { const ResConfigEntry *resCfg = rci.outputClass->GetEntry(i); - IDocsumFieldWriter *writer = _overrideTable[resCfg->_enumValue]; + DocsumFieldWriter *writer = _overrideTable[resCfg->_enumValue]; if (! writer->isDefaultValue(docid, state)) { const Memory field_name(resCfg->_bindname.data(), resCfg->_bindname.size()); ObjectInserter inserter(docsum, field_name); @@ -188,7 +188,7 @@ DynamicDocsumWriter::insertDocsum(const ResolveClassInfo & rci, uint32_t docid, vespalib::slime::Cursor & docsum = topInserter.insertObject(); for (uint32_t i = 0; i < rci.outputClass->GetNumEntries(); ++i) { const ResConfigEntry *outCfg = rci.outputClass->GetEntry(i); - IDocsumFieldWriter *writer = _overrideTable[outCfg->_enumValue]; + DocsumFieldWriter *writer = _overrideTable[outCfg->_enumValue]; const Memory field_name(outCfg->_bindname.data(), outCfg->_bindname.size()); ObjectInserter inserter(docsum, field_name); if (writer != nullptr) { @@ -230,7 +230,7 @@ DynamicDocsumWriter::DynamicDocsumWriter( ResultConfig *config, KeywordExtractor { LOG_ASSERT(config != nullptr); _classInfoTable = new ResultClass::DynamicInfo[_numClasses]; - _overrideTable = new IDocsumFieldWriter*[_numEnumValues]; + _overrideTable = new DocsumFieldWriter*[_numEnumValues]; uint32_t i = 0; for (ResultConfig::iterator it(config->begin()), mt(config->end()); it != mt; it++, i++) { @@ -279,7 +279,7 @@ DynamicDocsumWriter::SetDefaultOutputClass(uint32_t classID) bool -DynamicDocsumWriter::Override(const char *fieldName, IDocsumFieldWriter *writer) +DynamicDocsumWriter::Override(const char *fieldName, DocsumFieldWriter *writer) { uint32_t fieldEnumValue = _resultConfig->GetFieldNameEnum().Lookup(fieldName); @@ -324,7 +324,7 @@ DynamicDocsumWriter::InitState(IAttributeManager & attrMan, GetDocsumsState *sta state->_attributes.resize(_numEnumValues); state->_fieldWriterStates.resize(_numFieldWriterStates); for (size_t i(0); i < state->_attributes.size(); i++) { - const IDocsumFieldWriter *fw = _overrideTable[i]; + const DocsumFieldWriter *fw = _overrideTable[i]; if (fw) { const vespalib::string & attributeName = fw->getAttributeName(); if (!attributeName.empty()) { diff --git a/searchsummary/src/vespa/searchsummary/docsummary/docsumwriter.h b/searchsummary/src/vespa/searchsummary/docsummary/docsumwriter.h index e70e3db8655..b3182221b68 100644 --- a/searchsummary/src/vespa/searchsummary/docsummary/docsumwriter.h +++ b/searchsummary/src/vespa/searchsummary/docsummary/docsumwriter.h @@ -7,12 +7,14 @@ #include "resultconfig.h" #include "docsumstore.h" #include "keywordextractor.h" -#include "docsumfieldwriter.h" +#include "docsum_field_writer.h" #include <vespa/searchlib/util/rawbuf.h> #include <vespa/fastlib/text/unicodeutil.h> #include <vespa/fastlib/text/wordfolder.h> -using search::IAttributeManager; +namespace search { class IAttributeManager; } + +namespace vespalib { class Slime; } namespace search::docsummary { @@ -36,7 +38,7 @@ public: }; virtual ~IDocsumWriter() {} - virtual void InitState(IAttributeManager & attrMan, GetDocsumsState *state) = 0; + virtual void InitState(search::IAttributeManager & attrMan, GetDocsumsState *state) = 0; virtual uint32_t WriteDocsum(uint32_t docid, GetDocsumsState *state, IDocsumStore *docinfos, search::RawBuf *target) = 0; virtual void insertDocsum(const ResolveClassInfo & rci, uint32_t docid, GetDocsumsState *state, @@ -58,7 +60,7 @@ private: uint32_t _numEnumValues; uint32_t _numFieldWriterStates; ResultClass::DynamicInfo *_classInfoTable; - IDocsumFieldWriter **_overrideTable; + DocsumFieldWriter** _overrideTable; void resolveInputClass(ResolveClassInfo &rci, uint32_t id) const; @@ -73,8 +75,8 @@ public: ResultConfig *GetResultConfig() { return _resultConfig; } bool SetDefaultOutputClass(uint32_t classID); - bool Override(const char *fieldName, IDocsumFieldWriter *writer); - void InitState(IAttributeManager & attrMan, GetDocsumsState *state) override; + bool Override(const char *fieldName, DocsumFieldWriter *writer); + void InitState(search::IAttributeManager & attrMan, GetDocsumsState *state) override; uint32_t WriteDocsum(uint32_t docid, GetDocsumsState *state, IDocsumStore *docinfos, search::RawBuf *target) override; diff --git a/searchsummary/src/vespa/searchsummary/docsummary/empty_dfw.cpp b/searchsummary/src/vespa/searchsummary/docsummary/empty_dfw.cpp new file mode 100644 index 00000000000..3d3b1e11626 --- /dev/null +++ b/searchsummary/src/vespa/searchsummary/docsummary/empty_dfw.cpp @@ -0,0 +1,19 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include "empty_dfw.h" + +namespace search::docsummary { + +EmptyDFW::EmptyDFW() = default; + +EmptyDFW::~EmptyDFW() = default; + +void +EmptyDFW::insertField(uint32_t, GetDocsumsState *, ResType, vespalib::slime::Inserter &target) +{ + // insert explicitly-empty field? + // target.insertNix(); + (void)target; +} + +} diff --git a/searchsummary/src/vespa/searchsummary/docsummary/empty_dfw.h b/searchsummary/src/vespa/searchsummary/docsummary/empty_dfw.h new file mode 100644 index 00000000000..d43eed8e9c6 --- /dev/null +++ b/searchsummary/src/vespa/searchsummary/docsummary/empty_dfw.h @@ -0,0 +1,22 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#pragma once + +#include "simple_dfw.h" + +namespace search::docsummary { + +/* + * Class for writing empty document summaries. + */ +class EmptyDFW : public SimpleDFW +{ +public: + EmptyDFW(); + ~EmptyDFW() override; + + bool IsGenerated() const override { return true; } + void insertField(uint32_t docid, GetDocsumsState *state, ResType type, vespalib::slime::Inserter &target) override; +}; + +} diff --git a/searchsummary/src/vespa/searchsummary/docsummary/geoposdfw.cpp b/searchsummary/src/vespa/searchsummary/docsummary/geoposdfw.cpp index 8f627ac1b9a..c3806d6e7ea 100644 --- a/searchsummary/src/vespa/searchsummary/docsummary/geoposdfw.cpp +++ b/searchsummary/src/vespa/searchsummary/docsummary/geoposdfw.cpp @@ -6,6 +6,7 @@ #include <vespa/searchlib/common/location.h> #include <vespa/vespalib/util/jsonwriter.h> #include <vespa/vespalib/data/slime/cursor.h> +#include <vespa/vespalib/data/slime/inserter.h> #include <vespa/vespalib/stllike/asciistream.h> #include <vespa/vespalib/util/issue.h> #include <climits> diff --git a/searchsummary/src/vespa/searchsummary/docsummary/juniperdfw.h b/searchsummary/src/vespa/searchsummary/docsummary/juniperdfw.h index d9a657038c4..5e2ec517a47 100644 --- a/searchsummary/src/vespa/searchsummary/docsummary/juniperdfw.h +++ b/searchsummary/src/vespa/searchsummary/docsummary/juniperdfw.h @@ -4,14 +4,14 @@ #include "general_result.h" #include "resultconfig.h" -#include "docsumfieldwriter.h" +#include "docsum_field_writer.h" #include <vespa/searchlib/util/rawbuf.h> #include <vespa/vespalib/data/slime/inserter.h> #include <vespa/juniper/rpinterface.h> namespace search::docsummary { -class JuniperDFW : public IDocsumFieldWriter +class JuniperDFW : public DocsumFieldWriter { public: virtual bool Init( @@ -21,7 +21,7 @@ public: const char *inputField); protected: JuniperDFW(juniper::Juniper * juniper); - virtual ~JuniperDFW(); + ~JuniperDFW() override; uint32_t _inputFieldEnumValue; std::unique_ptr<juniper::Config> _juniperConfig; diff --git a/searchsummary/src/vespa/searchsummary/docsummary/matched_elements_filter_dfw.cpp b/searchsummary/src/vespa/searchsummary/docsummary/matched_elements_filter_dfw.cpp index c2115201d9b..c05fec7a0ce 100644 --- a/searchsummary/src/vespa/searchsummary/docsummary/matched_elements_filter_dfw.cpp +++ b/searchsummary/src/vespa/searchsummary/docsummary/matched_elements_filter_dfw.cpp @@ -1,7 +1,8 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include "docsumstate.h" #include "matched_elements_filter_dfw.h" +#include "docsumstate.h" +#include "general_result.h" #include "struct_fields_resolver.h" #include "summaryfieldconverter.h" #include <vespa/document/fieldvalue/document.h> @@ -40,21 +41,21 @@ MatchedElementsFilterDFW::MatchedElementsFilterDFW(const std::string& input_fiel { } -std::unique_ptr<IDocsumFieldWriter> +std::unique_ptr<DocsumFieldWriter> MatchedElementsFilterDFW::create(const std::string& input_field_name, uint32_t input_field_enum, std::shared_ptr<MatchingElementsFields> matching_elems_fields) { return std::make_unique<MatchedElementsFilterDFW>(input_field_name, input_field_enum, std::move(matching_elems_fields)); } -std::unique_ptr<IDocsumFieldWriter> +std::unique_ptr<DocsumFieldWriter> MatchedElementsFilterDFW::create(const std::string& input_field_name, uint32_t input_field_enum, search::attribute::IAttributeContext& attr_ctx, std::shared_ptr<MatchingElementsFields> matching_elems_fields) { StructFieldsResolver resolver(input_field_name, attr_ctx, false); if (resolver.has_error()) { - return std::unique_ptr<IDocsumFieldWriter>(); + return std::unique_ptr<DocsumFieldWriter>(); } resolver.apply_to(*matching_elems_fields); return std::make_unique<MatchedElementsFilterDFW>(input_field_name, input_field_enum, std::move(matching_elems_fields)); diff --git a/searchsummary/src/vespa/searchsummary/docsummary/matched_elements_filter_dfw.h b/searchsummary/src/vespa/searchsummary/docsummary/matched_elements_filter_dfw.h index 505a2557408..b117da541d6 100644 --- a/searchsummary/src/vespa/searchsummary/docsummary/matched_elements_filter_dfw.h +++ b/searchsummary/src/vespa/searchsummary/docsummary/matched_elements_filter_dfw.h @@ -2,7 +2,11 @@ #pragma once -#include "docsumfieldwriter.h" +#include "docsum_field_writer.h" +#include <memory> +#include <vector> + +namespace search { class MatchingElementsFields; } namespace search::attribute { class IAttributeContext; } @@ -13,7 +17,7 @@ namespace search::docsummary { * (array of primitive, weighted set of primitive, map of primitives, map of struct, array of struct) * that is retrieved from the document store. */ -class MatchedElementsFilterDFW : public IDocsumFieldWriter { +class MatchedElementsFilterDFW : public DocsumFieldWriter { private: std::string _input_field_name; uint32_t _input_field_enum; @@ -24,11 +28,11 @@ private: public: MatchedElementsFilterDFW(const std::string& input_field_name, uint32_t input_field_enum, std::shared_ptr<MatchingElementsFields> matching_elems_fields); - static std::unique_ptr<IDocsumFieldWriter> create(const std::string& input_field_name, uint32_t input_field_enum, - std::shared_ptr<MatchingElementsFields> matching_elems_fields); - static std::unique_ptr<IDocsumFieldWriter> create(const std::string& input_field_name, uint32_t input_field_enum, - search::attribute::IAttributeContext& attr_ctx, - std::shared_ptr<MatchingElementsFields> matching_elems_fields); + static std::unique_ptr<DocsumFieldWriter> create(const std::string& input_field_name, uint32_t input_field_enum, + std::shared_ptr<MatchingElementsFields> matching_elems_fields); + static std::unique_ptr<DocsumFieldWriter> create(const std::string& input_field_name, uint32_t input_field_enum, + search::attribute::IAttributeContext& attr_ctx, + std::shared_ptr<MatchingElementsFields> matching_elems_fields); ~MatchedElementsFilterDFW(); bool IsGenerated() const override { return false; } void insertField(uint32_t docid, GeneralResult* result, GetDocsumsState *state, diff --git a/searchsummary/src/vespa/searchsummary/docsummary/positionsdfw.cpp b/searchsummary/src/vespa/searchsummary/docsummary/positionsdfw.cpp index 1fcb0a49be1..7f3a929a62f 100644 --- a/searchsummary/src/vespa/searchsummary/docsummary/positionsdfw.cpp +++ b/searchsummary/src/vespa/searchsummary/docsummary/positionsdfw.cpp @@ -270,8 +270,9 @@ PositionsDFW::UP PositionsDFW::create(const char *attribute_name, IAttributeMana return std::make_unique<PositionsDFW>(attribute_name, useV8geoPositions); } -AbsDistanceDFW::UP AbsDistanceDFW::create(const char *attribute_name, IAttributeManager *attribute_manager) { - AbsDistanceDFW::UP ret; +std::unique_ptr<DocsumFieldWriter> +AbsDistanceDFW::create(const char *attribute_name, IAttributeManager *attribute_manager) { + std::unique_ptr<DocsumFieldWriter> ret; if (attribute_manager != nullptr) { if (!attribute_name) { LOG(debug, "createAbsDistanceDFW: missing attribute name '%p'", attribute_name); diff --git a/searchsummary/src/vespa/searchsummary/docsummary/positionsdfw.h b/searchsummary/src/vespa/searchsummary/docsummary/positionsdfw.h index b3e041c1379..d9445abd2ff 100644 --- a/searchsummary/src/vespa/searchsummary/docsummary/positionsdfw.h +++ b/searchsummary/src/vespa/searchsummary/docsummary/positionsdfw.h @@ -45,7 +45,7 @@ public: void insertField(uint32_t docid, GetDocsumsState *state, ResType type, vespalib::slime::Inserter &target) override; - static UP create(const char *attribute_name, IAttributeManager *index_man); + static std::unique_ptr<DocsumFieldWriter> create(const char *attribute_name, IAttributeManager *index_man); }; diff --git a/searchsummary/src/vespa/searchsummary/docsummary/rankfeaturesdfw.cpp b/searchsummary/src/vespa/searchsummary/docsummary/rankfeaturesdfw.cpp index 38b58ef94fc..5d3b104189b 100644 --- a/searchsummary/src/vespa/searchsummary/docsummary/rankfeaturesdfw.cpp +++ b/searchsummary/src/vespa/searchsummary/docsummary/rankfeaturesdfw.cpp @@ -3,6 +3,7 @@ #include "rankfeaturesdfw.h" #include "docsumstate.h" #include <vespa/vespalib/data/slime/cursor.h> +#include <vespa/vespalib/data/slime/inserter.h> namespace search::docsummary { diff --git a/searchsummary/src/vespa/searchsummary/docsummary/rankfeaturesdfw.h b/searchsummary/src/vespa/searchsummary/docsummary/rankfeaturesdfw.h index eab9fab60b2..91f9e80d303 100644 --- a/searchsummary/src/vespa/searchsummary/docsummary/rankfeaturesdfw.h +++ b/searchsummary/src/vespa/searchsummary/docsummary/rankfeaturesdfw.h @@ -2,11 +2,13 @@ #pragma once -#include "summaryfeaturesdfw.h" +#include "simple_dfw.h" namespace search::docsummary { -class RankFeaturesDFW : public ISimpleDFW +class IDocsumEnvironment; + +class RankFeaturesDFW : public SimpleDFW { private: IDocsumEnvironment * _env; diff --git a/searchsummary/src/vespa/searchsummary/docsummary/res_type.h b/searchsummary/src/vespa/searchsummary/docsummary/res_type.h new file mode 100644 index 00000000000..02c9f1522a4 --- /dev/null +++ b/searchsummary/src/vespa/searchsummary/docsummary/res_type.h @@ -0,0 +1,30 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#pragma once + +namespace search::docsummary { + +/** + * This enumeration contains values denoting the different types of + * docsum fields. NOTE: The internal implementation depends on RES_INT + * having the value 0. All types < RES_STRING must be fixed size and + * all types > RES_STRING must be variable size. + **/ +enum ResType { + RES_INT = 0, + RES_SHORT, + RES_BOOL, + RES_BYTE, + RES_FLOAT, + RES_DOUBLE, + RES_INT64, + RES_STRING, + RES_DATA, + RES_LONG_STRING, + RES_LONG_DATA, + RES_JSONSTRING, + RES_TENSOR, + RES_FEATUREDATA +}; + +} diff --git a/searchsummary/src/vespa/searchsummary/docsummary/res_type_utils.cpp b/searchsummary/src/vespa/searchsummary/docsummary/res_type_utils.cpp new file mode 100644 index 00000000000..98cc8372ac1 --- /dev/null +++ b/searchsummary/src/vespa/searchsummary/docsummary/res_type_utils.cpp @@ -0,0 +1,29 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include "res_type_utils.h" + +namespace search::docsummary { + +const char * +ResTypeUtils::GetResTypeName(ResType type) +{ + switch (type) { + case RES_INT: return "integer"; + case RES_SHORT: return "short"; + case RES_BYTE: return "byte"; + case RES_BOOL: return "bool"; + case RES_FLOAT: return "float"; + case RES_DOUBLE: return "double"; + case RES_INT64: return "int64"; + case RES_STRING: return "string"; + case RES_DATA: return "data"; + case RES_LONG_STRING: return "longstring"; + case RES_LONG_DATA: return "longdata"; + case RES_JSONSTRING: return "jsonstring"; + case RES_TENSOR: return "tensor"; + case RES_FEATUREDATA: return "featuredata"; + } + return "unknown-type"; +} + +} diff --git a/searchsummary/src/vespa/searchsummary/docsummary/res_type_utils.h b/searchsummary/src/vespa/searchsummary/docsummary/res_type_utils.h new file mode 100644 index 00000000000..194a008c179 --- /dev/null +++ b/searchsummary/src/vespa/searchsummary/docsummary/res_type_utils.h @@ -0,0 +1,102 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#pragma once + +#include "res_type.h" + +namespace search::docsummary { + + +/* + * Utilitiy functions for checking if result type is ok. + */ +struct ResTypeUtils +{ + /** + * Determine if a result field type is of variable size. + * + * @return true for variable size field types, false for fixed + * size field types + **/ + static bool IsVariableSize(ResType t) { return (t >= RES_STRING); } + + + /** + * Determine if a pair of result field types are binary + * compatible. A pair of types are binary compatible if the packed + * representation is identical. + * + * @return true if the given types are binary compatible. + * @param a enum value of a result field type. + * @param b enum value of a result field type. + **/ + static bool IsBinaryCompatible(ResType a, ResType b) + { + if (a == b) { + return true; + } + switch (a) { + case RES_BYTE: + case RES_BOOL: + return (b == RES_BYTE || b == RES_BOOL); + case RES_STRING: + case RES_DATA: + return (b == RES_STRING || b == RES_DATA); + case RES_LONG_STRING: + case RES_LONG_DATA: + case RES_FEATUREDATA: + case RES_JSONSTRING: + return (b == RES_LONG_STRING || b == RES_LONG_DATA || + b == RES_FEATUREDATA || b == RES_JSONSTRING); + default: + return false; + } + return false; + } + + + /** + * Determine if a pair of result field types are runtime + * compatible. A pair of types are runtime compatible if the + * unpacked (@ref ResEntry) representation is identical. + * + * @return true if the given types are runtime compatible. + * @param a enum value of a result field type. + * @param b enum value of a result field type. + **/ + static bool IsRuntimeCompatible(ResType a, ResType b) + { + switch (a) { + case RES_INT: + case RES_SHORT: + case RES_BYTE: + case RES_BOOL: + return (b == RES_INT || b == RES_SHORT || b == RES_BYTE || b == RES_BOOL); + case RES_FLOAT: + case RES_DOUBLE: + return (b == RES_FLOAT || b == RES_DOUBLE); + case RES_INT64: + return b == RES_INT64; + case RES_STRING: + case RES_LONG_STRING: + case RES_JSONSTRING: + return (b == RES_STRING || b == RES_LONG_STRING || b == RES_JSONSTRING); + case RES_DATA: + case RES_LONG_DATA: + return (b == RES_DATA || b == RES_LONG_DATA); + case RES_TENSOR: + return (b == RES_TENSOR); + case RES_FEATUREDATA: + return (b == RES_FEATUREDATA); + } + return false; + } + + /** + * @return the name of the given result field type. + * @param resType enum value of a result field type. + **/ + static const char *GetResTypeName(ResType type); +}; + +} diff --git a/searchsummary/src/vespa/searchsummary/docsummary/resultclass.h b/searchsummary/src/vespa/searchsummary/docsummary/resultclass.h index d6d247238cd..47feed70e97 100644 --- a/searchsummary/src/vespa/searchsummary/docsummary/resultclass.h +++ b/searchsummary/src/vespa/searchsummary/docsummary/resultclass.h @@ -2,57 +2,15 @@ #pragma once +#include "docsum_blob_entry_filter.h" #include <vespa/searchlib/util/rawbuf.h> #include <vespa/vespalib/stllike/string.h> #include <vespa/vespalib/stllike/hash_map.h> #include <vespa/searchlib/util/stringenum.h> -#include <bitset> namespace search::docsummary { /** - * This enumeration contains values denoting the different types of - * docsum fields. NOTE: The internal implementation depends on RES_INT - * having the value 0. All types < RES_STRING must be fixed size and - * all types > RES_STRING must be variable size. - **/ -enum ResType { - RES_INT = 0, - RES_SHORT, - RES_BOOL, - RES_BYTE, - RES_FLOAT, - RES_DOUBLE, - RES_INT64, - RES_STRING, - RES_DATA, - RES_LONG_STRING, - RES_LONG_DATA, - RES_JSONSTRING, - RES_TENSOR, - RES_FEATUREDATA -}; - -/* - * Class containing the set of result types not stored in docsum blobs. - * This is used for gradual migration towards elimination of docsum blobs. - */ -class DocsumBlobEntryFilter { - std::bitset<14> _skip_types; - -public: - DocsumBlobEntryFilter() - : _skip_types() - { - } - bool skip(ResType type) const noexcept { return _skip_types.test(type); } - DocsumBlobEntryFilter &add_skip(ResType type) { - _skip_types.set(type); - return *this; - } -}; - -/** * This struct describes a single docsum field (name and type). A * docsum blob is unpacked into an array of ResEntry instances * by interpreting it as described by an array of ResConfigEntry diff --git a/searchsummary/src/vespa/searchsummary/docsummary/resultconfig.cpp b/searchsummary/src/vespa/searchsummary/docsummary/resultconfig.cpp index 4096e26a6e3..168b4c81374 100644 --- a/searchsummary/src/vespa/searchsummary/docsummary/resultconfig.cpp +++ b/searchsummary/src/vespa/searchsummary/docsummary/resultconfig.cpp @@ -1,6 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "resultconfig.h" +#include "resultclass.h" #include <vespa/vespalib/util/exceptions.h> #include <vespa/vespalib/stllike/hash_map.hpp> #include <atomic> @@ -45,28 +46,6 @@ ResultConfig::~ResultConfig() } -const char * -ResultConfig::GetResTypeName(ResType type) -{ - switch (type) { - case RES_INT: return "integer"; - case RES_SHORT: return "short"; - case RES_BYTE: return "byte"; - case RES_BOOL: return "bool"; - case RES_FLOAT: return "float"; - case RES_DOUBLE: return "double"; - case RES_INT64: return "int64"; - case RES_STRING: return "string"; - case RES_DATA: return "data"; - case RES_LONG_STRING: return "longstring"; - case RES_LONG_DATA: return "longdata"; - case RES_JSONSTRING: return "jsonstring"; - case RES_TENSOR: return "tensor"; - case RES_FEATUREDATA: return "featuredata"; - } - return "unknown-type"; -} - void ResultConfig::Reset() { diff --git a/searchsummary/src/vespa/searchsummary/docsummary/resultconfig.h b/searchsummary/src/vespa/searchsummary/docsummary/resultconfig.h index 1438aee73ce..945eef8514f 100644 --- a/searchsummary/src/vespa/searchsummary/docsummary/resultconfig.h +++ b/searchsummary/src/vespa/searchsummary/docsummary/resultconfig.h @@ -2,14 +2,15 @@ #pragma once -#include "resultclass.h" -#include "general_result.h" +#include "docsum_blob_entry_filter.h" +#include "res_type_utils.h" #include <vespa/config-summary.h> -#include <vespa/searchlib/util/rawbuf.h> #include <vespa/searchlib/util/stringenum.h> namespace search::docsummary { +class ResultClass; + /** * This class represents the overall result configuration. A result * configuration may contain multiple result classes, where each @@ -31,7 +32,7 @@ private: ResultConfig& operator=(const ResultConfig &); typedef vespalib::hash_map<vespalib::string, uint32_t> NameMap; - typedef vespalib::hash_map<uint32_t, ResultClass::UP> IdMap; + typedef vespalib::hash_map<uint32_t, std::unique_ptr<ResultClass>> IdMap; uint32_t _defaultSummaryId; bool _useV8geoPositions; search::util::StringEnum _fieldEnum; @@ -95,85 +96,9 @@ public: static uint32_t NoClassID() { return static_cast<uint32_t>(-1); } - /** - * Determine if a result field type is of variable size. - * - * @return true for variable size field types, false for fixed - * size field types - **/ - static bool IsVariableSize(ResType t) { return (t >= RES_STRING); } - - - /** - * Determine if a pair of result field types are binary - * compatible. A pair of types are binary compatible if the packed - * representation is identical. - * - * @return true if the given types are binary compatible. - * @param a enum value of a result field type. - * @param b enum value of a result field type. - **/ - static bool IsBinaryCompatible(ResType a, ResType b) - { - if (a == b) { - return true; - } - switch (a) { - case RES_BYTE: - case RES_BOOL: - return (b == RES_BYTE || b == RES_BOOL); - case RES_STRING: - case RES_DATA: - return (b == RES_STRING || b == RES_DATA); - case RES_LONG_STRING: - case RES_LONG_DATA: - case RES_FEATUREDATA: - case RES_JSONSTRING: - return (b == RES_LONG_STRING || b == RES_LONG_DATA || - b == RES_FEATUREDATA || b == RES_JSONSTRING); - default: - return false; - } - return false; - } - - - /** - * Determine if a pair of result field types are runtime - * compatible. A pair of types are runtime compatible if the - * unpacked (@ref ResEntry) representation is identical. - * - * @return true if the given types are runtime compatible. - * @param a enum value of a result field type. - * @param b enum value of a result field type. - **/ - static bool IsRuntimeCompatible(ResType a, ResType b) - { - switch (a) { - case RES_INT: - case RES_SHORT: - case RES_BYTE: - case RES_BOOL: - return (b == RES_INT || b == RES_SHORT || b == RES_BYTE || b == RES_BOOL); - case RES_FLOAT: - case RES_DOUBLE: - return (b == RES_FLOAT || b == RES_DOUBLE); - case RES_INT64: - return b == RES_INT64; - case RES_STRING: - case RES_LONG_STRING: - case RES_JSONSTRING: - return (b == RES_STRING || b == RES_LONG_STRING || b == RES_JSONSTRING); - case RES_DATA: - case RES_LONG_DATA: - return (b == RES_DATA || b == RES_LONG_DATA); - case RES_TENSOR: - return (b == RES_TENSOR); - case RES_FEATUREDATA: - return (b == RES_FEATUREDATA); - } - return false; - } + static bool IsVariableSize(ResType t) { return ResTypeUtils::IsVariableSize(t); } + static bool IsBinaryCompatible(ResType a, ResType b) { return ResTypeUtils::IsBinaryCompatible(a, b); } + static bool IsRuntimeCompatible(ResType a, ResType b) { return ResTypeUtils::IsRuntimeCompatible(a, b); } // whether last config seen wanted useV8geoPositions = true static bool wantedV8geoPositions(); @@ -182,7 +107,7 @@ public: * @return the name of the given result field type. * @param resType enum value of a result field type. **/ - static const char *GetResTypeName(ResType type); + static const char *GetResTypeName(ResType type) { return ResTypeUtils::GetResTypeName(type); } /** * Discard the current configuration and start over. After this diff --git a/searchsummary/src/vespa/searchsummary/docsummary/resultpacker.cpp b/searchsummary/src/vespa/searchsummary/docsummary/resultpacker.cpp index 4cf36785f69..66b15c1c8a9 100644 --- a/searchsummary/src/vespa/searchsummary/docsummary/resultpacker.cpp +++ b/searchsummary/src/vespa/searchsummary/docsummary/resultpacker.cpp @@ -1,6 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "resultpacker.h" +#include "resultconfig.h" #include <vespa/searchcommon/common/undefinedvalues.h> #include <vespa/vespalib/util/size_literals.h> diff --git a/searchsummary/src/vespa/searchsummary/docsummary/resultpacker.h b/searchsummary/src/vespa/searchsummary/docsummary/resultpacker.h index f2460f3d3c3..816433652b8 100644 --- a/searchsummary/src/vespa/searchsummary/docsummary/resultpacker.h +++ b/searchsummary/src/vespa/searchsummary/docsummary/resultpacker.h @@ -2,9 +2,14 @@ #pragma once -#include "resultconfig.h" +#include "res_type_utils.h" +#include "resultclass.h" +#include <vespa/searchlib/util/rawbuf.h> namespace search::docsummary { + +class ResultConfig; + /** * An Object of this class may be used to create docsum blobs. A * single blob is created by first indicating what result class the @@ -27,11 +32,8 @@ private: const ResConfigEntry *_cfgEntry; // current field of current blob bool _error; // error flag for current blob - static const char *GetResTypeName(ResType type) - { return ResultConfig::GetResTypeName(type); } - - static bool IsBinaryCompatible(ResType a, ResType b) - { return ResultConfig::IsBinaryCompatible(a, b); } + static const char *GetResTypeName(ResType type) { return ResTypeUtils::GetResTypeName(type); } + static bool IsBinaryCompatible(ResType a, ResType b) { return ResTypeUtils::IsBinaryCompatible(a, b); } void WarnType(ResType type) const; void SetFormatError(ResType type); diff --git a/searchsummary/src/vespa/searchsummary/docsummary/simple_dfw.cpp b/searchsummary/src/vespa/searchsummary/docsummary/simple_dfw.cpp new file mode 100644 index 00000000000..8cc872378bd --- /dev/null +++ b/searchsummary/src/vespa/searchsummary/docsummary/simple_dfw.cpp @@ -0,0 +1,13 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include "simple_dfw.h" + +namespace search::docsummary { + +void +SimpleDFW::insertField(uint32_t docid, GeneralResult *, GetDocsumsState *state, ResType type, vespalib::slime::Inserter &target) +{ + insertField(docid, state, type, target); +} + +} diff --git a/searchsummary/src/vespa/searchsummary/docsummary/simple_dfw.h b/searchsummary/src/vespa/searchsummary/docsummary/simple_dfw.h new file mode 100644 index 00000000000..abebb7de2b8 --- /dev/null +++ b/searchsummary/src/vespa/searchsummary/docsummary/simple_dfw.h @@ -0,0 +1,20 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#pragma once + +#include "docsum_field_writer.h" + +namespace search::docsummary { + +/* + * Abstract class for writing document summaries that don't need + * access to a document retrieved from IDocsumStore. + */ +class SimpleDFW : public DocsumFieldWriter +{ +public: + virtual void insertField(uint32_t docid, GetDocsumsState *state, ResType type, vespalib::slime::Inserter &target) = 0; + void insertField(uint32_t docid, GeneralResult *, GetDocsumsState *state, ResType type, vespalib::slime::Inserter &target) override; +}; + +} diff --git a/searchsummary/src/vespa/searchsummary/docsummary/struct_map_attribute_combiner_dfw.cpp b/searchsummary/src/vespa/searchsummary/docsummary/struct_map_attribute_combiner_dfw.cpp index aec55977546..38a9cc8c50b 100644 --- a/searchsummary/src/vespa/searchsummary/docsummary/struct_map_attribute_combiner_dfw.cpp +++ b/searchsummary/src/vespa/searchsummary/docsummary/struct_map_attribute_combiner_dfw.cpp @@ -9,6 +9,7 @@ #include <vespa/searchlib/common/matching_elements.h> #include <vespa/searchlib/common/matching_elements_fields.h> #include <vespa/vespalib/data/slime/cursor.h> +#include <vespa/vespalib/data/slime/inserter.h> #include <vespa/vespalib/util/stash.h> #include <algorithm> #include <cassert> diff --git a/searchsummary/src/vespa/searchsummary/docsummary/summaryfeaturesdfw.cpp b/searchsummary/src/vespa/searchsummary/docsummary/summaryfeaturesdfw.cpp index f920b7be0cd..c28e986612c 100644 --- a/searchsummary/src/vespa/searchsummary/docsummary/summaryfeaturesdfw.cpp +++ b/searchsummary/src/vespa/searchsummary/docsummary/summaryfeaturesdfw.cpp @@ -3,6 +3,7 @@ #include "summaryfeaturesdfw.h" #include "docsumstate.h" #include <vespa/vespalib/data/slime/cursor.h> +#include <vespa/vespalib/data/slime/inserter.h> #include <vespa/log/log.h> LOG_SETUP(".searchlib.docsummary.summaryfeaturesdfw"); diff --git a/searchsummary/src/vespa/searchsummary/docsummary/summaryfeaturesdfw.h b/searchsummary/src/vespa/searchsummary/docsummary/summaryfeaturesdfw.h index 6c4084b0221..d12feb69182 100644 --- a/searchsummary/src/vespa/searchsummary/docsummary/summaryfeaturesdfw.h +++ b/searchsummary/src/vespa/searchsummary/docsummary/summaryfeaturesdfw.h @@ -2,13 +2,13 @@ #pragma once -#include "docsumfieldwriter.h" +#include "simple_dfw.h" namespace search::docsummary { class IDocsumEnvironment; -class SummaryFeaturesDFW : public ISimpleDFW +class SummaryFeaturesDFW : public SimpleDFW { private: IDocsumEnvironment * _env; diff --git a/searchsummary/src/vespa/searchsummary/docsummary/textextractordfw.cpp b/searchsummary/src/vespa/searchsummary/docsummary/textextractordfw.cpp index d12418dc254..dc6d9524ee4 100644 --- a/searchsummary/src/vespa/searchsummary/docsummary/textextractordfw.cpp +++ b/searchsummary/src/vespa/searchsummary/docsummary/textextractordfw.cpp @@ -1,8 +1,11 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "textextractordfw.h" -#include "tokenizer.h" #include "docsumstate.h" +#include "general_result.h" +#include "tokenizer.h" +#include "resultconfig.h" +#include <vespa/vespalib/data/slime/inserter.h> #include <vespa/log/log.h> LOG_SETUP(".searchlib.docsummary.textextractordfw"); diff --git a/searchsummary/src/vespa/searchsummary/docsummary/textextractordfw.h b/searchsummary/src/vespa/searchsummary/docsummary/textextractordfw.h index 10764e5c21d..3bce2ae5cd7 100644 --- a/searchsummary/src/vespa/searchsummary/docsummary/textextractordfw.h +++ b/searchsummary/src/vespa/searchsummary/docsummary/textextractordfw.h @@ -2,14 +2,16 @@ #pragma once -#include "docsumfieldwriter.h" +#include "docsum_field_writer.h" namespace search::docsummary { +class ResultConfig; + /** * This is the docsum field writer used to extract the original text from a disk summary on the juniper format. **/ -class TextExtractorDFW : public IDocsumFieldWriter +class TextExtractorDFW : public DocsumFieldWriter { private: TextExtractorDFW(const TextExtractorDFW &); @@ -19,7 +21,7 @@ private: public: TextExtractorDFW(); - ~TextExtractorDFW() {} + ~TextExtractorDFW() override = default; bool init(const vespalib::string & fieldName, const vespalib::string & inputField, const ResultConfig & config); bool IsGenerated() const override { return false; } void insertField(uint32_t docid, GeneralResult *gres, GetDocsumsState *state, diff --git a/storage/src/tests/persistence/common/persistenceproviderwrapper.cpp b/storage/src/tests/persistence/common/persistenceproviderwrapper.cpp index 7e0b96b1d82..fcb56c4a553 100644 --- a/storage/src/tests/persistence/common/persistenceproviderwrapper.cpp +++ b/storage/src/tests/persistence/common/persistenceproviderwrapper.cpp @@ -106,11 +106,11 @@ PersistenceProviderWrapper::putAsync(const spi::Bucket& bucket, spi::Timestamp t } void -PersistenceProviderWrapper::removeAsync(const spi::Bucket& bucket, std::vector<TimeStampAndDocumentId> ids, +PersistenceProviderWrapper::removeAsync(const spi::Bucket& bucket, std::vector<spi::IdAndTimestamp> ids, spi::OperationComplete::UP onComplete) { - for (const TimeStampAndDocumentId & stampedId : ids) { - LOG_SPI("remove(" << bucket << ", " << stampedId.first << ", " << stampedId.second << ")"); + for (const auto & stampedId : ids) { + LOG_SPI("remove(" << bucket << ", " << stampedId.timestamp << ", " << stampedId.id << ")"); } CHECK_ERROR_ASYNC(spi::RemoveResult, FAIL_REMOVE, onComplete); _spi.removeAsync(bucket, std::move(ids), std::move(onComplete)); diff --git a/storage/src/tests/persistence/common/persistenceproviderwrapper.h b/storage/src/tests/persistence/common/persistenceproviderwrapper.h index 3c93bc91d85..ec7ca70d7c7 100644 --- a/storage/src/tests/persistence/common/persistenceproviderwrapper.h +++ b/storage/src/tests/persistence/common/persistenceproviderwrapper.h @@ -106,7 +106,7 @@ public: spi::BucketIdListResult listBuckets(BucketSpace bucketSpace) const override; spi::BucketInfoResult getBucketInfo(const spi::Bucket&) const override; void putAsync(const spi::Bucket&, spi::Timestamp, spi::DocumentSP, spi::OperationComplete::UP) override; - void removeAsync(const spi::Bucket&, std::vector<TimeStampAndDocumentId> ids, spi::OperationComplete::UP) override; + void removeAsync(const spi::Bucket&, std::vector<spi::IdAndTimestamp> ids, spi::OperationComplete::UP) override; void removeIfFoundAsync(const spi::Bucket&, spi::Timestamp, const spi::DocumentId&, spi::OperationComplete::UP) override; void updateAsync(const spi::Bucket&, spi::Timestamp, spi::DocumentUpdateSP, spi::OperationComplete::UP) override; spi::GetResult get(const spi::Bucket&, const document::FieldSet&, const spi::DocumentId&, spi::Context&) const override; diff --git a/storage/src/vespa/storage/config/stor-communicationmanager.def b/storage/src/vespa/storage/config/stor-communicationmanager.def index 75a3344b618..3083fb37081 100644 --- a/storage/src/vespa/storage/config/stor-communicationmanager.def +++ b/storage/src/vespa/storage/config/stor-communicationmanager.def @@ -38,20 +38,17 @@ mbus.tcp_no_delay bool default=true restart ## Number of threads for network. mbus.num_network_threads int default=1 restart -## Number of workers threads for messagebus -## Any value below 1 will be 1. -mbus.num_threads int default=4 restart - ## The number of events in the queue of a network (FNET) thread before it is woken up. mbus.events_before_wakeup int default=1 restart ## Enable to use above thread pool for encoding replies ## False will use network(fnet) thread +## Deprecated and void mbus.dispatch_on_encode bool default=true restart ## Enable to use above thread pool for decoding replies ## False will use network(fnet) thread -## Todo: Change default once verified in large scale deployment. +## Deprecated and void mbus.dispatch_on_decode bool default=true restart ## The number of network (FNET) threads used by the shared rpc resource. diff --git a/storage/src/vespa/storage/persistence/asynchandler.cpp b/storage/src/vespa/storage/persistence/asynchandler.cpp index f5d29fb32a7..d5bf733a30c 100644 --- a/storage/src/vespa/storage/persistence/asynchandler.cpp +++ b/storage/src/vespa/storage/persistence/asynchandler.cpp @@ -114,13 +114,13 @@ bucketStatesAreSemanticallyEqual(const api::BucketInfo& a, const api::BucketInfo class UnrevertableRemoveEntryProcessor : public BucketProcessor::EntryProcessor { public: - using DocumentIdsAndTimeStamps = std::vector<std::pair<spi::Timestamp, spi::DocumentId>>; + using DocumentIdsAndTimeStamps = std::vector<spi::IdAndTimestamp>; UnrevertableRemoveEntryProcessor(DocumentIdsAndTimeStamps & to_remove) : _to_remove(to_remove) {} void process(spi::DocEntry& entry) override { - _to_remove.emplace_back(entry.getTimestamp(), *entry.getDocumentId()); + _to_remove.emplace_back(*entry.getDocumentId(), entry.getTimestamp()); } private: DocumentIdsAndTimeStamps & _to_remove; diff --git a/storage/src/vespa/storage/persistence/mergehandler.cpp b/storage/src/vespa/storage/persistence/mergehandler.cpp index 012d5c2619d..ae68a694c90 100644 --- a/storage/src/vespa/storage/persistence/mergehandler.cpp +++ b/storage/src/vespa/storage/persistence/mergehandler.cpp @@ -522,9 +522,9 @@ MergeHandler::applyDiffEntry(std::shared_ptr<ApplyBucketDiffState> async_results _clock, _env._metrics.merge_handler_metrics.put_latency); _spi.putAsync(bucket, timestamp, std::move(doc), std::move(complete)); } else { - std::vector<spi::PersistenceProvider::TimeStampAndDocumentId> ids; - ids.emplace_back(timestamp, e._docName); - auto complete = std::make_unique<ApplyBucketDiffEntryComplete>(std::move(async_results), ids[0].second, + std::vector<spi::IdAndTimestamp> ids; + ids.emplace_back(document::DocumentId(e._docName), timestamp); + auto complete = std::make_unique<ApplyBucketDiffEntryComplete>(std::move(async_results), ids[0].id, std::move(throttle_token), "remove", _clock, _env._metrics.merge_handler_metrics.remove_latency); _spi.removeAsync(bucket, std::move(ids), std::move(complete)); diff --git a/storage/src/vespa/storage/persistence/provider_error_wrapper.cpp b/storage/src/vespa/storage/persistence/provider_error_wrapper.cpp index 1be9679c641..9e55c0f9088 100644 --- a/storage/src/vespa/storage/persistence/provider_error_wrapper.cpp +++ b/storage/src/vespa/storage/persistence/provider_error_wrapper.cpp @@ -152,7 +152,7 @@ ProviderErrorWrapper::putAsync(const spi::Bucket &bucket, spi::Timestamp ts, spi } void -ProviderErrorWrapper::removeAsync(const spi::Bucket &bucket, std::vector<TimeStampAndDocumentId> ids, +ProviderErrorWrapper::removeAsync(const spi::Bucket &bucket, std::vector<spi::IdAndTimestamp> ids, spi::OperationComplete::UP onComplete) { onComplete->addResultHandler(this); diff --git a/storage/src/vespa/storage/persistence/provider_error_wrapper.h b/storage/src/vespa/storage/persistence/provider_error_wrapper.h index 7bd406a8758..82447fe4549 100644 --- a/storage/src/vespa/storage/persistence/provider_error_wrapper.h +++ b/storage/src/vespa/storage/persistence/provider_error_wrapper.h @@ -58,7 +58,7 @@ public: void register_error_listener(std::shared_ptr<ProviderErrorListener> listener); void putAsync(const spi::Bucket &, spi::Timestamp, spi::DocumentSP, spi::OperationComplete::UP) override; - void removeAsync(const spi::Bucket&, std::vector<TimeStampAndDocumentId>, spi::OperationComplete::UP) override; + void removeAsync(const spi::Bucket&, std::vector<spi::IdAndTimestamp>, spi::OperationComplete::UP) override; void removeIfFoundAsync(const spi::Bucket&, spi::Timestamp, const document::DocumentId&, spi::OperationComplete::UP) override; void updateAsync(const spi::Bucket &, spi::Timestamp, spi::DocumentUpdateSP, spi::OperationComplete::UP) override; void setActiveStateAsync(const spi::Bucket& b, spi::BucketInfo::ActiveState newState, spi::OperationComplete::UP onComplete) override; diff --git a/storage/src/vespa/storage/storageserver/communicationmanager.cpp b/storage/src/vespa/storage/storageserver/communicationmanager.cpp index 16ee59bffb1..7fb6685a8b5 100644 --- a/storage/src/vespa/storage/storageserver/communicationmanager.cpp +++ b/storage/src/vespa/storage/storageserver/communicationmanager.cpp @@ -352,7 +352,6 @@ void CommunicationManager::configure(std::unique_ptr<CommunicationManagerConfig> LOG(debug, "setting up slobrok config from id: '%s", _configUri.getConfigId().c_str()); mbus::RPCNetworkParams params(_configUri); params.setConnectionExpireSecs(config->mbus.rpctargetcache.ttl); - params.setNumThreads(std::max(1, config->mbus.numThreads)); params.setNumNetworkThreads(std::max(1, config->mbus.numNetworkThreads)); params.setNumRpcTargets(std::max(1, config->mbus.numRpcTargets)); params.events_before_wakeup(std::max(1, config->mbus.eventsBeforeWakeup)); diff --git a/streamingvisitors/src/vespa/vsm/vsm/docsumconfig.cpp b/streamingvisitors/src/vespa/vsm/vsm/docsumconfig.cpp index ce0706a03fb..fac801550d4 100644 --- a/streamingvisitors/src/vespa/vsm/vsm/docsumconfig.cpp +++ b/streamingvisitors/src/vespa/vsm/vsm/docsumconfig.cpp @@ -1,14 +1,16 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include <vespa/vsm/vsm/docsumconfig.h> -#include <vespa/searchsummary/docsummary/docsumfieldwriter.h> +#include <vespa/searchsummary/docsummary/copy_dfw.h> +#include <vespa/searchsummary/docsummary/empty_dfw.h> #include <vespa/searchsummary/docsummary/matched_elements_filter_dfw.h> +#include <vespa/searchsummary/docsummary/resultconfig.h> #include <vespa/searchlib/common/matching_elements_fields.h> #include <vespa/vsm/config/config-vsmfields.h> #include <vespa/vsm/config/config-vsmsummary.h> using search::MatchingElementsFields; -using search::docsummary::IDocsumFieldWriter; +using search::docsummary::DocsumFieldWriter; using search::docsummary::CopyDFW; using search::docsummary::EmptyDFW; using search::docsummary::MatchedElementsFilterDFW; @@ -41,10 +43,10 @@ DynamicDocsumConfig::DynamicDocsumConfig(search::docsummary::IDocsumEnvironment* { } -IDocsumFieldWriter::UP +std::unique_ptr<DocsumFieldWriter> DynamicDocsumConfig::createFieldWriter(const string & fieldName, const string & overrideName, const string & argument, bool & rc, std::shared_ptr<search::MatchingElementsFields> matching_elems_fields) { - IDocsumFieldWriter::UP fieldWriter; + std::unique_ptr<DocsumFieldWriter> fieldWriter; if ((overrideName == "staticrank") || (overrideName == "ranklog") || (overrideName == "label") || diff --git a/streamingvisitors/src/vespa/vsm/vsm/docsumconfig.h b/streamingvisitors/src/vespa/vsm/vsm/docsumconfig.h index 11010c04e90..a660c544d7d 100644 --- a/streamingvisitors/src/vespa/vsm/vsm/docsumconfig.h +++ b/streamingvisitors/src/vespa/vsm/vsm/docsumconfig.h @@ -20,7 +20,7 @@ private: public: DynamicDocsumConfig(search::docsummary::IDocsumEnvironment* env, search::docsummary::DynamicDocsumWriter* writer, std::shared_ptr<VsmfieldsConfig> vsm_fields_config); private: - std::unique_ptr<search::docsummary::IDocsumFieldWriter> + std::unique_ptr<search::docsummary::DocsumFieldWriter> createFieldWriter(const string & fieldName, const string & overrideName, const string & cf, bool & rc, std::shared_ptr<search::MatchingElementsFields> matching_elems_fields) override; }; diff --git a/streamingvisitors/src/vespa/vsm/vsm/vsm-adapter.cpp b/streamingvisitors/src/vespa/vsm/vsm/vsm-adapter.cpp index 14f44527887..0d329cd9783 100644 --- a/streamingvisitors/src/vespa/vsm/vsm/vsm-adapter.cpp +++ b/streamingvisitors/src/vespa/vsm/vsm/vsm-adapter.cpp @@ -138,6 +138,7 @@ VSMAdapter::configure(const VSMConfigSnapshot & snapshot) // init result config DocsumBlobEntryFilter docsum_blob_entry_filter; docsum_blob_entry_filter.add_skip(search::docsummary::RES_INT); + docsum_blob_entry_filter.add_skip(search::docsummary::RES_TENSOR); std::unique_ptr<ResultConfig> resCfg(new ResultConfig(docsum_blob_entry_filter)); if ( ! resCfg->ReadConfig(*summary.get(), _configId.c_str())) { throw std::runtime_error("(re-)configuration of VSM (docsum tools) failed due to bad summary config"); diff --git a/vdslib/src/test/java/com/yahoo/vdslib/distribution/DistributionTestFactory.java b/vdslib/src/test/java/com/yahoo/vdslib/distribution/DistributionTestFactory.java index 084d0e9185d..915f8dd67d5 100644 --- a/vdslib/src/test/java/com/yahoo/vdslib/distribution/DistributionTestFactory.java +++ b/vdslib/src/test/java/com/yahoo/vdslib/distribution/DistributionTestFactory.java @@ -108,6 +108,7 @@ public class DistributionTestFactory extends CrossPlatformTestFactory { } } + @SuppressWarnings("deprecation") private static StorDistributionConfig.Builder deserializeConfig(String s) { return new StorDistributionConfig.Builder( new ConfigGetter<>(StorDistributionConfig.class).getConfig("raw:" + s)); diff --git a/vespa-osgi-testrunner/pom.xml b/vespa-osgi-testrunner/pom.xml index ebb1240a198..0032fe33de0 100644 --- a/vespa-osgi-testrunner/pom.xml +++ b/vespa-osgi-testrunner/pom.xml @@ -80,6 +80,11 @@ <artifactId>org.apache.felix.framework</artifactId> <scope>provided</scope> </dependency> + <dependency> + <groupId>com.fasterxml.jackson.core</groupId> + <artifactId>jackson-core</artifactId> + <scope>provided</scope> + </dependency> </dependencies> <build> diff --git a/vespa-osgi-testrunner/src/main/java/com/yahoo/vespa/testrunner/TestRunnerHandler.java b/vespa-osgi-testrunner/src/main/java/com/yahoo/vespa/testrunner/TestRunnerHandler.java index aa51971583b..ef4b402d33b 100644 --- a/vespa-osgi-testrunner/src/main/java/com/yahoo/vespa/testrunner/TestRunnerHandler.java +++ b/vespa-osgi-testrunner/src/main/java/com/yahoo/vespa/testrunner/TestRunnerHandler.java @@ -1,6 +1,8 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.testrunner; +import com.fasterxml.jackson.core.JsonFactory; +import com.fasterxml.jackson.core.JsonGenerator; import com.yahoo.component.annotation.Inject; import com.yahoo.component.provider.ComponentRegistry; import com.yahoo.container.jdisc.EmptyResponse; @@ -40,6 +42,7 @@ import static java.nio.charset.StandardCharsets.UTF_8; public class TestRunnerHandler extends ThreadedHttpRequestHandler { private static final DateTimeFormatter formatter = DateTimeFormatter.ofPattern("HH:mm:ss.SSS"); + private static final JsonFactory factory = new JsonFactory(); private final TestRunner testRunner; @@ -109,9 +112,10 @@ public class TestRunnerHandler extends ThreadedHttpRequestHandler { return path.substring(lastSlash + 1); } - static void render(OutputStream out, Collection<LogRecord> log) throws IOException { - out.write("{\"logRecords\":[".getBytes(UTF_8)); - boolean first = true; + private static void render(OutputStream out, Collection<LogRecord> log) throws IOException { + var json = factory.createGenerator(out); + json.writeStartObject(); + json.writeArrayFieldStart("logRecords"); for (LogRecord record : log) { String message = record.getMessage() == null ? "" : record.getMessage(); if (record.getThrown() != null) { @@ -119,20 +123,19 @@ public class TestRunnerHandler extends ThreadedHttpRequestHandler { record.getThrown().printStackTrace(new PrintStream(buffer)); message += (message.isEmpty() ? "" : "\n") + buffer; } - - if (first) first = false; - else out.write(','); - out.write(""" - {"id":%d,"at":%d,"type":"%s","message":"%s"} - """.formatted(record.getSequenceNumber(), - record.getMillis(), - typeOf(record.getLevel()), - message).getBytes(UTF_8)); + json.writeStartObject(); + json.writeNumberField("id", record.getSequenceNumber()); + json.writeNumberField("at", record.getMillis()); + json.writeStringField("type", typeOf(record.getLevel())); + json.writeStringField("message", message); + json.writeEndObject(); } - out.write("]}".getBytes(UTF_8)); + json.writeEndArray(); + json.writeEndObject(); + json.close(); } - public static String typeOf(Level level) { + private static String typeOf(Level level) { return level.getName().equals("html") ? "html" : level.intValue() < Level.INFO.intValue() ? "debug" : level.intValue() < Level.WARNING.intValue() ? "info" @@ -140,116 +143,105 @@ public class TestRunnerHandler extends ThreadedHttpRequestHandler { : "error"; } - static void render(OutputStream out, TestReport report) throws IOException { - out.write('{'); + private static void render(OutputStream out, TestReport report) throws IOException { + JsonGenerator json = factory.createGenerator(out); + json.writeStartObject(); - out.write("\"report\":".getBytes(UTF_8)); - render(out, (Node) report.root()); + json.writeFieldName("report"); + render(json, (Node) report.root()); // TODO jonmv: remove - out.write(",\"summary\":{".getBytes(UTF_8)); + json.writeObjectFieldStart("summary"); - renderSummary(out, report); + renderSummary(json, report); - out.write(",\"failures\":[".getBytes(UTF_8)); - renderFailures(out, report.root(), true); - out.write("]".getBytes(UTF_8)); + json.writeArrayFieldStart("failures"); + renderFailures(json, report.root()); + json.writeEndArray(); - out.write("}".getBytes(UTF_8)); + json.writeEndObject(); // TODO jonmv: remove - out.write(",\"output\":[".getBytes(UTF_8)); - renderOutput(out, report.root(), true); - out.write("]".getBytes(UTF_8)); + json.writeArrayFieldStart("output"); + renderOutput(json, report.root()); + json.writeEndArray(); - out.write('}'); + json.writeEndObject(); + json.close(); } - static void renderSummary(OutputStream out, TestReport report) throws IOException { + private static void renderSummary(JsonGenerator json, TestReport report) throws IOException { Map<TestReport.Status, Long> tally = report.root().tally(); - out.write(""" - "success":%d,"failed":%d,"ignored":%d,"aborted":%d,"inconclusive":%d - """.formatted(tally.getOrDefault(TestReport.Status.successful, 0L), - tally.getOrDefault(TestReport.Status.failed, 0L) + tally.getOrDefault(TestReport.Status.error, 0L), - tally.getOrDefault(TestReport.Status.skipped, 0L), - tally.getOrDefault(TestReport.Status.aborted, 0L), - tally.getOrDefault(TestReport.Status.inconclusive, 0L)).getBytes(UTF_8)); + json.writeNumberField("success", tally.getOrDefault(TestReport.Status.successful, 0L)); + json.writeNumberField("failed", tally.getOrDefault(TestReport.Status.failed, 0L) + tally.getOrDefault(TestReport.Status.error, 0L)); + json.writeNumberField("ignored", tally.getOrDefault(TestReport.Status.skipped, 0L)); + json.writeNumberField("aborted", tally.getOrDefault(TestReport.Status.aborted, 0L)); + json.writeNumberField("inconclusive", tally.getOrDefault(TestReport.Status.inconclusive, 0L)); } - static boolean renderFailures(OutputStream out, Node node, boolean first) throws IOException { + private static void renderFailures(JsonGenerator json, Node node) throws IOException { if (node instanceof FailureNode) { - if (first) first = false; - else out.write(','); - String message = ((FailureNode) node).thrown().getMessage(); - out.write(""" - {"testName":"%s","testError":%s,"exception":"%s"} - """.formatted(node.parent.name(), - message == null ? null : '"' + message + '"', - ExceptionUtils.getStackTraceAsString(((FailureNode) node).thrown())).getBytes(UTF_8)); + json.writeStartObject(); + json.writeStringField("testName", node.parent.name()); + json.writeStringField("testError", ((FailureNode) node).thrown().getMessage()); + json.writeStringField("exception", ExceptionUtils.getStackTraceAsString(((FailureNode) node).thrown())); + json.writeEndObject(); } else { for (Node child : node.children()) - first = renderFailures(out, child, first); + renderFailures(json, child); } - return first; } - static boolean renderOutput(OutputStream out, Node node, boolean first) throws IOException { + private static void renderOutput(JsonGenerator json, Node node) throws IOException { if (node instanceof OutputNode) { for (LogRecord record : ((OutputNode) node).log()) - if (record.getMessage() != null) { - if (first) first = false; - else out.write(','); - out.write(('"' + formatter.format(record.getInstant().atOffset(ZoneOffset.UTC)) + " " + record.getMessage() + '"').getBytes(UTF_8)); - } + if (record.getMessage() != null) + json.writeString(formatter.format(record.getInstant().atOffset(ZoneOffset.UTC)) + " " + record.getMessage()); } else { for (Node child : node.children()) - first = renderOutput(out, child, first); + renderOutput(json, child); } - return first; } - static void render(OutputStream out, Node node) throws IOException { - out.write('{'); - if (node instanceof NamedNode) render(out, (NamedNode) node); - if (node instanceof OutputNode) render(out, (OutputNode) node); + private static void render(JsonGenerator json, Node node) throws IOException { + json.writeStartObject(); + if (node instanceof NamedNode) render(json, (NamedNode) node); + if (node instanceof OutputNode) render(json, (OutputNode) node); if ( ! node.children().isEmpty()) { - out.write(",\"children\":[".getBytes(UTF_8)); - boolean first = true; + json.writeArrayFieldStart("children"); for (Node child : node.children) { - if (first) first = false; - else out.write(','); - render(out, child); + render(json, child); } - out.write(']'); + json.writeEndArray(); } - out.write('}'); + json.writeEndObject(); } - static void render(OutputStream out, NamedNode node) throws IOException { + private static void render(JsonGenerator json, NamedNode node) throws IOException { String type = node instanceof FailureNode ? "failure" : node instanceof TestNode ? "test" : "container"; - out.write(""" - "type":"%s","name":"%s","status":"%s","start":%d,"duration":%d - """.formatted(type,node.name(), node.status().name(), node.start().toEpochMilli(), node.duration().toMillis()).getBytes(UTF_8)); + json.writeStringField("type", type); + json.writeStringField("name", node.name()); + json.writeStringField("status", node.status().name()); + json.writeNumberField("start", node.start().toEpochMilli()); + json.writeNumberField("duration", node.duration().toMillis()); } - static void render(OutputStream out, OutputNode node) throws IOException { - out.write("\"type\":\"output\",\"children\":[".getBytes(UTF_8)); - boolean first = true; + private static void render(JsonGenerator json, OutputNode node) throws IOException { + json.writeStringField("type", "output"); + json.writeArrayFieldStart("children"); for (LogRecord record : node.log()) { - if (first) first = false; - else out.write(','); - out.write(""" - {"message":"%s","at":%d,"level":"%s"} - """.formatted((record.getLoggerName() == null ? "" : record.getLoggerName() + ": ") + - (record.getMessage() != null ? record.getMessage() : "") + - (record.getThrown() != null ? (record.getMessage() != null ? "\n" : "") + traceToString(record.getThrown()) : ""), - record.getInstant().toEpochMilli(), - typeOf(record.getLevel())).getBytes(UTF_8)); + json.writeStartObject(); + json.writeStringField("message", (record.getLoggerName() == null ? "" : record.getLoggerName() + ": ") + + (record.getMessage() != null ? record.getMessage() : "") + + (record.getThrown() != null ? (record.getMessage() != null ? "\n" : "") + traceToString(record.getThrown()) : "")); + json.writeNumberField("at", record.getInstant().toEpochMilli()); + json.writeStringField("level", typeOf(record.getLevel())); + json.writeEndObject(); } - out.write(']'); + json.writeEndArray(); } private static String traceToString(Throwable thrown) { @@ -258,17 +250,17 @@ public class TestRunnerHandler extends ThreadedHttpRequestHandler { return buffer.toString(UTF_8); } - interface Renderer { + private interface Renderer { void render(OutputStream out) throws IOException; } - static class CustomJsonResponse extends HttpResponse { + private static class CustomJsonResponse extends HttpResponse { private final Renderer renderer; - CustomJsonResponse(Renderer renderer) { + private CustomJsonResponse(Renderer renderer) { super(200); this.renderer = renderer; } diff --git a/vespa-osgi-testrunner/src/test/java/com/yahoo/vespa/test/samples/SampleTest.java b/vespa-osgi-testrunner/src/test/java/com/yahoo/vespa/test/samples/SampleTest.java index bc878353d4b..b0e5119c06e 100644 --- a/vespa-osgi-testrunner/src/test/java/com/yahoo/vespa/test/samples/SampleTest.java +++ b/vespa-osgi-testrunner/src/test/java/com/yahoo/vespa/test/samples/SampleTest.java @@ -71,7 +71,7 @@ public class SampleTest { @Test void successful() { log.log(new Level("html", INFO.intValue()) { }, "<body />"); - log.log(INFO, "Very informative"); + log.log(INFO, "Very informative: \"\\n\": \n"); log.log(WARNING, "Oh no", new IllegalArgumentException("error", new RuntimeException("wrapped"))); } diff --git a/vespa-osgi-testrunner/src/test/java/com/yahoo/vespa/testrunner/TestRunnerHandlerTest.java b/vespa-osgi-testrunner/src/test/java/com/yahoo/vespa/testrunner/TestRunnerHandlerTest.java index 5ce737d7649..6d6fbbf2cf1 100644 --- a/vespa-osgi-testrunner/src/test/java/com/yahoo/vespa/testrunner/TestRunnerHandlerTest.java +++ b/vespa-osgi-testrunner/src/test/java/com/yahoo/vespa/testrunner/TestRunnerHandlerTest.java @@ -27,7 +27,6 @@ import java.util.Collection; import java.util.List; import java.util.concurrent.CompletableFuture; import java.util.concurrent.Executors; -import java.util.logging.Level; import java.util.logging.LogRecord; import static com.yahoo.jdisc.http.HttpRequest.Method.GET; @@ -136,14 +135,6 @@ class TestRunnerHandlerTest { } } - /* Creates a LogRecord that has a known instant and sequence number to get predictable serialization results. */ - private static LogRecord logRecord(String logMessage) { - LogRecord logRecord = new LogRecord(Level.INFO, logMessage); - logRecord.setInstant(testInstant); - logRecord.setSequenceNumber(0); - return logRecord; - } - private static class MockRunner implements TestRunner { private final TestRunner.Status status; diff --git a/vespa-osgi-testrunner/src/test/resources/output.json b/vespa-osgi-testrunner/src/test/resources/output.json index 847ae5800e9..2b4aa9e5599 100644 --- a/vespa-osgi-testrunner/src/test/resources/output.json +++ b/vespa-osgi-testrunner/src/test/resources/output.json @@ -1,133 +1,133 @@ { "logRecords": [ { - "id": 18, + "id": 2, "at": 0, "type": "info", "message": "spam" }, { - "id": 21, + "id": 5, "at": 0, "type": "info", "message": "spam" }, { - "id": 22, + "id": 6, "at": 0, "type": "error", "message": "java.lang.NoClassDefFoundError\n\tat com.yahoo.vespa.test.samples.SampleTest.error(SampleTest.java:87)\n" }, { - "id": 25, + "id": 9, "at": 0, "type": "info", "message": "spam" }, { - "id": 26, + "id": 10, "at": 0, "type": "info", "message": "I have a bad feeling about this" }, { - "id": 27, + "id": 11, "at": 0, "type": "error", "message": "org.opentest4j.AssertionFailedError: baz ==> expected: <foo> but was: <bar>\n\tat org.junit.jupiter.api.AssertionUtils.fail(AssertionUtils.java:55)\n\tat org.junit.jupiter.api.AssertionUtils.failNotEqual(AssertionUtils.java:62)\n\tat org.junit.jupiter.api.AssertEquals.assertEquals(AssertEquals.java:182)\n\tat org.junit.jupiter.api.Assertions.assertEquals(Assertions.java:1152)\n\tat com.yahoo.vespa.test.samples.SampleTest.failing(SampleTest.java:81)\n" }, { - "id": 31, + "id": 15, "at": 0, "type": "info", "message": "spam" }, { - "id": 32, + "id": 16, "at": 0, "type": "info", "message": "I'm here with Erwin today; Erwin, what can you tell us about your cat?" }, { - "id": 33, + "id": 17, "at": 0, "type": "warning", "message": "ai.vespa.hosted.cd.InconclusiveTestException: the cat is both dead _and_ alive\n\tat com.yahoo.vespa.test.samples.SampleTest.inconclusive(SampleTest.java:93)\n" }, { - "id": 36, + "id": 20, "at": 0, "type": "info", "message": "spam" }, { - "id": 37, + "id": 21, "at": 0, "type": "info", "message": "<body />" }, { - "id": 38, + "id": 22, "at": 0, "type": "info", - "message": "Very informative" + "message": "Very informative: \"\\n\": \n" }, { - "id": 39, + "id": 23, "at": 0, "type": "warning", "message": "Oh no\njava.lang.IllegalArgumentException: error\n\tat com.yahoo.vespa.test.samples.SampleTest.successful(SampleTest.java:75)\nCaused by: java.lang.RuntimeException: wrapped\n\t... 1 more\n" }, { - "id": 43, + "id": 27, "at": 0, "type": "info", "message": "spam" }, { - "id": 46, + "id": 30, "at": 0, "type": "info", "message": "Catch me if you can!" }, { - "id": 50, + "id": 34, "at": 0, "type": "error", "message": "org.opentest4j.AssertionFailedError: no charm\n\tat org.junit.jupiter.api.AssertionUtils.fail(AssertionUtils.java:39)\n\tat org.junit.jupiter.api.Assertions.fail(Assertions.java:134)\n\tat com.yahoo.vespa.test.samples.SampleTest$Inner.lambda$others$1(SampleTest.java:105)\n" }, { - "id": 54, + "id": 38, "at": 0, "type": "info", "message": "spam" }, { - "id": 2, + "id": 67, "at": 0, "type": "error", "message": "org.opentest4j.AssertionFailedError\n\tat org.junit.jupiter.api.AssertionUtils.fail(AssertionUtils.java:35)\n\tat org.junit.jupiter.api.Assertions.fail(Assertions.java:115)\n\tat com.yahoo.vespa.test.samples.FailingTestAndBothAftersTest.test(FailingTestAndBothAftersTest.java:19)\n\tSuppressed: java.lang.RuntimeException\n\t\tat com.yahoo.vespa.test.samples.FailingTestAndBothAftersTest.moreFail(FailingTestAndBothAftersTest.java:16)\n" }, { - "id": 4, + "id": 69, "at": 0, "type": "error", "message": "java.lang.RuntimeException\n\tat com.yahoo.vespa.test.samples.FailingTestAndBothAftersTest.fail(FailingTestAndBothAftersTest.java:13)\n" }, { - "id": 7, + "id": 72, "at": 0, "type": "error", "message": "org.junit.platform.commons.JUnitException: @BeforeAll method 'void com.yahoo.vespa.test.samples.WrongBeforeAllTest.wrong()' must be static unless the test class is annotated with @TestInstance(Lifecycle.PER_CLASS).\n" }, { - "id": 11, + "id": 76, "at": 0, "type": "error", "message": "java.lang.NullPointerException\n\tat com.yahoo.vespa.test.samples.FailingExtensionTest$FailingExtension.<init>(FailingExtensionTest.java:19)\n" }, { - "id": 15, + "id": 80, "at": 12000, "type": "error", "message": "java.lang.ClassNotFoundException: School's out all summer!\n" diff --git a/vespa-osgi-testrunner/src/test/resources/report.json b/vespa-osgi-testrunner/src/test/resources/report.json index fa76c222f93..443694e2e0c 100644 --- a/vespa-osgi-testrunner/src/test/resources/report.json +++ b/vespa-osgi-testrunner/src/test/resources/report.json @@ -191,7 +191,7 @@ "level": "info" }, { - "message": "com.yahoo.vespa.test.samples.SampleTest: Very informative", + "message": "com.yahoo.vespa.test.samples.SampleTest: Very informative: \"\\n\": \n", "at": 0, "level": "info" }, @@ -546,7 +546,7 @@ "00:00:00.000 I'm here with Erwin today; Erwin, what can you tell us about your cat?", "00:00:00.000 spam", "00:00:00.000 <body />", - "00:00:00.000 Very informative", + "00:00:00.000 Very informative: \"\\n\": \n", "00:00:00.000 Oh no", "00:00:00.000 spam", "00:00:00.000 Catch me if you can!", diff --git a/vespaclient-core/src/main/java/com/yahoo/vespaclient/ClusterList.java b/vespaclient-core/src/main/java/com/yahoo/vespaclient/ClusterList.java index d60463df581..c42d48b4821 100644 --- a/vespaclient-core/src/main/java/com/yahoo/vespaclient/ClusterList.java +++ b/vespaclient-core/src/main/java/com/yahoo/vespaclient/ClusterList.java @@ -20,6 +20,7 @@ public class ClusterList { this.contentClusters = List.copyOf(contentClusters); } + @SuppressWarnings("deprecation") public ClusterList(String configId) { this(new ConfigGetter<>(ClusterListConfig.class).getConfig(configId)); } |