diff options
92 files changed, 1059 insertions, 585 deletions
diff --git a/bundle-plugin/src/main/java/com/yahoo/container/plugin/mojo/AssembleTestBundleMojo.java b/bundle-plugin/src/main/java/com/yahoo/container/plugin/mojo/AssembleTestBundleMojo.java index 117d2cdc87e..acf0950decd 100644 --- a/bundle-plugin/src/main/java/com/yahoo/container/plugin/mojo/AssembleTestBundleMojo.java +++ b/bundle-plugin/src/main/java/com/yahoo/container/plugin/mojo/AssembleTestBundleMojo.java @@ -27,7 +27,7 @@ public class AssembleTestBundleMojo extends AbstractAssembleBundleMojo { @Override public void execute() throws MojoExecutionException { Artifacts.ArtifactSet artifacts = Artifacts.getArtifacts( - project, TestBundleDependencyScopeTranslator.from(project.getArtifactMap(), testBundleScopeOverrides)); + project, TestBundleDependencyScopeTranslator.from(project.getArtifacts(), testBundleScopeOverrides)); JarArchiver archiver = new JarArchiver(); addDirectory(archiver, Paths.get(project.getBuild().getOutputDirectory())); addDirectory(archiver, Paths.get(project.getBuild().getTestOutputDirectory())); diff --git a/bundle-plugin/src/main/java/com/yahoo/container/plugin/mojo/GenerateTestBundleOsgiManifestMojo.java b/bundle-plugin/src/main/java/com/yahoo/container/plugin/mojo/GenerateTestBundleOsgiManifestMojo.java index 0b10627d396..5708d90f82f 100644 --- a/bundle-plugin/src/main/java/com/yahoo/container/plugin/mojo/GenerateTestBundleOsgiManifestMojo.java +++ b/bundle-plugin/src/main/java/com/yahoo/container/plugin/mojo/GenerateTestBundleOsgiManifestMojo.java @@ -40,7 +40,7 @@ public class GenerateTestBundleOsgiManifestMojo extends AbstractGenerateOsgiMani public void execute() throws MojoExecutionException { try { Artifacts.ArtifactSet artifactSet = Artifacts.getArtifacts( - project, TestBundleDependencyScopeTranslator.from(project.getArtifactMap(), testBundleScopeOverrides)); + project, TestBundleDependencyScopeTranslator.from(project.getArtifacts(), testBundleScopeOverrides)); List<File> providedJars = artifactSet.getJarArtifactsProvided().stream() .map(Artifact::getFile) diff --git a/bundle-plugin/src/main/java/com/yahoo/container/plugin/util/TestBundleDependencyScopeTranslator.java b/bundle-plugin/src/main/java/com/yahoo/container/plugin/util/TestBundleDependencyScopeTranslator.java index bd6151aea9f..65606633dee 100644 --- a/bundle-plugin/src/main/java/com/yahoo/container/plugin/util/TestBundleDependencyScopeTranslator.java +++ b/bundle-plugin/src/main/java/com/yahoo/container/plugin/util/TestBundleDependencyScopeTranslator.java @@ -4,13 +4,17 @@ package com.yahoo.container.plugin.util; import org.apache.maven.artifact.Artifact; import java.util.Arrays; +import java.util.Collection; import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Objects; +import java.util.function.Function; import java.util.logging.Logger; +import java.util.stream.Collectors; import static java.util.stream.Collectors.toList; +import static java.util.stream.Collectors.toMap; /** * Translates the scope of dependencies when constructing a test bundle. @@ -40,11 +44,12 @@ public class TestBundleDependencyScopeTranslator implements Artifacts.ScopeTrans return Objects.requireNonNull(dependencyScopes.get(artifact), () -> "Could not lookup scope for " + artifact); } - public static TestBundleDependencyScopeTranslator from(Map<String, Artifact> dependencies, String rawConfig) { + public static TestBundleDependencyScopeTranslator from(Collection<Artifact> dependencies, String rawConfig) { List<DependencyOverride> dependencyOverrides = toDependencyOverrides(rawConfig); Map<Artifact, String> dependencyScopes = new HashMap<>(); - for (Artifact dependency : dependencies.values()) { - dependencyScopes.put(dependency, getScopeForDependency(dependency, dependencyOverrides, dependencies)); + Map<String, Artifact> dependenciesById = dependencies.stream().collect(toMap(Artifact::getId, Function.identity())); + for (Artifact dependency : dependencies) { + dependencyScopes.put(dependency, getScopeForDependency(dependency, dependencyOverrides, dependenciesById)); } return new TestBundleDependencyScopeTranslator(dependencyScopes); } @@ -66,12 +71,6 @@ public class TestBundleDependencyScopeTranslator implements Artifacts.ScopeTrans return new DependencyOverride(elements[0], elements[1], elements[2]); } - private static String stripVersionAndScope(String idInDependencyTrail) { - int firstDelimiter = idInDependencyTrail.indexOf(':'); - int secondDelimiter = idInDependencyTrail.indexOf(':', firstDelimiter + 1); - return idInDependencyTrail.substring(0, secondDelimiter); - } - private static String getScopeForDependency( Artifact dependency, List<DependencyOverride> overrides, Map<String, Artifact> otherArtifacts) { String oldScope = dependency.getScope(); @@ -95,7 +94,7 @@ public class TestBundleDependencyScopeTranslator implements Artifacts.ScopeTrans private static List<Artifact> dependencyTrailOf(Artifact artifact, Map<String, Artifact> otherArtifacts) { return artifact.getDependencyTrail().stream() .skip(1) // Maven project itself is the first entry - .map(parentId -> otherArtifacts.get(stripVersionAndScope(parentId))) + .map(otherArtifacts::get) .filter(Objects::nonNull) .collect(toList()); } diff --git a/bundle-plugin/src/test/java/com/yahoo/container/plugin/util/TestBundleDependencyScopeTranslatorTest.java b/bundle-plugin/src/test/java/com/yahoo/container/plugin/util/TestBundleDependencyScopeTranslatorTest.java index d5986b00077..ada2f8d208a 100644 --- a/bundle-plugin/src/test/java/com/yahoo/container/plugin/util/TestBundleDependencyScopeTranslatorTest.java +++ b/bundle-plugin/src/test/java/com/yahoo/container/plugin/util/TestBundleDependencyScopeTranslatorTest.java @@ -10,7 +10,9 @@ import org.junit.Test; import java.util.ArrayList; import java.util.List; import java.util.Map; +import java.util.Set; import java.util.TreeMap; +import java.util.TreeSet; import static org.junit.Assert.assertEquals; @@ -23,7 +25,7 @@ public class TestBundleDependencyScopeTranslatorTest { @Test public void test_dependencies_are_translated_to_compile_scope_by_default() { - Map<String, Artifact> artifacts = new TreeMap<>(); + Set<Artifact> artifacts = new TreeSet<>(); Artifact a = createArtifact(artifacts, "a", "test", List.of()); Artifact aa = createArtifact(artifacts, "a-a", "test", List.of("a")); Artifact ab = createArtifact(artifacts, "a-b", "test", List.of("a")); @@ -39,7 +41,7 @@ public class TestBundleDependencyScopeTranslatorTest { @Test public void non_test_scope_dependencies_keep_original_scope() { - Map<String, Artifact> artifacts = new TreeMap<>(); + Set<Artifact> artifacts = new TreeSet<>(); Artifact a = createArtifact(artifacts, "a", "provided", List.of()); Artifact aa = createArtifact(artifacts, "a-a", "provided", List.of("a")); Artifact ab = createArtifact(artifacts, "a-b", "provided", List.of("a")); @@ -60,7 +62,7 @@ public class TestBundleDependencyScopeTranslatorTest { @Test public void ordering_in_config_string_determines_translation() { - Map<String, Artifact> artifacts = new TreeMap<>(); + Set<Artifact> artifacts = new TreeSet<>(); Artifact a = createArtifact(artifacts, "a", "test", List.of()); Artifact aa = createArtifact(artifacts, "a-a", "test", List.of("a")); { @@ -83,7 +85,7 @@ public class TestBundleDependencyScopeTranslatorTest { @Test public void transitive_non_test_dependencies_of_test_dependencies_keep_original_scope() { - Map<String, Artifact> artifacts = new TreeMap<>(); + Set<Artifact> artifacts = new TreeSet<>(); Artifact a = createArtifact(artifacts, "a", "test", List.of()); Artifact aa = createArtifact(artifacts, "a-a", "test", List.of("a")); Artifact ab = createArtifact(artifacts, "a-b", "test", List.of("a")); @@ -105,16 +107,37 @@ public class TestBundleDependencyScopeTranslatorTest { assertScope(translator, bb, "provided"); } - private static Artifact createArtifact( - Map<String, Artifact> artifacts, String artifactId, String scope, List<String> transitiveDependents) { - Artifact artifact = createArtifact(artifactId, scope, transitiveDependents); - artifacts.put(simpleId(artifactId), artifact); + @Test + public void different_classifiers_are_handled_separately() { + Set<Artifact> artifacts = new TreeSet<>(); + Artifact a = createArtifact(artifacts, "a", "test", List.of()); + Artifact ab = createArtifact(artifacts, "a-b", "provided", List.of("a")); + Artifact ac = createArtifact(artifacts, "a-c", "classy", "test", List.of("a")); + + TestBundleDependencyScopeTranslator withoutOverrides = TestBundleDependencyScopeTranslator.from(artifacts, ""); + assertScope(withoutOverrides, a, "compile"); + assertScope(withoutOverrides, ab, "provided"); + assertScope(withoutOverrides, ac, "compile"); + + TestBundleDependencyScopeTranslator withOverrides = TestBundleDependencyScopeTranslator.from(artifacts, "com.test:a:test"); + assertScope(withOverrides, a, "test"); + assertScope(withOverrides, ab, "provided"); + assertScope(withOverrides, ac, "test"); + } + + private static Artifact createArtifact(Set<Artifact> artifacts, String artifactId, String scope, List<String> transitiveDependents) { + return createArtifact(artifacts, artifactId, null, scope, transitiveDependents); + } + + private static Artifact createArtifact(Set<Artifact> artifacts, String artifactId, String classifier, String scope, List<String> transitiveDependents) { + Artifact artifact = createArtifact(artifactId, classifier, scope, transitiveDependents); + artifacts.add(artifact); return artifact; } - private static Artifact createArtifact(String artifactId, String scope, List<String> transitiveDependents) { + private static Artifact createArtifact(String artifactId, String classifier, String scope, List<String> transitiveDependents) { Artifact artifact = new DefaultArtifact( - GROUP_ID, artifactId, "1.0", scope, "jar", /*classifier*/null, new DefaultArtifactHandler("jar")); + GROUP_ID, artifactId, "1.0", scope, "jar", classifier, new DefaultArtifactHandler("jar")); List<String> dependencyTrail = new ArrayList<>(); dependencyTrail.add(GROUP_ID + "my-project:container-plugin:1-SNAPSHOT"); transitiveDependents.forEach(dependent -> dependencyTrail.add(fullId(dependent))); diff --git a/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java b/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java index 08f96b34acb..61a946e9880 100644 --- a/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java +++ b/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java @@ -116,6 +116,7 @@ public interface ModelContext { @ModelFeatureFlag(owners = {"baldersheim"}) default boolean enableBitVectors() { return false; } @ModelFeatureFlag(owners = {"hmusum"}) default Architecture adminClusterArchitecture() { return Architecture.getDefault(); } @ModelFeatureFlag(owners = {"tokle"}) default boolean enableProxyProtocolMixedMode() { return true; } + @ModelFeatureFlag(owners = {"arnej"}) default String logFileCompressionAlgorithm(String defVal) { return defVal; } } /** Warning: As elsewhere in this package, do not make backwards incompatible changes that will break old config models! */ diff --git a/config-model/src/main/java/com/yahoo/schema/processing/DictionaryProcessor.java b/config-model/src/main/java/com/yahoo/schema/processing/DictionaryProcessor.java index 3209fd1703d..9c6c446b82d 100644 --- a/config-model/src/main/java/com/yahoo/schema/processing/DictionaryProcessor.java +++ b/config-model/src/main/java/com/yahoo/schema/processing/DictionaryProcessor.java @@ -19,9 +19,11 @@ import com.yahoo.vespa.model.container.search.QueryProfiles; * @author baldersheim */ public class DictionaryProcessor extends Processor { + public DictionaryProcessor(Schema schema, DeployLogger deployLogger, RankProfileRegistry rankProfileRegistry, QueryProfiles queryProfiles) { super(schema, deployLogger, rankProfileRegistry, queryProfiles); } + @Override public void process(boolean validate, boolean documentsOnly) { for (SDField field : schema.allConcreteFields()) { @@ -51,4 +53,5 @@ public class DictionaryProcessor extends Processor { } } } + } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/Logserver.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/Logserver.java index c87eeab23a5..d5a0b97a416 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/admin/Logserver.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/Logserver.java @@ -1,6 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.model.admin; +import com.yahoo.config.model.deploy.DeployState; import com.yahoo.config.model.producer.AbstractConfigProducer; import com.yahoo.vespa.model.AbstractService; import com.yahoo.vespa.model.PortAllocBridge; @@ -16,6 +17,7 @@ public class Logserver extends AbstractService { private static final long serialVersionUID = 1L; private static final String logArchiveDir = "$ROOT/logs/vespa/logarchive"; + private String compressionType = "gzip"; public Logserver(AbstractConfigProducer parent) { super(parent, "logserver"); @@ -27,6 +29,12 @@ public class Logserver extends AbstractService { setProp("clustername", "admin"); } + @Override + public void initService(DeployState deployState) { + super.initService(deployState); + this.compressionType = deployState.featureFlags().logFileCompressionAlgorithm("gzip"); + } + /** * @return the startup command for the logserver */ @@ -44,6 +52,8 @@ public class Logserver extends AbstractService { sb.append("-Dlogserver.rpcListenPort=").append(getRelativePort(0)); sb.append(" "); sb.append("-Dlogserver.logarchive.dir=" + logArchiveDir); + sb.append(" "); + sb.append("-Dlogserver.logarchive.compression=" + compressionType); return sb.toString(); } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/LogserverContainer.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/LogserverContainer.java index 251ef48c9f7..62e99576c95 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/admin/LogserverContainer.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/LogserverContainer.java @@ -23,8 +23,9 @@ public class LogserverContainer extends Container { super(parent, "" + 0, 0, deployState); if (deployState.isHosted() && deployState.getProperties().applicationId().instance().isTester()) useDynamicPorts(); LogserverContainerCluster cluster = (LogserverContainerCluster) parent; - addComponent(new AccessLogComponent( - cluster, AccessLogType.jsonAccessLog, CompressionType.GZIP, Optional.of(cluster.getName()), true)); + addComponent(new AccessLogComponent(cluster, AccessLogType.jsonAccessLog, + deployState.featureFlags().logFileCompressionAlgorithm("zstd"), + Optional.of(cluster.getName()), true)); } @Override diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/clustercontroller/ClusterControllerContainer.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/clustercontroller/ClusterControllerContainer.java index 45e9b3a905a..f303da6c9f0 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/admin/clustercontroller/ClusterControllerContainer.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/clustercontroller/ClusterControllerContainer.java @@ -66,7 +66,7 @@ public class ClusterControllerContainer extends Container implements "/cluster/v2/*", CLUSTERCONTROLLER_BUNDLE); addComponent(new AccessLogComponent(containerCluster().orElse(null), AccessLogComponent.AccessLogType.jsonAccessLog, - AccessLogComponent.CompressionType.GZIP, + deployState.featureFlags().logFileCompressionAlgorithm("zstd"), Optional.of("controller"), deployState.isHosted())); diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainer.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainer.java index f159362d750..03bdf4eb12a 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainer.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainer.java @@ -59,7 +59,7 @@ public class MetricsProxyContainer extends Container implements setProp("index", String.valueOf(index)); addNodeSpecificComponents(); addComponent(new AccessLogComponent(containerCluster().orElse(null), AccessLogComponent.AccessLogType.jsonAccessLog, - AccessLogComponent.CompressionType.ZSTD, + "zstd", Optional.of("metrics-proxy"), deployState.isHosted())); } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerCluster.java index 79412368eae..4755f674f69 100755 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerCluster.java @@ -146,6 +146,7 @@ public abstract class ContainerCluster<CONTAINER extends Container> private final boolean isHostedVespa; private final boolean zooKeeperLocalhostAffinity; private final int numAvailableProcessors; + private final String compressionType; private final Map<String, String> concreteDocumentTypes = new LinkedHashMap<>(); @@ -166,7 +167,8 @@ public abstract class ContainerCluster<CONTAINER extends Container> this.isHostedVespa = stateIsHosted(deployState); this.zone = (deployState != null) ? deployState.zone() : Zone.defaultZone(); this.zooKeeperLocalhostAffinity = zooKeeperLocalhostAffinity; - numAvailableProcessors = deployState.featureFlags().availableProcessors(); + this.numAvailableProcessors = deployState.featureFlags().availableProcessors(); + this.compressionType = deployState.featureFlags().logFileCompressionAlgorithm("zstd"); componentGroup = new ComponentGroup<>(this, "component"); @@ -282,7 +284,8 @@ public abstract class ContainerCluster<CONTAINER extends Container> container.setOwner(this); container.setClusterName(name); container.setProp("clustername", name) - .setProp("index", this.containers.size()); + .setProp("index", this.containers.size()) + .setProp("clustertype", "container"); containers.add(container); } @@ -526,7 +529,6 @@ public abstract class ContainerCluster<CONTAINER extends Container> } public void addDefaultSearchAccessLog() { - var compressionType = isHostedVespa ? AccessLogComponent.CompressionType.ZSTD : AccessLogComponent.CompressionType.GZIP; // In hosted Vespa with one application container per node we do not use the container name to distinguish log files Optional<String> clusterName = isHostedVespa ? Optional.empty() : Optional.of(getName()); addComponent(new AccessLogComponent(this, AccessLogComponent.AccessLogType.jsonAccessLog, compressionType, clusterName, isHostedVespa)); diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/component/AccessLogComponent.java b/config-model/src/main/java/com/yahoo/vespa/model/container/component/AccessLogComponent.java index cc0415a7630..79c108cd867 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/component/AccessLogComponent.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/component/AccessLogComponent.java @@ -32,7 +32,7 @@ public final class AccessLogComponent extends SimpleComponent implements AccessL private final int queueSize; private final Integer bufferSize; - public AccessLogComponent(ContainerCluster<?> cluster, AccessLogType logType, CompressionType compressionType, Optional<String> clusterName, boolean isHostedVespa) + public AccessLogComponent(ContainerCluster<?> cluster, AccessLogType logType, String compressionType, Optional<String> clusterName, boolean isHostedVespa) { // In hosted Vespa we do not use the clusterName when setting up application ContainerCluster logging this(logType, @@ -55,7 +55,7 @@ public final class AccessLogComponent extends SimpleComponent implements AccessL } public AccessLogComponent(AccessLogType logType, - CompressionType compressionType, + String compressionType, String fileNamePattern, String rotationInterval, Boolean compressOnRotation, @@ -70,7 +70,7 @@ public final class AccessLogComponent extends SimpleComponent implements AccessL this.compression = compressOnRotation; this.isHostedVespa = isHostedVespa; this.symlinkName = symlinkName; - this.compressionType = compressionType; + this.compressionType = "zstd".equals(compressionType) ? CompressionType.ZSTD :CompressionType.GZIP; this.queueSize = (queueSize == null) ? 256 : queueSize; this.bufferSize = bufferSize; } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/AccessLogBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/AccessLogBuilder.java index b939109dab1..3462fb8bcfc 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/AccessLogBuilder.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/AccessLogBuilder.java @@ -54,9 +54,10 @@ public class AccessLogBuilder { @Override protected AccessLogComponent doBuild(DeployState deployState, AbstractConfigProducer<?> ancestor, Element spec) { + String fallback = deployState.featureFlags().logFileCompressionAlgorithm("zstd"); return new AccessLogComponent( accessLogType, - compressionType(spec, isHostedVespa), + compressionType(spec, fallback), fileNamePattern(spec), rotationInterval(spec), compressOnRotation(spec), @@ -93,21 +94,10 @@ public class AccessLogBuilder { return nullIfEmpty(spec.getAttribute("fileNamePattern")); } - private static CompressionType compressionType(Element spec, boolean isHostedVespa) { - CompressionType fallback = isHostedVespa ? CompressionType.ZSTD : CompressionType.GZIP; + private static String compressionType(Element spec, String fallback) { return Optional.ofNullable(spec.getAttribute("compressionType")) - .filter(value -> !value.isBlank()) - .map(value -> { - switch (value) { - case "gzip": - return CompressionType.GZIP; - case "zstd": - return CompressionType.ZSTD; - default: - throw new IllegalArgumentException("Unknown compression type: " + value); - } - }) - .orElse(fallback); + .filter(value -> !value.isBlank()) + .orElse(fallback); } } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ConfigServerContainerModelBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ConfigServerContainerModelBuilder.java index 9871fed5b7c..1aa4333a18c 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ConfigServerContainerModelBuilder.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ConfigServerContainerModelBuilder.java @@ -47,7 +47,7 @@ public class ConfigServerContainerModelBuilder extends ContainerModelBuilder { if (isHosted()){ cluster.addComponent( new AccessLogComponent( - AccessLogComponent.AccessLogType.jsonAccessLog, AccessLogComponent.CompressionType.ZSTD, + AccessLogComponent.AccessLogType.jsonAccessLog, "zstd", "logs/vespa/configserver/access-json.log.%Y%m%d%H%M%S", null, true, true, "access-json.log", 1024,256*1024)); cluster.addComponent(new ConnectionLogComponent(cluster, FileConnectionLog.class, "configserver")); } else { diff --git a/config-model/src/test/java/com/yahoo/schema/processing/DictionaryTestCase.java b/config-model/src/test/java/com/yahoo/schema/processing/DictionaryTestCase.java index 1956b87a689..45a546259ae 100644 --- a/config-model/src/test/java/com/yahoo/schema/processing/DictionaryTestCase.java +++ b/config-model/src/test/java/com/yahoo/schema/processing/DictionaryTestCase.java @@ -62,7 +62,7 @@ public class DictionaryTestCase { Schema verifyDictionaryControl(Dictionary.Type expected, String type, String ... cfg) throws ParseException { String def = TestUtil.joinLines( - "search test {", + "schema test {", " document test {", " field n1 type " + type + " {", " indexing: summary | attribute", diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java index ba9bf7761bd..30116b7ee69 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java @@ -209,6 +209,7 @@ public class ModelContextImpl implements ModelContext { private final Architecture adminClusterArchitecture; private final boolean enableProxyProtocolMixedMode; private final boolean sharedStringRepoNoReclaim; + private final String logFileCompressionAlgorithm; public FeatureFlags(FlagSource source, ApplicationId appId, Version version) { this.defaultTermwiseLimit = flagValue(source, appId, version, Flags.DEFAULT_TERM_WISE_LIMIT); @@ -254,6 +255,7 @@ public class ModelContextImpl implements ModelContext { this.adminClusterArchitecture = Architecture.valueOf(flagValue(source, appId, version, PermanentFlags.ADMIN_CLUSTER_NODE_ARCHITECTURE)); this.enableProxyProtocolMixedMode = flagValue(source, appId, version, Flags.ENABLE_PROXY_PROTOCOL_MIXED_MODE); this.sharedStringRepoNoReclaim = flagValue(source, appId, version, Flags.SHARED_STRING_REPO_NO_RECLAIM); + this.logFileCompressionAlgorithm = flagValue(source, appId, version, Flags.LOG_FILE_COMPRESSION_ALGORITHM); } @Override public double defaultTermwiseLimit() { return defaultTermwiseLimit; } @@ -301,6 +303,13 @@ public class ModelContextImpl implements ModelContext { @Override public Architecture adminClusterArchitecture() { return adminClusterArchitecture; } @Override public boolean enableProxyProtocolMixedMode() { return enableProxyProtocolMixedMode; } @Override public boolean sharedStringRepoNoReclaim() { return sharedStringRepoNoReclaim; } + @Override public String logFileCompressionAlgorithm(String defVal) { + var fflag = this.logFileCompressionAlgorithm; + if (fflag != null && ! fflag.equals("")) { + return fflag; + } + return defVal; + } private static <V> V flagValue(FlagSource source, ApplicationId appId, Version vespaVersion, UnboundFlag<? extends V, ?, ?> flag) { return flag.bindTo(source) diff --git a/container-core/src/main/java/com/yahoo/container/handler/LogReader.java b/container-core/src/main/java/com/yahoo/container/handler/LogReader.java index 250e3c8b607..2d60dc3f37b 100644 --- a/container-core/src/main/java/com/yahoo/container/handler/LogReader.java +++ b/container-core/src/main/java/com/yahoo/container/handler/LogReader.java @@ -104,14 +104,15 @@ class LogReader { try { in = Files.newInputStream(log); } - catch (NoSuchFileException e) { + catch (NoSuchFileException e) { // File may have been compressed since we found it. if ( ! zipped) try { - in = Files.newInputStream(Paths.get(log.toString() + ".gz")); + in = Files.newInputStream(Paths.get(log + ".gz")); zipped = true; } catch (NoSuchFileException ignored) { } } + this.reader = new BufferedReader(new InputStreamReader(zipped ? new GZIPInputStream(in) : in, UTF_8)); this.from = from; this.to = to; @@ -252,6 +253,7 @@ class LogReader { .toInstant() .plus(Duration.ofSeconds(1)); } + // TODO: accept .zst files when the io.airlift library supports streamed input. throw new IllegalArgumentException("Unrecognized file pattern for file at '" + path + "'"); } diff --git a/container-search-gui/src/main/java/com/yahoo/search/query/gui/GUIHandler.java b/container-search-gui/src/main/java/com/yahoo/search/query/gui/GUIHandler.java index 8c8b65a4a5a..e370b1f19d3 100644 --- a/container-search-gui/src/main/java/com/yahoo/search/query/gui/GUIHandler.java +++ b/container-search-gui/src/main/java/com/yahoo/search/query/gui/GUIHandler.java @@ -17,6 +17,7 @@ import com.yahoo.search.config.IndexInfoConfig; import com.yahoo.search.query.Model; import com.yahoo.search.query.Presentation; import com.yahoo.search.query.Ranking; +import com.yahoo.search.query.Trace; import com.yahoo.search.query.ranking.Diversity; import com.yahoo.search.query.ranking.MatchPhase; import com.yahoo.search.query.restapi.ErrorResponse; @@ -192,11 +193,13 @@ public class GUIHandler extends ThreadedHttpRequestHandler { json.set("childMap", childMap); ArrayNode levelZeroParameters = jsonMapper.createArrayNode().add(MinimalQueryInserter.YQL.toString()).add(Query.HITS.toString()).add(Query.OFFSET.toString()) - .add("queryProfile").add(Query.NO_CACHE.toString()).add(Query.GROUPING_SESSION_CACHE.toString()) - .add(Query.SEARCH_CHAIN.toString()).add(Query.TIMEOUT.toString()).add("trace").add("tracelevel") - .add(Query.TRACE_LEVEL.toString()).add(Query.EXPLAIN_LEVEL.toString()).add("explainlevel").add(Model.MODEL).add(Ranking.RANKING).add("collapse").add("collapsesize").add("collapsefield") - .add(Presentation.PRESENTATION).add("pos").add("streaming").add("rules").add(RecallSearcher.recallName.toString()).add("user") - .add("metrics").add(""); + .add("queryProfile").add(Query.NO_CACHE.toString()).add(Query.GROUPING_SESSION_CACHE.toString()) + .add(Query.SEARCH_CHAIN.toString()).add(Query.TIMEOUT.toString()).add("trace") + .add("tracelevel").add("traceLevel") // TODO: Remove on Vespa 9 + .add("explainLevel").add("explainlevel") // TODO: Remove on Vespa 9 + .add(Model.MODEL).add(Ranking.RANKING).add("collapse").add("collapsesize").add("collapsefield") + .add(Presentation.PRESENTATION).add("pos").add("streaming").add("rules").add(RecallSearcher.recallName.toString()).add("user") + .add("metrics").add(""); json.set("levelZeroParameters", levelZeroParameters); return jsonMapper.writerWithDefaultPrettyPrinter().writeValueAsString(json); diff --git a/container-search-gui/src/main/resources/gui/_includes/index.html b/container-search-gui/src/main/resources/gui/_includes/index.html index 833c9352798..bac486077a4 100644 --- a/container-search-gui/src/main/resources/gui/_includes/index.html +++ b/container-search-gui/src/main/resources/gui/_includes/index.html @@ -57,7 +57,7 @@ <span class="icon-bar"></span> <span class="icon-bar"></span> </button> - <a class="navbar-brand" href="http://www.vespa.ai">Vespa. Big data. Real time.</a> + <a class="navbar-brand" href="https://vespa.ai">Vespa. Big data. Real time.</a> </div> <!-- Collect the nav links, forms, and other content for toggling --> @@ -68,7 +68,7 @@ <li><a href="https://twitter.com/vespaengine">Twitter</a> <li><a href="https://docs.vespa.ai">Docs</a> <li><a href="https://github.com/vespa-engine">GitHub</a> - <li><a href="https://docs.vespa.ai/en/vespa-quick-start.html">Get Started Now</a> + <li><a href="https://docs.vespa.ai/en/getting-started.html">Get Started Now</a> </ul> </div> </div> @@ -128,7 +128,7 @@ <span> â—‹ Autocompletion of YQL-syntax</span> </br> <span> â—‹ Drop-down lists of all valid parameters</span> </br> <span> â—‹ Sending both POST and GET-requests to <i>Vespa</i></span> </br> - <span> â—‹ Easy access to the <a href="https://docs.vespa.ai/en/reference/search-api-reference.html">documentation</a> of each parameter</span> </br> + <span> â—‹ Easy access to the <a href="https://docs.vespa.ai/en/reference/query-api-reference.html">documentation</a> of each parameter</span> </br> <span> â—‹ Conversion of POST- to GET-query</span> </br> <span> â—‹ Pasting already built JSON-query</span> </br> <span> â—‹ View and copy the response of queries</span> </br> @@ -137,7 +137,7 @@ </br> <div class="intro-param" id="help"> <div class="help-title">Help</div> - <span> If you find errors, spelling mistakes, faulty pieces of code or want to improve the querybuilder, please submit a pull request or create an <a href="https://github.com/vespa-engine/vespa/issues">issue</a>.</span> </br> + <span>Please submit a <a href="https://github.com/vespa-engine/vespa/blob/master/container-search-gui/src/main/resources/gui/_includes/index.html">pull request</a> or create an <a href="https://github.com/vespa-engine/vespa/issues">issue</a> for fixes to the querybuilder.</span> </br> </div> </div> </div> @@ -168,7 +168,7 @@ <div class="footer-title">Community</div> <ul class="quicklinks"> <li><a href="https://github.com/vespa-engine/vespa/blob/master/CONTRIBUTING.md">Contributing</a> - <li><a href="http://stackoverflow.com/questions/tagged/vespa">Stack Overflow</a> + <li><a href="https://stackoverflow.com/questions/tagged/vespa">Stack Overflow</a> <li><a href="https://gitter.im/vespa-engine/Lobby">Gitter</a> </ul> </div> @@ -975,13 +975,5 @@ } </script> - <!-- Global Site Tag (gtag.js) - Google Analytics --> - <script async src="https://www.googletagmanager.com/gtag/js?id=UA-107187180-1"></script> - <script> - window.dataLayer = window.dataLayer || []; - function gtag(){dataLayer.push(arguments)}; - gtag('js', new Date()); - gtag('config', 'UA-107187180-1'); - </script> </body> </html> diff --git a/container-search/abi-spec.json b/container-search/abi-spec.json index 7fe1702be8f..7f100df4e2c 100644 --- a/container-search/abi-spec.json +++ b/container-search/abi-spec.json @@ -1983,6 +1983,7 @@ "public com.yahoo.search.query.Select getSelect()", "public com.yahoo.search.query.Ranking getRanking()", "public com.yahoo.search.query.Model getModel()", + "public com.yahoo.search.query.Trace getTrace()", "public com.yahoo.container.jdisc.HttpRequest getHttpRequest()", "public com.yahoo.search.query.SessionId getSessionId()", "public com.yahoo.search.query.SessionId getSessionId(java.lang.String)", @@ -1993,15 +1994,16 @@ "public bridge synthetic java.lang.Object clone()" ], "fields": [ + "public com.yahoo.search.query.Trace trace", "public static final com.yahoo.processing.request.CompoundName OFFSET", "public static final com.yahoo.processing.request.CompoundName HITS", "public static final com.yahoo.processing.request.CompoundName QUERY_PROFILE", "public static final com.yahoo.processing.request.CompoundName SEARCH_CHAIN", - "public static final com.yahoo.processing.request.CompoundName TRACE_LEVEL", - "public static final com.yahoo.processing.request.CompoundName EXPLAIN_LEVEL", "public static final com.yahoo.processing.request.CompoundName NO_CACHE", "public static final com.yahoo.processing.request.CompoundName GROUPING_SESSION_CACHE", "public static final com.yahoo.processing.request.CompoundName TIMEOUT", + "public static final com.yahoo.processing.request.CompoundName TRACE_LEVEL", + "public static final com.yahoo.processing.request.CompoundName EXPLAIN_LEVEL", "public static final java.util.List nativeProperties" ] }, @@ -5613,6 +5615,46 @@ "public static final java.lang.String LOWERCASE" ] }, + "com.yahoo.search.query.Trace": { + "superClass": "java.lang.Object", + "interfaces": [ + "java.lang.Cloneable" + ], + "attributes": [ + "public" + ], + "methods": [ + "public static com.yahoo.search.query.profile.types.QueryProfileType getArgumentType()", + "public void <init>(com.yahoo.search.Query)", + "public int getLevel()", + "public void setLevel(int)", + "public boolean isTraceable(int)", + "public void setExplainLevel(int)", + "public int getExplainLevel()", + "public boolean getTimestamps()", + "public void setTimestamps(boolean)", + "public boolean getQuery()", + "public void setQuery(boolean)", + "public void trace(java.lang.String, int)", + "public void trace(java.lang.Object, int)", + "public void trace(java.lang.String, boolean, int)", + "public varargs void trace(boolean, int, java.lang.Object[])", + "public void traceProperties()", + "public com.yahoo.search.query.Trace cloneFor(com.yahoo.search.Query)", + "public boolean equals(java.lang.Object)", + "public int hashCode()", + "public com.yahoo.search.query.Trace clone()", + "public java.lang.String toString()", + "public bridge synthetic java.lang.Object clone()" + ], + "fields": [ + "public static final java.lang.String TRACE", + "public static final java.lang.String LEVEL", + "public static final java.lang.String EXPLAIN_LEVEL", + "public static final java.lang.String TIMESTAMPS", + "public static final java.lang.String QUERY" + ] + }, "com.yahoo.search.query.UniqueRequestId": { "superClass": "java.lang.Object", "interfaces": [], @@ -5992,6 +6034,7 @@ "public void <init>(com.yahoo.search.query.profile.compiled.CompiledQueryProfile)", "public void <init>(com.yahoo.search.query.profile.compiled.CompiledQueryProfile, com.yahoo.language.process.Embedder)", "public void <init>(com.yahoo.search.query.profile.compiled.CompiledQueryProfile, java.util.Map)", + "public void <init>(com.yahoo.search.query.profile.compiled.CompiledQueryProfile, java.util.Map, ai.vespa.cloud.ZoneInfo)", "public com.yahoo.search.query.profile.compiled.CompiledQueryProfile getQueryProfile()", "public java.lang.Object get(com.yahoo.processing.request.CompoundName, java.util.Map, com.yahoo.processing.request.Properties)", "public void set(com.yahoo.processing.request.CompoundName, java.lang.Object, java.util.Map)", @@ -6687,7 +6730,7 @@ "public" ], "methods": [ - "public void <init>(java.util.Map, ai.vespa.cloud.ZoneInfo)", + "public void <init>(java.util.Map)", "public java.lang.Object get(com.yahoo.processing.request.CompoundName, java.util.Map, com.yahoo.processing.request.Properties)", "public void set(com.yahoo.processing.request.CompoundName, java.lang.Object, java.util.Map)", "public java.util.Map listProperties(com.yahoo.processing.request.CompoundName, java.util.Map, com.yahoo.processing.request.Properties)" @@ -8188,7 +8231,6 @@ "methods": [ "public void <init>(com.yahoo.container.core.ChainsConfig, com.yahoo.search.config.IndexInfoConfig, com.yahoo.search.config.SchemaInfoConfig, com.yahoo.container.QrSearchersConfig, com.yahoo.component.provider.ComponentRegistry, com.yahoo.vespa.configdefinition.SpecialtokensConfig, com.yahoo.language.Linguistics, com.yahoo.component.provider.ComponentRegistry, java.util.concurrent.Executor)", "public void <init>(com.yahoo.container.core.ChainsConfig, com.yahoo.search.config.IndexInfoConfig, com.yahoo.search.schema.SchemaInfo, com.yahoo.container.QrSearchersConfig, com.yahoo.component.provider.ComponentRegistry, com.yahoo.vespa.configdefinition.SpecialtokensConfig, com.yahoo.language.Linguistics, com.yahoo.component.provider.ComponentRegistry, java.util.concurrent.Executor)", - "public void <init>(com.yahoo.container.core.ChainsConfig, com.yahoo.search.config.IndexInfoConfig, com.yahoo.container.QrSearchersConfig, com.yahoo.component.provider.ComponentRegistry, com.yahoo.vespa.configdefinition.SpecialtokensConfig, com.yahoo.language.Linguistics, com.yahoo.component.provider.ComponentRegistry, java.util.concurrent.Executor)", "public com.yahoo.search.searchchain.Execution newExecution(com.yahoo.component.chain.Chain)", "public com.yahoo.search.searchchain.Execution newExecution(java.lang.String)", "public com.yahoo.search.searchchain.SearchChainRegistry searchChainRegistry()", diff --git a/container-search/src/main/java/com/yahoo/prelude/Pong.java b/container-search/src/main/java/com/yahoo/prelude/Pong.java index b6deee61b81..ecd6e302ccc 100644 --- a/container-search/src/main/java/com/yahoo/prelude/Pong.java +++ b/container-search/src/main/java/com/yahoo/prelude/Pong.java @@ -42,22 +42,6 @@ public class Pong { this.error = error; } - /** - * @deprecated do not use. Additional errors are ignored. - */ - @Deprecated - public void addError(ErrorMessage error) { } - - /** - * @deprecated use error() instead - */ - @Deprecated - public ErrorMessage getError(int i) { - if (i > 1) throw new IllegalArgumentException("No error at position " + i); - if (i == 0 && error.isEmpty()) throw new IllegalArgumentException("No error at position " + i); - return error.get(); - } - public Optional<ErrorMessage> error() { return error; } /** Returns the number of active documents in the backend responding in this Pong, if available */ @@ -66,27 +50,6 @@ public class Pong { /** Returns true if the pinged node is currently blocking write operations due to being full */ public boolean isBlockingWrites() { return isBlockingWrites; } - /** - * Returns Optional.empty() - * - * @return empty - * @deprecated do not use. There is always one pong per node. - */ - @Deprecated - public Optional<Integer> activeNodes() { - return Optional.empty(); - } - - /** - * Returns a list containing 0 or 1 errors - * - * @deprecated use error() instead - */ - @Deprecated - public List<ErrorMessage> getErrors() { - return error.stream().collect(Collectors.toList()); - } - /** Returns whether there is an error or not */ public boolean badResponse() { return error.isPresent(); } diff --git a/container-search/src/main/java/com/yahoo/prelude/cluster/ClusterSearcher.java b/container-search/src/main/java/com/yahoo/prelude/cluster/ClusterSearcher.java index 496e057a243..729aebf2fc2 100644 --- a/container-search/src/main/java/com/yahoo/prelude/cluster/ClusterSearcher.java +++ b/container-search/src/main/java/com/yahoo/prelude/cluster/ClusterSearcher.java @@ -204,7 +204,7 @@ public class ClusterSearcher extends Searcher { private void validateQueryTimeout(Query query) { if (query.getTimeout() <= maxQueryTimeout) return; - if (query.isTraceable(2)) { + if (query.getTrace().isTraceable(2)) { query.trace("Query timeout (" + query.getTimeout() + " ms) > max query timeout (" + maxQueryTimeout + " ms). Setting timeout to " + maxQueryTimeout + " ms.", 2); } @@ -215,7 +215,7 @@ public class ClusterSearcher extends Searcher { if ( ! query.getRanking().getQueryCache() ) return; if (query.getTimeout() <= maxQueryCacheTimeout) return; - if (query.isTraceable(2)) { + if (query.getTrace().isTraceable(2)) { query.trace("Query timeout (" + query.getTimeout() + " ms) > max query cache timeout (" + maxQueryCacheTimeout + " ms). Disabling query cache.", 2); } diff --git a/container-search/src/main/java/com/yahoo/prelude/fastsearch/FastSearcher.java b/container-search/src/main/java/com/yahoo/prelude/fastsearch/FastSearcher.java index 33ad8d8c9a8..edd11974f8c 100644 --- a/container-search/src/main/java/com/yahoo/prelude/fastsearch/FastSearcher.java +++ b/container-search/src/main/java/com/yahoo/prelude/fastsearch/FastSearcher.java @@ -106,7 +106,7 @@ public class FastSearcher extends VespaBackEndSearcher { return new Result(query,ErrorMessage.createTimeout(e.getMessage())); } catch (IOException e) { Result result = new Result(query); - if (query.getTraceLevel() >= 1) + if (query.getTrace().getLevel() >= 1) query.trace(getName() + " error response: " + result, false, 1); result.hits().addError(ErrorMessage.createBackendCommunicationError(getName() + " failed: "+ e.getMessage())); return result; diff --git a/container-search/src/main/java/com/yahoo/prelude/fastsearch/VespaBackEndSearcher.java b/container-search/src/main/java/com/yahoo/prelude/fastsearch/VespaBackEndSearcher.java index 584249fa1f3..e184037bc90 100644 --- a/container-search/src/main/java/com/yahoo/prelude/fastsearch/VespaBackEndSearcher.java +++ b/container-search/src/main/java/com/yahoo/prelude/fastsearch/VespaBackEndSearcher.java @@ -190,7 +190,7 @@ public abstract class VespaBackEndSearcher extends PingableSearcher { Result result = doSearch2(query, execution); - if (query.getTraceLevel() >= 1) + if (query.getTrace().getLevel() >= 1) query.trace(getName() + " dispatch response: " + result, false, 1); result.trace(getName()); return result; @@ -243,7 +243,7 @@ public abstract class VespaBackEndSearcher extends PingableSearcher { } void traceQuery(String sourceName, String type, Query query, int offset, int hits, int level, Optional<String> quotedSummaryClass) { - if ((query.getTraceLevel()<level) || query.properties().getBoolean(TRACE_DISABLE)) return; + if ((query.getTrace().getLevel()<level) || query.properties().getBoolean(TRACE_DISABLE)) return; StringBuilder s = new StringBuilder(); s.append(sourceName).append(" ").append(type).append(" to dispatch: ") @@ -314,12 +314,12 @@ public abstract class VespaBackEndSearcher extends PingableSearcher { } query.trace(s.toString(), false, level); - if (query.isTraceable(level + 1)) { + if (query.getTrace().isTraceable(level + 1)) { query.trace("Current state of query tree: " + new TextualQueryRepresentation(query.getModel().getQueryTree().getRoot()), false, level+1); } - if (query.isTraceable(level + 2)) { + if (query.getTrace().isTraceable(level + 2)) { query.trace("YQL+ representation: " + query.yqlRepresentation(), level+2); } } diff --git a/container-search/src/main/java/com/yahoo/prelude/query/parser/AllParser.java b/container-search/src/main/java/com/yahoo/prelude/query/parser/AllParser.java index 8cc820d8f76..545bb8e777f 100644 --- a/container-search/src/main/java/com/yahoo/prelude/query/parser/AllParser.java +++ b/container-search/src/main/java/com/yahoo/prelude/query/parser/AllParser.java @@ -21,7 +21,7 @@ import static com.yahoo.prelude.query.parser.Token.Kind.MINUS; import static com.yahoo.prelude.query.parser.Token.Kind.SPACE; /** - * Parser for queries of type all. + * Parser for queries of type all and weakAnd. * * @author Steinar Knutsen * @author bratseth @@ -31,9 +31,9 @@ public class AllParser extends SimpleParser { private final boolean weakAnd; /** - * Creates an And parser + * Creates an all/weakAnd parser * - * @param weakAnd false to parse into AndItem (by default), true to parse to WeakAnd + * @param weakAnd false to parse into AndItem (by default), true to parse to WeakAndItem */ public AllParser(ParserEnvironment environment, boolean weakAnd) { super(environment); diff --git a/container-search/src/main/java/com/yahoo/prelude/querytransform/LiteralBoostSearcher.java b/container-search/src/main/java/com/yahoo/prelude/querytransform/LiteralBoostSearcher.java index 47a5213c041..28d4d9dff67 100644 --- a/container-search/src/main/java/com/yahoo/prelude/querytransform/LiteralBoostSearcher.java +++ b/container-search/src/main/java/com/yahoo/prelude/querytransform/LiteralBoostSearcher.java @@ -44,7 +44,7 @@ public class LiteralBoostSearcher extends Searcher { if (newRankTerms.getItemCount() > 0) addTopLevelRankTerms(newRankTerms, query); - if (query.getTraceLevel() >= 2 && newRankTerms.getItemCount() > 0) + if (query.getTrace().getLevel() >= 2 && newRankTerms.getItemCount() > 0) query.trace("Added rank terms for possible literal field matches.", true, 2); } diff --git a/container-search/src/main/java/com/yahoo/prelude/querytransform/NormalizingSearcher.java b/container-search/src/main/java/com/yahoo/prelude/querytransform/NormalizingSearcher.java index a847a84e134..b7fd1069a6b 100644 --- a/container-search/src/main/java/com/yahoo/prelude/querytransform/NormalizingSearcher.java +++ b/container-search/src/main/java/com/yahoo/prelude/querytransform/NormalizingSearcher.java @@ -53,11 +53,11 @@ public class NormalizingSearcher extends Searcher { } protected void normalize(Query query, IndexFacts.Session indexFacts) { - String oldQuery = (query.getTraceLevel() >= 2) ? query.getModel().getQueryTree().getRoot().toString() : null; + String oldQuery = (query.getTrace().getLevel() >= 2) ? query.getModel().getQueryTree().getRoot().toString() : null; normalizeBody(query, indexFacts); - if (query.getTraceLevel() >= 2 && ! query.getModel().getQueryTree().getRoot().toString().equals(oldQuery)) + if (query.getTrace().getLevel() >= 2 && ! query.getModel().getQueryTree().getRoot().toString().equals(oldQuery)) query.trace(getFunctionName(), true, 2); } diff --git a/container-search/src/main/java/com/yahoo/prelude/semantics/RuleBase.java b/container-search/src/main/java/com/yahoo/prelude/semantics/RuleBase.java index 4980b035876..5a168d42779 100644 --- a/container-search/src/main/java/com/yahoo/prelude/semantics/RuleBase.java +++ b/container-search/src/main/java/com/yahoo/prelude/semantics/RuleBase.java @@ -312,14 +312,14 @@ public class RuleBase { * If there is an error, this query is destroyed (unusable) */ public String analyze(Query query, int traceLevel) { - int queryTraceLevel = query.getTraceLevel(); + int queryTraceLevel = query.getTrace().getLevel(); if (traceLevel > 0 && queryTraceLevel == 0) - query.setTraceLevel(1); + query.getTrace().setLevel(1); matchAutomata(query, traceLevel); String error = analyzer.evaluate(query, traceLevel); - query.setTraceLevel(queryTraceLevel); + query.getTrace().setLevel(queryTraceLevel); return error; } diff --git a/container-search/src/main/java/com/yahoo/prelude/semantics/SemanticSearcher.java b/container-search/src/main/java/com/yahoo/prelude/semantics/SemanticSearcher.java index e25f00bdc80..42a2b4f4e9b 100644 --- a/container-search/src/main/java/com/yahoo/prelude/semantics/SemanticSearcher.java +++ b/container-search/src/main/java/com/yahoo/prelude/semantics/SemanticSearcher.java @@ -84,7 +84,7 @@ public class SemanticSearcher extends Searcher { if (query.properties().getBoolean(rulesOff)) return execution.search(query); - int traceLevel = query.properties().getInteger(tracelevelRules, query.getTraceLevel() - 2); + int traceLevel = query.properties().getInteger(tracelevelRules, query.getTrace().getLevel() - 2); if (traceLevel < 0) traceLevel = 0; RuleBase ruleBase = resolveRuleBase(query); if (ruleBase == null) diff --git a/container-search/src/main/java/com/yahoo/search/Query.java b/container-search/src/main/java/com/yahoo/search/Query.java index dcbda80ecab..48885e4b3da 100644 --- a/container-search/src/main/java/com/yahoo/search/Query.java +++ b/container-search/src/main/java/com/yahoo/search/Query.java @@ -12,10 +12,10 @@ import com.yahoo.search.schema.SchemaInfo; import com.yahoo.search.dispatch.Dispatcher; import com.yahoo.search.federation.FederationSearcher; import com.yahoo.search.query.Model; +import com.yahoo.search.query.Trace; import com.yahoo.search.query.ParameterParser; import com.yahoo.search.query.Presentation; import com.yahoo.search.query.Properties; -import com.yahoo.search.query.QueryTree; import com.yahoo.search.query.Ranking; import com.yahoo.search.query.Select; import com.yahoo.search.query.SessionId; @@ -50,14 +50,11 @@ import java.nio.ByteBuffer; import java.util.ArrayList; import java.util.Collections; import java.util.HashMap; -import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Objects; import java.util.Set; import java.util.concurrent.TimeUnit; -import java.util.logging.Level; -import java.util.logging.Logger; /** * A search query containing all the information required to produce a Result. @@ -72,7 +69,7 @@ import java.util.logging.Logger; * </ul> * * <p> - * The properties has three sources + * The properties have three sources * <ol> * <li>They may be set in some Searcher component already executed for this Query - the properties acts as * a blackboard for communicating arbitrary objects between Searcher components. @@ -130,6 +127,9 @@ public class Query extends com.yahoo.processing.Request implements Cloneable { } + /** The time this query was created */ + private long startTime; + //-------------- Query properties treated as fields in Query --------------- /** The offset from the most relevant hits found from this query */ @@ -138,12 +138,6 @@ public class Query extends com.yahoo.processing.Request implements Cloneable { /** The number of hits to return */ private int hits = 10; - /** The query context level, 0 means no tracing */ - private int traceLevel = 0; - - /** The query explain level, 0 means no explaining */ - private int explainLevel = 0; - // The timeout to be used when dumping rank features private static final long dumpTimeout = (6 * 60 * 1000); // 6 minutes private static final long defaultTimeout = 500; @@ -181,12 +175,8 @@ public class Query extends com.yahoo.processing.Request implements Cloneable { /** The selection of where-clause and grouping */ private Select select = new Select(this); - //---------------- Tracing ---------------------------------------------------- - - private static final Logger log = Logger.getLogger(Query.class.getName()); - - /** The time this query was created */ - private long startTime; + /** How this query should be traced */ + public Trace trace = new Trace(this); //---------------- Static property handling ------------------------------------ @@ -195,12 +185,18 @@ public class Query extends com.yahoo.processing.Request implements Cloneable { public static final CompoundName QUERY_PROFILE = new CompoundName("queryProfile"); public static final CompoundName SEARCH_CHAIN = new CompoundName("searchChain"); - public static final CompoundName TRACE_LEVEL = new CompoundName("traceLevel"); - public static final CompoundName EXPLAIN_LEVEL = new CompoundName("explainLevel"); + public static final CompoundName NO_CACHE = new CompoundName("noCache"); public static final CompoundName GROUPING_SESSION_CACHE = new CompoundName("groupingSessionCache"); public static final CompoundName TIMEOUT = new CompoundName("timeout"); + /** @deprecated use Trace.LEVEL */ + @Deprecated // TODO: Remove on Vespa 9 + public static final CompoundName TRACE_LEVEL = new CompoundName("traceLevel"); + + /** @deprecated use Trace.EXPLAIN_LEVEL */ + @Deprecated // TODO: Remove on Vespa 9 + public static final CompoundName EXPLAIN_LEVEL = new CompoundName("explainLevel"); private static final QueryProfileType argumentType; static { @@ -213,8 +209,6 @@ public class Query extends com.yahoo.processing.Request implements Cloneable { argumentType.addField(new FieldDescription(HITS.toString(), "integer", "hits count")); argumentType.addField(new FieldDescription(QUERY_PROFILE.toString(), "string")); argumentType.addField(new FieldDescription(SEARCH_CHAIN.toString(), "string")); - argumentType.addField(new FieldDescription(TRACE_LEVEL.toString(), "integer", "tracelevel")); - argumentType.addField(new FieldDescription(EXPLAIN_LEVEL.toString(), "integer", "explainlevel")); argumentType.addField(new FieldDescription(NO_CACHE.toString(), "boolean", "nocache")); argumentType.addField(new FieldDescription(GROUPING_SESSION_CACHE.toString(), "boolean", "groupingSessionCache")); argumentType.addField(new FieldDescription(TIMEOUT.toString(), "string", "timeout")); @@ -225,6 +219,7 @@ public class Query extends com.yahoo.processing.Request implements Cloneable { argumentType.addField(new FieldDescription(Dispatcher.DISPATCH, new QueryProfileFieldType(Dispatcher.getArgumentType()))); argumentType.addField(new FieldDescription(Ranking.RANKING, new QueryProfileFieldType(Ranking.getArgumentType()))); argumentType.addField(new FieldDescription(Presentation.PRESENTATION, new QueryProfileFieldType(Presentation.getArgumentType()))); + argumentType.addField(new FieldDescription(Trace.TRACE, new QueryProfileFieldType(Trace.getArgumentType()))); argumentType.freeze(); } public static QueryProfileType getArgumentType() { return argumentType; } @@ -260,6 +255,7 @@ public class Query extends com.yahoo.processing.Request implements Cloneable { registry.register(Select.getArgumentType().unfrozen()); registry.register(Ranking.getArgumentType().unfrozen()); registry.register(Presentation.getArgumentType().unfrozen()); + registry.register(Trace.getArgumentType().unfrozen()); registry.register(DefaultProperties.argumentType.unfrozen()); } @@ -369,7 +365,7 @@ public class Query extends com.yahoo.processing.Request implements Cloneable { startTime = httpRequest.getJDiscRequest().creationTime(TimeUnit.MILLISECONDS); if (queryProfile != null) { // Move all request parameters to the query profile - Properties queryProfileProperties = new QueryProfileProperties(queryProfile, embedders); + Properties queryProfileProperties = new QueryProfileProperties(queryProfile, embedders, zoneInfo); properties().chain(queryProfileProperties); setPropertiesFromRequestMap(requestMap, properties(), true); @@ -377,11 +373,11 @@ public class Query extends com.yahoo.processing.Request implements Cloneable { properties().chain(new RankProfileInputProperties(schemaInfo, this, embedders)) .chain(new QueryProperties(this, queryProfile.getRegistry(), embedders)) .chain(new ModelObjectMap()) - .chain(new RequestContextProperties(requestMap, zoneInfo)) + .chain(new RequestContextProperties(requestMap)) .chain(queryProfileProperties) .chain(new DefaultProperties()); - // Pass the values from the query profile which maps through a field in the Query object model + // Pass values from the query profile which maps to a field in the Query object model // through the property chain to cause those values to be set in the Query object model with // the right types according to query profiles setFieldsFrom(queryProfileProperties, requestMap); @@ -403,7 +399,7 @@ public class Query extends com.yahoo.processing.Request implements Cloneable { } properties().setParentQuery(this); - traceProperties(); + trace.traceProperties(); } public Query(Query query) { @@ -478,59 +474,6 @@ public class Query extends com.yahoo.processing.Request implements Cloneable { public Properties properties() { return (Properties)super.properties(); } /** - * Traces how properties was resolved and from where. Done after the fact to avoid special handling - * of tracelevel, which is the property deciding whether this needs to be done - */ - private void traceProperties() { - if (traceLevel == 0) return; - CompiledQueryProfile profile = null; - QueryProfileProperties profileProperties = properties().getInstance(QueryProfileProperties.class); - if (profileProperties != null) - profile = profileProperties.getQueryProfile(); - - if (profile == null) - trace("No query profile is used", false, 1); - else - trace("Using " + profile.toString(), false, 1); - - if (traceLevel < 4) return; - StringBuilder b = new StringBuilder("Resolved properties:\n"); - Set<String> mentioned = new HashSet<>(); - for (Map.Entry<String,String> requestProperty : requestProperties().entrySet() ) { - Object resolvedValue = properties().get(requestProperty.getKey(), requestProperties()); - if (resolvedValue == null && requestProperty.getKey().equals("queryProfile")) - resolvedValue = requestProperty.getValue(); - - b.append(requestProperty.getKey()); - b.append(": "); - b.append(resolvedValue); // (may be null) - b.append(" ("); - - if (profile != null && ! profile.isOverridable(new CompoundName(requestProperty.getKey()), requestProperties())) - b.append("from query profile - unoverridable, ignoring request value"); - else - b.append("from request"); - b.append(")\n"); - mentioned.add(requestProperty.getKey()); - } - if (profile != null) { - appendQueryProfileProperties(profile, mentioned, b); - } - trace(b.toString(),false,4); - } - - private Map<String, String> requestProperties() { - return httpRequest.propertyMap(); - } - - private void appendQueryProfileProperties(CompiledQueryProfile profile, Set<String> mentioned, StringBuilder b) { - for (var property : profile.listValuesWithSources(CompoundName.empty, requestProperties(), properties()).entrySet()) { - if ( ! mentioned.contains(property.getKey())) - b.append(property.getKey()).append(": ").append(property.getValue()).append("\n"); - } - } - - /** * Validates this query * * @return the reason if it is invalid, null if it is valid @@ -599,35 +542,30 @@ public class Query extends com.yahoo.processing.Request implements Cloneable { */ public void resetTimeout() { this.startTime = System.currentTimeMillis(); } - /** - * Sets the context level of this query, 0 means no tracing - * Higher numbers means increasingly more tracing - */ - public void setTraceLevel(int traceLevel) { this.traceLevel = traceLevel; } - /** - * Sets the explain level of this query, 0 means no tracing - * Higher numbers means increasingly more explaining - */ - public void setExplainLevel(int explainLevel) { this.explainLevel = explainLevel; } + /** @deprecated use getTrace().setLevel(level) */ + @Deprecated // TODO: Remove on Vespa 9 + public void setTraceLevel(int traceLevel) { trace.setLevel(traceLevel); } - /** - * Returns the context level of this query, 0 means no tracing - * Higher numbers means increasingly more tracing - */ - public int getTraceLevel() { return traceLevel; } + /** @deprecated use getTrace().setExplainLevel(level) */ + @Deprecated // TODO: Remove on Vespa 9 + public void setExplainLevel(int explainLevel) { trace.setExplainLevel(explainLevel); } - /** - * Returns the explain level of this query, 0 means no tracing - * Higher numbers means increasingly more explaining - */ - public int getExplainLevel() { return explainLevel; } + /** @deprecated use getTrace().setLevel(level) */ + @Deprecated // TODO: Remove on Vespa 9 + public int getTraceLevel() { return trace.getLevel(); } + + /** @deprecated use getTrace().getExplainLevel(level) */ + @Deprecated // TODO: Remove on Vespa 9 + public int getExplainLevel() { return getTrace().getExplainLevel(); } /** * Returns the context level of this query, 0 means no tracing * Higher numbers means increasingly more tracing + * + * @deprecated use getTrace().isTraceable(level) */ - public final boolean isTraceable(int level) { return traceLevel >= level; } - + @Deprecated // TODO: Remove on Vespa 9 + public final boolean isTraceable(int level) { return trace.isTraceable(level); } /** Returns whether this query should never be served from a cache. Default is false */ public boolean getNoCache() { return noCache; } @@ -711,65 +649,24 @@ public class Query extends com.yahoo.processing.Request implements Cloneable { return model.getQueryTree().encode(buffer); } - /** - * Adds a context message to this query and to the info log, - * if the context level of the query is sufficiently high. - * The context information will be carried over to the result at creation. - * The message parameter will be included <i>with</i> XML escaping. - * - * @param message the message to add - * @param traceLevel the context level of the message, this method will do nothing - * if the traceLevel of the query is lower than this value - */ + /** Calls getTrace().trace(message, traceLevel). */ public void trace(String message, int traceLevel) { - trace(message, false, traceLevel); + trace.trace(message, traceLevel); } + /** Calls getTrace().trace(message, traceLevel). */ public void trace(Object message, int traceLevel) { - if ( ! isTraceable(traceLevel)) return; - getContext(true).trace(message, 0); + trace.trace(message, traceLevel); } - /** - * Adds a trace message to this query - * if the trace level of the query is sufficiently high. - * - * @param message the message to add - * @param includeQuery true to append the query root stringValue at the end of the message - * @param traceLevel the context level of the message, this method will do nothing - * if the traceLevel of the query is lower than this value - */ + /** Calls getTrace().trace(message, includeQuery, traceLevel). */ public void trace(String message, boolean includeQuery, int traceLevel) { - if ( ! isTraceable(traceLevel)) return; - - if (includeQuery) - message += ": [" + queryTreeText() + "]"; - - log.log(Level.FINE,message); - - // Pass 0 as traceLevel as the trace level check is already done above, - // and it is not propagated to trace until execution has started - // (it is done in the execution.search method) - getContext(true).trace(message, 0); + trace.trace(message, includeQuery, traceLevel); } - /** - * Adds a trace message to this query - * if the trace level of the query is sufficiently high. - * - * @param includeQuery true to append the query root stringValue at the end of the message - * @param traceLevel the context level of the message, this method will do nothing - * if the traceLevel of the query is lower than this value - * @param messages the messages whose toStrings will be concatenated into the trace message. - * Concatenation will only happen if the trace level is sufficiently high. - */ + /** Calls getTrace().trace(message, traceLevel, messages). */ public void trace(boolean includeQuery, int traceLevel, Object... messages) { - if ( ! isTraceable(traceLevel)) return; - - StringBuilder concatenated = new StringBuilder(); - for (Object message : messages) - concatenated.append(message); - trace(concatenated.toString(), includeQuery, traceLevel); + trace.trace(includeQuery, traceLevel, messages); } /** @@ -780,47 +677,27 @@ public class Query extends com.yahoo.processing.Request implements Cloneable { * by an IllegalStateException. In other words, intended use is create the * new query, and attach the context to the invoking query as soon as the new * query is properly initialized. - * * <p> * This method will always set the argument query's context level to the context * level of this query. * - * @param query - * The query which should be traced as a part of this query. - * @throws IllegalStateException - * If the query given as argument already has context - * information. + * @param query the query which should be traced as a part of this query + * @throws IllegalStateException if the query given as argument already has context information */ public void attachContext(Query query) throws IllegalStateException { - query.setTraceLevel(getTraceLevel()); - query.setExplainLevel(getExplainLevel()); - if (context == null) { - // Nothing to attach to. This is about the same as - // getTraceLevel() == 0, - // but is a direct test of what will make the function superfluous. - return; - } + query.getTrace().setLevel(getTrace().getLevel()); + query.getTrace().setExplainLevel(getTrace().getExplainLevel()); + if (context == null) return; if (query.getContext(false) != null) { // If we added the other query's context info as a subnode in this // query's context tree, we would have to check for loops in the // context graph. If we simply created a new node without checking, // we might silently overwrite useful information. - throw new IllegalStateException("Query to attach already has context information stored."); + throw new IllegalStateException("Query to attach already has context information stored"); } query.context = context; } - private String queryTreeText() { - QueryTree root = getModel().getQueryTree(); - - if (getTraceLevel() < 2) - return root.toString(); - if (getTraceLevel() < 6) - return yqlRepresentation(); - else - return "\n" + yqlRepresentation() + "\n" + new TextualQueryRepresentation(root.getRoot()) + "\n"; - } - /** * Serialize this query as YQL+. This method will never throw exceptions, * but instead return a human readable error message if a problem occurred while @@ -1000,6 +877,7 @@ public class Query extends com.yahoo.processing.Request implements Cloneable { clone.model = model.cloneFor(clone); clone.select = select.cloneFor(clone); clone.ranking = ranking.cloneFor(clone); + clone.trace = trace.cloneFor(clone); clone.presentation = (Presentation) presentation.clone(); clone.context = getContext(true).cloneFor(clone); @@ -1008,8 +886,6 @@ public class Query extends com.yahoo.processing.Request implements Cloneable { assert (clone.properties().getParentQuery() == clone); clone.setTimeout(getTimeout()); - clone.setTraceLevel(getTraceLevel()); - clone.setExplainLevel(getExplainLevel()); clone.setHits(getHits()); clone.setOffset(getOffset()); clone.setNoCache(getNoCache()); @@ -1029,6 +905,9 @@ public class Query extends com.yahoo.processing.Request implements Cloneable { /** Returns the query representation model to be used for this query, never null */ public Model getModel() { return model; } + /** Returns the trace settings and facade API. */ + public Trace getTrace() { return trace; } + /** * Return the HTTP request which caused this query. This will never be null * when running with queries from the network. diff --git a/container-search/src/main/java/com/yahoo/search/Result.java b/container-search/src/main/java/com/yahoo/search/Result.java index b31a4fb6e24..1fd69f15012 100644 --- a/container-search/src/main/java/com/yahoo/search/Result.java +++ b/container-search/src/main/java/com/yahoo/search/Result.java @@ -245,7 +245,7 @@ public final class Result extends com.yahoo.processing.Response implements Clone * @param name the name of the searcher instance returning this result */ public void trace(String name) { - if (hits().getQuery().getTraceLevel() < 5) { + if (hits().getQuery().getTrace().getLevel() < 5) { return; } StringBuilder hitBuffer = new StringBuilder(name); diff --git a/container-search/src/main/java/com/yahoo/search/Searcher.java b/container-search/src/main/java/com/yahoo/search/Searcher.java index 473adfa17db..63dba2864f7 100644 --- a/container-search/src/main/java/com/yahoo/search/Searcher.java +++ b/container-search/src/main/java/com/yahoo/search/Searcher.java @@ -163,7 +163,7 @@ public abstract class Searcher extends Processor { } else { int fillRejectTraceAt = 3; - if (result.getQuery().getTraceLevel() >= fillRejectTraceAt) + if (result.getQuery().getTrace().getLevel() >= fillRejectTraceAt) result.getQuery().trace("Ignoring fill(" + summaryClass + "): " + ( result.hits().getFilled() == null ? "Hits are unfillable" : "Hits already filled" ) + ": result.hits().getFilled()=" + result.hits().getFilled(), fillRejectTraceAt); diff --git a/container-search/src/main/java/com/yahoo/search/cluster/ClusterSearcher.java b/container-search/src/main/java/com/yahoo/search/cluster/ClusterSearcher.java index 71758666b99..4af6757db8c 100644 --- a/container-search/src/main/java/com/yahoo/search/cluster/ClusterSearcher.java +++ b/container-search/src/main/java/com/yahoo/search/cluster/ClusterSearcher.java @@ -162,7 +162,7 @@ public abstract class ClusterSearcher<T> extends PingableSearcher implements Nod if (timedOut(query)) return new Result(query, ErrorMessage.createTimeout("No time left for searching")); - if (query.getTraceLevel() >= 8) + if (query.getTrace().getLevel() >= 8) query.trace("Trying " + connection, false, 8); result = robustSearch(query, execution, connection); @@ -170,7 +170,7 @@ public abstract class ClusterSearcher<T> extends PingableSearcher implements Nod if ( ! shouldRetry(query, result)) return result; - if (query.getTraceLevel() >= 6) + if (query.getTrace().getLevel() >= 6) query.trace("Error from connection " + connection + " : " + result.hits().getError(), false, 6); if (result.hits().getError().getCode() == Error.TIMEOUT.code) diff --git a/container-search/src/main/java/com/yahoo/search/dispatch/rpc/ProtobufSerialization.java b/container-search/src/main/java/com/yahoo/search/dispatch/rpc/ProtobufSerialization.java index 82c570a9975..8b2457606ab 100644 --- a/container-search/src/main/java/com/yahoo/search/dispatch/rpc/ProtobufSerialization.java +++ b/container-search/src/main/java/com/yahoo/search/dispatch/rpc/ProtobufSerialization.java @@ -89,12 +89,12 @@ public class ProtobufSerialization { } public static int getTraceLevelForBackend(Query query) { - int traceLevel = query.getTraceLevel(); + int traceLevel = query.getTrace().getLevel(); if (query.getModel().getExecution().trace().getForceTimestamps()) { traceLevel = Math.max(traceLevel, 5); // Backend produces timing information on level 4 and 5 } - if (query.getExplainLevel() > 0) { - traceLevel = Math.max(traceLevel, query.getExplainLevel() + 5); + if (query.getTrace().getExplainLevel() > 0) { + traceLevel = Math.max(traceLevel, query.getTrace().getExplainLevel() + 5); } return traceLevel; } @@ -157,7 +157,7 @@ public class ProtobufSerialization { if (includeQueryData) { mergeQueryDataToDocsumRequest(query, builder); } - if (query.getTraceLevel() >= 3) { + if (query.getTrace().getLevel() >= 3) { query.trace((includeQueryData ? "ProtoBuf: Resending " : "Not resending ") + "query during document summary fetching", 3); } @@ -250,7 +250,7 @@ public class ProtobufSerialization { if ( ! slimeTrace.isEmpty()) { var traces = new Value.ArrayValue(); traces.add(new SlimeAdapter(BinaryFormat.decode(slimeTrace.toByteArray()).get())); - query.trace(traces, query.getTraceLevel()); + query.trace(traces, query.getTrace().getLevel()); } return result; } diff --git a/container-search/src/main/java/com/yahoo/search/federation/FederationSearcher.java b/container-search/src/main/java/com/yahoo/search/federation/FederationSearcher.java index adf03340c6c..21b4d1d538f 100644 --- a/container-search/src/main/java/com/yahoo/search/federation/FederationSearcher.java +++ b/container-search/src/main/java/com/yahoo/search/federation/FederationSearcher.java @@ -487,7 +487,7 @@ public class FederationSearcher extends ForkingSearcher { private void traceTargets(Query query, Collection<Target> targets) { int traceFederationLevel = 2; - if ( ! query.isTraceable(traceFederationLevel)) return; + if ( ! query.getTrace().isTraceable(traceFederationLevel)) return; query.trace("Federating to " + targets, traceFederationLevel); } @@ -537,7 +537,7 @@ public class FederationSearcher extends ForkingSearcher { } } - if (query.getTraceLevel()>=4) + if (query.getTrace().getLevel()>=4) query.trace("Got " + group.getConcreteSize() + " hits from " + group.getId(),false, 4); mergedResults.hits().add(group); } diff --git a/container-search/src/main/java/com/yahoo/search/grouping/vespa/GroupingExecutor.java b/container-search/src/main/java/com/yahoo/search/grouping/vespa/GroupingExecutor.java index 694aa76e5f8..2ba33f60ea1 100644 --- a/container-search/src/main/java/com/yahoo/search/grouping/vespa/GroupingExecutor.java +++ b/container-search/src/main/java/com/yahoo/search/grouping/vespa/GroupingExecutor.java @@ -219,7 +219,7 @@ public class GroupingExecutor extends Searcher { if (lastPass > 0) { baseRoot = origRoot.clone(); } - if (query.isTraceable(3) && query.getGroupingSessionCache()) { + if (query.getTrace().isTraceable(3) && query.getGroupingSessionCache()) { query.trace("Grouping in " + (lastPass + 1) + " passes. SessionId='" + query.getSessionId() + "'.", 3); } for (int pass = 0; pass <= lastPass; ++pass) { @@ -242,7 +242,7 @@ public class GroupingExecutor extends Searcher { // noinspection ConstantConditions passRoot = baseRoot.clone(); } - if (query.isTraceable(4) && query.getGroupingSessionCache()) { + if (query.getTrace().isTraceable(4) && query.getGroupingSessionCache()) { query.trace("Grouping with session cache '" + query.getGroupingSessionCache() + "' enabled for pass #" + pass + ".", 4); } if (origRoot != passRoot) { diff --git a/container-search/src/main/java/com/yahoo/search/pagetemplates/result/PageTemplatesXmlRenderer.java b/container-search/src/main/java/com/yahoo/search/pagetemplates/result/PageTemplatesXmlRenderer.java index 6d69a2cb877..98789480a1e 100644 --- a/container-search/src/main/java/com/yahoo/search/pagetemplates/result/PageTemplatesXmlRenderer.java +++ b/container-search/src/main/java/com/yahoo/search/pagetemplates/result/PageTemplatesXmlRenderer.java @@ -122,7 +122,7 @@ public class PageTemplatesXmlRenderer extends AsynchronousSectionedRenderer<Resu } private void queryContext(XMLWriter writer, Query owner) { - if (owner.getTraceLevel()!=0) { + if (owner.getTrace().getLevel()!=0) { XMLWriter xmlWriter=XMLWriter.from(writer); xmlWriter.openTag("meta").attribute("type", QueryContext.ID); TraceNode traceRoot = owner.getModel().getExecution().trace().traceNode().root(); diff --git a/container-search/src/main/java/com/yahoo/search/query/Model.java b/container-search/src/main/java/com/yahoo/search/query/Model.java index dbaab3045bf..63fc386963a 100644 --- a/container-search/src/main/java/com/yahoo/search/query/Model.java +++ b/container-search/src/main/java/com/yahoo/search/query/Model.java @@ -248,7 +248,7 @@ public class Model implements Cloneable { try { Parser parser = ParserFactory.newInstance(type, ParserEnvironment.fromExecutionContext(execution.context())); queryTree = parser.parse(Parsable.fromQueryModel(this)); - if (parent.getTraceLevel() >= 2) + if (parent.getTrace().getLevel() >= 2) parent.trace("Query parsed to: " + parent.yqlRepresentation(), 2); } catch (IllegalArgumentException e) { diff --git a/container-search/src/main/java/com/yahoo/search/query/Trace.java b/container-search/src/main/java/com/yahoo/search/query/Trace.java new file mode 100644 index 00000000000..9f056b14c21 --- /dev/null +++ b/container-search/src/main/java/com/yahoo/search/query/Trace.java @@ -0,0 +1,242 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.search.query; + +import com.yahoo.api.annotations.Beta; +import com.yahoo.prelude.query.textualrepresentation.TextualQueryRepresentation; +import com.yahoo.processing.request.CompoundName; +import com.yahoo.search.Query; +import com.yahoo.search.query.profile.QueryProfileProperties; +import com.yahoo.search.query.profile.compiled.CompiledQueryProfile; +import com.yahoo.search.query.profile.types.FieldDescription; +import com.yahoo.search.query.profile.types.QueryProfileType; + +import java.util.HashSet; +import java.util.Map; +import java.util.Objects; +import java.util.Set; +import java.util.logging.Level; +import java.util.logging.Logger; + +/** + * Trace settings and methods for tracing a query. + * The actual trace is a tree structure stored in the query execution. + * + * @author bratseth + */ +@Beta +public class Trace implements Cloneable { + + private static final Logger log = Logger.getLogger(Trace.class.getName()); + + /** The type representing the property arguments consumed by this */ + private static final QueryProfileType argumentType; + + public static final String TRACE = "trace"; + public static final String LEVEL = "level"; + public static final String EXPLAIN_LEVEL = "explainLevel"; + public static final String TIMESTAMPS = "timestamps"; + public static final String QUERY = "query"; + + static { + argumentType = new QueryProfileType(TRACE); + argumentType.setStrict(true); + argumentType.setBuiltin(true); + argumentType.addField(new FieldDescription(LEVEL, "integer", "tracelevel traceLevel")); + argumentType.addField(new FieldDescription(EXPLAIN_LEVEL, "integer", "explainlevel explainLevel")); + argumentType.addField(new FieldDescription(TIMESTAMPS, "boolean")); + argumentType.addField(new FieldDescription(QUERY, "boolean")); + argumentType.freeze(); + } + + public static QueryProfileType getArgumentType() { return argumentType; } + + private Query parent; + + private int level = 0; + private int explainLevel = 0; + private boolean timestamps = false; + private boolean query = true; + + public Trace(Query parent) { + this.parent = Objects.requireNonNull(parent); + } + + /** Returns the level of detail we'll be tracing at in this query. The default level is 0; no tracing. */ + public int getLevel() { return level; } + public void setLevel(int level) { this.level = level; } + public boolean isTraceable(int level) { return level <= this.level; } + + /** Sets the explain level of this query, 0 means no tracing. Higher numbers means increasingly more explaining. */ + public void setExplainLevel(int explainLevel) { this.explainLevel = explainLevel; } + public int getExplainLevel() { return explainLevel; } + + /** Returns whether trace entries should have a timestamp. Default is false. */ + public boolean getTimestamps() { return timestamps; } + public void setTimestamps(boolean timestamps) { this.timestamps = timestamps; } + + /** Returns whether any trace entries should include the query. Default is true. */ + public boolean getQuery() { return query; } + public void setQuery(boolean query) { this.query = query; } + + /** + * Adds a context message to this query and to the info log, + * if the context level of the query is sufficiently high. + * The context information will be carried over to the result at creation. + * The message parameter will be included <i>with</i> XML escaping. + * + * @param message the message to add + * @param traceLevel the context level of the message, this method will do nothing + * if the traceLevel of the query is lower than this value + */ + public void trace(String message, int traceLevel) { + trace(message, false, traceLevel); + } + + public void trace(Object message, int traceLevel) { + if ( ! isTraceable(traceLevel)) return; + parent.getContext(true).trace(message, 0); + } + + /** + * Adds a trace message to this query + * if the trace level of the query is sufficiently high. + * + * @param message the message to add + * @param includeQuery true to append the query root stringValue at the end of the message + * @param traceLevel the context level of the message, this method will do nothing + * if the traceLevel of the query is lower than this value + */ + public void trace(String message, boolean includeQuery, int traceLevel) { + if ( ! isTraceable(traceLevel)) return; + + if (includeQuery && query) + message += ": [" + queryTreeText() + "]"; + + log.log(Level.FINE, message); + + // Pass 0 as traceLevel as the trace level check is already done above, + // and it is not propagated to trace until execution has started + // (it is done in the execution.search method) + parent.getContext(true).trace(message, 0); + } + + /** + * Adds a trace message to this query + * if the trace level of the query is sufficiently high. + * + * @param includeQuery true to append the query root stringValue at the end of the message + * @param traceLevel the context level of the message, this method will do nothing + * if the traceLevel of the query is lower than this value + * @param messages the messages whose toStrings will be concatenated into the trace message. + * Concatenation will only happen if the trace level is sufficiently high. + */ + public void trace(boolean includeQuery, int traceLevel, Object... messages) { + if ( ! isTraceable(traceLevel)) return; + + StringBuilder concatenated = new StringBuilder(); + for (Object message : messages) + concatenated.append(message); + trace(concatenated.toString(), includeQuery, traceLevel); + } + + /** + * Traces how properties was resolved and from where. Done after the fact to avoid special handling + * of tracelevel, which is the property deciding whether this needs to be done + */ + public void traceProperties() { + if (level == 0) return; + CompiledQueryProfile profile = null; + QueryProfileProperties profileProperties = parent.properties().getInstance(QueryProfileProperties.class); + if (profileProperties != null) + profile = profileProperties.getQueryProfile(); + + if (profile == null) + trace("No query profile is used", false, 1); + else + trace("Using " + profile.toString(), false, 1); + + if (level < 4) return; + StringBuilder b = new StringBuilder("Resolved properties:\n"); + Set<String> mentioned = new HashSet<>(); + for (Map.Entry<String,String> requestProperty : requestProperties().entrySet() ) { + Object resolvedValue = parent.properties().get(requestProperty.getKey(), requestProperties()); + if (resolvedValue == null && requestProperty.getKey().equals("queryProfile")) + resolvedValue = requestProperty.getValue(); + + b.append(requestProperty.getKey()); + b.append(": "); + b.append(resolvedValue); // (may be null) + b.append(" ("); + + if (profile != null && ! profile.isOverridable(new CompoundName(requestProperty.getKey()), requestProperties())) + b.append("from query profile - unoverridable, ignoring request value"); + else + b.append("from request"); + b.append(")\n"); + mentioned.add(requestProperty.getKey()); + } + if (profile != null) { + appendQueryProfileProperties(profile, mentioned, b); + } + trace(b.toString(),false,4); + } + + private void appendQueryProfileProperties(CompiledQueryProfile profile, Set<String> mentioned, StringBuilder b) { + for (var property : profile.listValuesWithSources(CompoundName.empty, requestProperties(), parent.properties()).entrySet()) { + if ( ! mentioned.contains(property.getKey())) + b.append(property.getKey()).append(": ").append(property.getValue()).append("\n"); + } + } + + private Map<String, String> requestProperties() { + return parent.getHttpRequest().propertyMap(); + } + + private String queryTreeText() { + QueryTree root = parent.getModel().getQueryTree(); + + if (level < 2) + return root.toString(); + if (level < 6) + return parent.yqlRepresentation(); + else + return "\n" + parent.yqlRepresentation() + "\n" + new TextualQueryRepresentation(root.getRoot()) + "\n"; + } + + public Trace cloneFor(Query parent) { + Trace trace = this.clone(); + trace.parent = parent; + return trace; + } + + @Override + public boolean equals(Object o) { + if (o == this ) return true; + if ( ! (o instanceof Trace)) return false; + Trace other = (Trace)o; + if (other.level != this.level) return false; + if (other.explainLevel != this.explainLevel) return false; + if (other.timestamps != this.timestamps) return false; + if (other.query != this.query) return false; + return true; + } + + @Override + public int hashCode() { return Objects.hash(level, explainLevel, timestamps, query); } + + @Override + public Trace clone() { + try { + return (Trace)super.clone(); + } + catch (CloneNotSupportedException e) { + throw new RuntimeException(e); + } + } + + @Override + public String toString() { + return "trace [level: " + level + ", explainLevel: " + explainLevel + ", timestamps: " + timestamps + ", query: " + query + "]"; + } + +} diff --git a/container-search/src/main/java/com/yahoo/search/query/context/QueryContext.java b/container-search/src/main/java/com/yahoo/search/query/context/QueryContext.java index 38f76c6ca49..c0ca5ac4731 100644 --- a/container-search/src/main/java/com/yahoo/search/query/context/QueryContext.java +++ b/container-search/src/main/java/com/yahoo/search/query/context/QueryContext.java @@ -87,7 +87,7 @@ public class QueryContext implements Cloneable { } public boolean render(Writer writer) throws java.io.IOException { - if (owner.getTraceLevel()!=0) { + if (owner.getTrace().getLevel()!=0) { XMLWriter xmlWriter=XMLWriter.from(writer); xmlWriter.openTag("meta").attribute("type",ID); TraceNode traceRoot=owner.getModel().getExecution().trace().traceNode().root(); diff --git a/container-search/src/main/java/com/yahoo/search/query/profile/QueryProfile.java b/container-search/src/main/java/com/yahoo/search/query/profile/QueryProfile.java index 989f12172b3..e3ab49f0e32 100644 --- a/container-search/src/main/java/com/yahoo/search/query/profile/QueryProfile.java +++ b/container-search/src/main/java/com/yahoo/search/query/profile/QueryProfile.java @@ -344,7 +344,7 @@ public class QueryProfile extends FreezableSimpleComponent implements Cloneable * a primitive value, a substitutable string, a query profile, or null if not found. */ public final Object lookup(String name, Map<String, String> context) { - return lookup(new CompoundName(name),true,DimensionBinding.createFrom(getDimensions(),context)); + return lookup(new CompoundName(name), true, DimensionBinding.createFrom(getDimensions(),context)); } /** Sets a value in this or any nested profile using null as context */ @@ -733,7 +733,7 @@ public class QueryProfile extends FreezableSimpleComponent implements Cloneable // ----------------- Private ---------------------------------------------------------------------------------- - private Boolean isDeclaredOverridable(CompoundName name,DimensionBinding dimensionBinding) { + private Boolean isDeclaredOverridable(CompoundName name, DimensionBinding dimensionBinding) { QueryProfile parent = lookupParentExact(name, true, dimensionBinding); if (parent.overridable == null) return null; return parent.overridable.get(name.last()); @@ -743,15 +743,15 @@ public class QueryProfile extends FreezableSimpleComponent implements Cloneable * Sets the overridability of a field in this profile, * this overrides the corresponding setting in the type (if any) */ - private void setOverridable(CompoundName fieldName, boolean overridable, DimensionBinding dimensionBinding) { - QueryProfile parent = lookupParentExact(fieldName, true, dimensionBinding); + private void setOverridable(CompoundName name, boolean overridable, DimensionBinding dimensionBinding) { + QueryProfile parent = lookupParentExact(name, true, dimensionBinding); if (dimensionBinding.isNull()) { if (parent.overridable == null) parent.overridable = new HashMap<>(); - parent.overridable.put(fieldName.last(), overridable); + parent.overridable.put(name.last(), overridable); } else { - variants.setOverridable(fieldName.last(), overridable, dimensionBinding.getValues()); + variants.setOverridable(name.last(), overridable, dimensionBinding.getValues()); } } diff --git a/container-search/src/main/java/com/yahoo/search/query/profile/QueryProfileProperties.java b/container-search/src/main/java/com/yahoo/search/query/profile/QueryProfileProperties.java index abd23c1822d..8c1a0ac1d25 100644 --- a/container-search/src/main/java/com/yahoo/search/query/profile/QueryProfileProperties.java +++ b/container-search/src/main/java/com/yahoo/search/query/profile/QueryProfileProperties.java @@ -1,6 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.search.query.profile; +import ai.vespa.cloud.ZoneInfo; import com.yahoo.collections.Pair; import com.yahoo.language.process.Embedder; import com.yahoo.processing.IllegalInputException; @@ -32,6 +33,7 @@ public class QueryProfileProperties extends Properties { private final CompiledQueryProfile profile; private final Map<String, Embedder> embedders; + private final ZoneInfo zoneInfo; // Note: The priority order is: values has precedence over references @@ -46,18 +48,26 @@ public class QueryProfileProperties extends Properties { private List<Pair<CompoundName, CompiledQueryProfile>> references = null; public QueryProfileProperties(CompiledQueryProfile profile) { - this(profile, Embedder.throwsOnUse.asMap()); + this(profile, Embedder.throwsOnUse.asMap(), ZoneInfo.defaultInfo()); } + @Deprecated // TODO: Remove on Vespa 9 public QueryProfileProperties(CompiledQueryProfile profile, Embedder embedder) { - this(profile, Map.of(Embedder.defaultEmbedderId, embedder)); + this(profile, Map.of(Embedder.defaultEmbedderId, embedder), ZoneInfo.defaultInfo()); } /** Creates an instance from a profile, throws an exception if the given profile is null */ + @Deprecated // TODO: Remove on Vespa 9 public QueryProfileProperties(CompiledQueryProfile profile, Map<String, Embedder> embedders) { + this(profile, embedders, ZoneInfo.defaultInfo()); + } + + /** Creates an instance from a profile, throws an exception if the given profile is null */ + public QueryProfileProperties(CompiledQueryProfile profile, Map<String, Embedder> embedders, ZoneInfo zoneInfo) { Validator.ensureNotNull("The profile wrapped by this cannot be null", profile); this.profile = profile; this.embedders = embedders; + this.zoneInfo = zoneInfo; } /** Returns the query profile backing this, or null if none */ @@ -67,6 +77,7 @@ public class QueryProfileProperties extends Properties { @Override public Object get(CompoundName name, Map<String, String> context, com.yahoo.processing.request.Properties substitution) { + context = contextWithZoneInfo(context); name = unalias(name, context); if (values != null && values.containsKey(name)) return values.get(name); // Returns this value, even if null @@ -92,11 +103,13 @@ public class QueryProfileProperties extends Properties { */ @Override public void set(CompoundName name, Object value, Map<String, String> context) { + context = contextWithZoneInfo(context); setOrCheckSettable(name, value, context, true); } @Override public void requireSettable(CompoundName name, Object value, Map<String, String> context) { + context = contextWithZoneInfo(context); setOrCheckSettable(name, value, context, false); } @@ -210,6 +223,8 @@ public class QueryProfileProperties extends Properties { @Override public Map<String, Object> listProperties(CompoundName path, Map<String, String> context, com.yahoo.processing.request.Properties substitution) { + context = contextWithZoneInfo(context); + path = unalias(path, context); if (context == null) context = Collections.emptyMap(); @@ -257,7 +272,7 @@ public class QueryProfileProperties extends Properties { return properties; } - public boolean isComplete(StringBuilder firstMissingName, Map<String,String> context) { + public boolean isComplete(StringBuilder firstMissingName, Map<String, String> context) { // Are all types reachable from this complete? if ( ! reachableTypesAreComplete(CompoundName.empty, profile, firstMissingName, context)) return false; @@ -272,6 +287,16 @@ public class QueryProfileProperties extends Properties { return true; } + private Map<String, String> contextWithZoneInfo(Map<String, String> context) { + if (zoneInfo == ZoneInfo.defaultInfo()) return context; + + Map<String, String> contextWithZoneInfo = context == null ? new HashMap<>() : new HashMap<>(context); + contextWithZoneInfo.putIfAbsent("environment", zoneInfo.zone().environment().name()); + contextWithZoneInfo.putIfAbsent("region", zoneInfo.zone().region()); + contextWithZoneInfo.putIfAbsent("instance", zoneInfo.application().instance()); + return Collections.unmodifiableMap(contextWithZoneInfo); + } + private boolean reachableTypesAreComplete(CompoundName prefix, CompiledQueryProfile profile, StringBuilder firstMissingName, Map<String,String> context) { for (Map.Entry<CompoundName, DimensionalValue<QueryProfileType>> typeEntry : profile.getTypes().entrySet()) { QueryProfileType type = typeEntry.getValue().get(context); diff --git a/container-search/src/main/java/com/yahoo/search/query/properties/QueryProperties.java b/container-search/src/main/java/com/yahoo/search/query/properties/QueryProperties.java index 4cced8d7923..f6e158cf04a 100644 --- a/container-search/src/main/java/com/yahoo/search/query/properties/QueryProperties.java +++ b/container-search/src/main/java/com/yahoo/search/query/properties/QueryProperties.java @@ -11,6 +11,7 @@ import com.yahoo.search.query.Presentation; import com.yahoo.search.query.Properties; import com.yahoo.search.query.Ranking; import com.yahoo.search.query.Select; +import com.yahoo.search.query.Trace; import com.yahoo.search.query.profile.compiled.CompiledQueryProfileRegistry; import com.yahoo.search.query.profile.types.ConversionContext; import com.yahoo.search.query.profile.types.FieldDescription; @@ -132,11 +133,16 @@ public class QueryProperties extends Properties { } else if (key.size() == 3 && key.get(1).equals(Presentation.FORMAT)) { if (key.last().equals(Presentation.TENSORS)) return query.getPresentation().getTensorShortForm(); } - } else if (key.size() == 1) { + } + else if (key.size() == 2 && key.first().equals(Trace.TRACE)) { + if (key.last().equals(Trace.LEVEL)) return query.getTrace().getLevel(); + if (key.last().equals(Trace.EXPLAIN_LEVEL)) return query.getTrace().getExplainLevel(); + if (key.last().equals(Trace.TIMESTAMPS)) return query.getTrace().getTimestamps(); + if (key.last().equals(Trace.QUERY)) return query.getTrace().getQuery(); + } + else if (key.size() == 1) { if (key.equals(Query.HITS)) return query.getHits(); if (key.equals(Query.OFFSET)) return query.getOffset(); - if (key.equals(Query.TRACE_LEVEL)) return query.getTraceLevel(); - if (key.equals(Query.EXPLAIN_LEVEL)) return query.getExplainLevel(); if (key.equals(Query.TIMEOUT)) return query.getTimeout(); if (key.equals(Query.NO_CACHE)) return query.getNoCache(); if (key.equals(Query.GROUPING_SESSION_CACHE)) return query.getGroupingSessionCache(); @@ -301,6 +307,16 @@ public class QueryProperties extends Properties { else throwIllegalParameter(key.last(), Presentation.PRESENTATION); } + else if (key.size() == 2 && key.first().equals(Trace.TRACE)) { + if (key.last().equals(Trace.LEVEL)) + query.getTrace().setLevel(asInteger(value, 0)); + if (key.last().equals(Trace.EXPLAIN_LEVEL)) + query.getTrace().setExplainLevel(asInteger(value, 0)); + if (key.last().equals(Trace.TIMESTAMPS)) + query.getTrace().setTimestamps(asBoolean(value, false)); + if (key.last().equals(Trace.QUERY)) + query.getTrace().setQuery(asBoolean(value, true)); + } else if (key.first().equals(Select.SELECT)) { if (key.size() == 1) { query.getSelect().setGroupingExpressionString(asString(value, "")); @@ -322,10 +338,6 @@ public class QueryProperties extends Properties { query.setHits(asInteger(value,10)); else if (key.equals(Query.OFFSET)) query.setOffset(asInteger(value,0)); - else if (key.equals(Query.TRACE_LEVEL)) - query.setTraceLevel(asInteger(value,0)); - else if (key.equals(Query.EXPLAIN_LEVEL)) - query.setExplainLevel(asInteger(value,0)); else if (key.equals(Query.TIMEOUT)) query.setTimeout(value.toString()); else if (key.equals(Query.NO_CACHE)) diff --git a/container-search/src/main/java/com/yahoo/search/query/properties/RequestContextProperties.java b/container-search/src/main/java/com/yahoo/search/query/properties/RequestContextProperties.java index 46aafcb11e3..ad212d05780 100644 --- a/container-search/src/main/java/com/yahoo/search/query/properties/RequestContextProperties.java +++ b/container-search/src/main/java/com/yahoo/search/query/properties/RequestContextProperties.java @@ -21,17 +21,8 @@ public class RequestContextProperties extends Properties { private final Map<String, String> context; - public RequestContextProperties(Map<String, String> properties, ZoneInfo zoneInfo) { - if (zoneInfo == ZoneInfo.defaultInfo()) { - context = properties; - } - else { - Map<String, String> context = new HashMap<>(properties); - context.putIfAbsent("environment", zoneInfo.zone().environment().name()); - context.putIfAbsent("region", zoneInfo.zone().region()); - context.putIfAbsent("instance", zoneInfo.application().instance()); - this.context = Collections.unmodifiableMap(context); - } + public RequestContextProperties(Map<String, String> properties) { + this.context = Collections.unmodifiableMap(properties); } @Override diff --git a/container-search/src/main/java/com/yahoo/search/querytransform/BooleanSearcher.java b/container-search/src/main/java/com/yahoo/search/querytransform/BooleanSearcher.java index d3bb1e7a81d..f43be20e0ac 100644 --- a/container-search/src/main/java/com/yahoo/search/querytransform/BooleanSearcher.java +++ b/container-search/src/main/java/com/yahoo/search/querytransform/BooleanSearcher.java @@ -39,7 +39,7 @@ public class BooleanSearcher extends Searcher { if (fieldName != null) { return search(query, execution, fieldName); } else { - if (query.isTraceable(5)) { + if (query.getTrace().isTraceable(5)) { query.trace("BooleanSearcher: Nothing added to query", false, 5); } } @@ -49,7 +49,7 @@ public class BooleanSearcher extends Searcher { private Result search(Query query, Execution execution, String fieldName) { String attributes = query.properties().getString(ATTRIBUTES); String rangeAttributes = query.properties().getString(RANGE_ATTRIBUTES); - if (query.isTraceable(5)) { + if (query.getTrace().isTraceable(5)) { query.trace("BooleanSearcher: fieldName(" + fieldName + "), attributes(" + attributes + "), rangeAttributes(" + rangeAttributes + ")", false, 5); } @@ -57,7 +57,7 @@ public class BooleanSearcher extends Searcher { if (attributes != null || rangeAttributes != null) { try { addPredicateTerm(query, fieldName, attributes, rangeAttributes); - if (query.isTraceable(4)) { + if (query.getTrace().isTraceable(4)) { query.trace("BooleanSearcher: Added boolean operator", true, 4); } } catch (TokenMgrException e) { @@ -68,7 +68,7 @@ public class BooleanSearcher extends Searcher { } } else { - if (query.isTraceable(5)) { + if (query.getTrace().isTraceable(5)) { query.trace("BooleanSearcher: Nothing added to query", false, 5); } } diff --git a/container-search/src/main/java/com/yahoo/search/rendering/JsonRenderer.java b/container-search/src/main/java/com/yahoo/search/rendering/JsonRenderer.java index 250e66ca9fb..58353cc5907 100644 --- a/container-search/src/main/java/com/yahoo/search/rendering/JsonRenderer.java +++ b/container-search/src/main/java/com/yahoo/search/rendering/JsonRenderer.java @@ -224,7 +224,7 @@ public class JsonRenderer extends AsynchronousSectionedRenderer<Result> { protected void renderTrace(Trace trace) throws IOException { if (!trace.traceNode().children().iterator().hasNext()) return; - if (getResult().getQuery().getTraceLevel() == 0) return; + if (getResult().getQuery().getTrace().getLevel() == 0) return; try { long basetime = trace.traceNode().timestamp(); diff --git a/container-search/src/main/java/com/yahoo/search/rendering/XmlRenderer.java b/container-search/src/main/java/com/yahoo/search/rendering/XmlRenderer.java index f93c70c0199..53e59d9deea 100644 --- a/container-search/src/main/java/com/yahoo/search/rendering/XmlRenderer.java +++ b/container-search/src/main/java/com/yahoo/search/rendering/XmlRenderer.java @@ -160,7 +160,7 @@ public final class XmlRenderer extends AsynchronousSectionedRenderer<Result> { @SuppressWarnings("UnusedParameters") public void queryContext(XMLWriter writer, QueryContext queryContext, Query owner) throws IOException { - if (owner.getTraceLevel()!=0) { + if (owner.getTrace().getLevel()!=0) { XMLWriter xmlWriter=XMLWriter.from(writer); xmlWriter.openTag("meta").attribute("type", QueryContext.ID); TraceNode traceRoot = owner.getModel().getExecution().trace().traceNode().root(); diff --git a/container-search/src/main/java/com/yahoo/search/searchchain/Execution.java b/container-search/src/main/java/com/yahoo/search/searchchain/Execution.java index 02537e63a6b..988021a6da0 100644 --- a/container-search/src/main/java/com/yahoo/search/searchchain/Execution.java +++ b/container-search/src/main/java/com/yahoo/search/searchchain/Execution.java @@ -494,7 +494,7 @@ public class Execution extends com.yahoo.processing.execution.Execution { // Transfer state between query and execution as the execution constructors does not do that completely query.getModel().setExecution(this); - trace().setTraceLevel(query.getTraceLevel()); + trace().setTraceLevel(query.getTrace().getLevel()); return (Result)super.process(query); } @@ -504,7 +504,7 @@ public class Execution extends com.yahoo.processing.execution.Execution { super.onInvoking(request,processor); final int traceDependencies = 6; Query query = (Query) request; - if (query.getTraceLevel() >= traceDependencies) { + if (query.getTrace().getLevel() >= traceDependencies) { query.trace(processor.getId() + " " + processor.getDependencies(), traceDependencies); } } diff --git a/container-search/src/main/java/com/yahoo/search/searchchain/ExecutionFactory.java b/container-search/src/main/java/com/yahoo/search/searchchain/ExecutionFactory.java index e99dafef886..bfc4219eabc 100644 --- a/container-search/src/main/java/com/yahoo/search/searchchain/ExecutionFactory.java +++ b/container-search/src/main/java/com/yahoo/search/searchchain/ExecutionFactory.java @@ -91,19 +91,6 @@ public class ExecutionFactory extends AbstractComponent { this.executor = executor != null ? executor : Executors.newSingleThreadExecutor(); } - /** @deprecated pass SchemaInfoConfig */ - @Deprecated - public ExecutionFactory(ChainsConfig chainsConfig, - IndexInfoConfig indexInfo, - QrSearchersConfig clusters, - ComponentRegistry<Searcher> searchers, - SpecialtokensConfig specialTokens, - Linguistics linguistics, - ComponentRegistry<Renderer> renderers, - Executor executor) { - this(chainsConfig, indexInfo, SchemaInfo.empty(), clusters, searchers, specialTokens, linguistics, renderers, executor); - } - private SearchChainRegistry createSearchChainRegistry(ComponentRegistry<Searcher> searchers, ChainsConfig chainsConfig) { SearchChainRegistry searchChainRegistry = new SearchChainRegistry(searchers); diff --git a/container-search/src/main/java/com/yahoo/vespa/streamingvisitors/VdsStreamingSearcher.java b/container-search/src/main/java/com/yahoo/vespa/streamingvisitors/VdsStreamingSearcher.java index 7f4de2e43f5..51035e3e313 100644 --- a/container-search/src/main/java/com/yahoo/vespa/streamingvisitors/VdsStreamingSearcher.java +++ b/container-search/src/main/java/com/yahoo/vespa/streamingvisitors/VdsStreamingSearcher.java @@ -45,9 +45,9 @@ import java.util.logging.Logger; */ public class VdsStreamingSearcher extends VespaBackEndSearcher { - private static final CompoundName streamingUserid=new CompoundName("streaming.userid"); - private static final CompoundName streamingGroupname=new CompoundName("streaming.groupname"); - private static final CompoundName streamingSelection=new CompoundName("streaming.selection"); + private static final CompoundName streamingUserid = new CompoundName("streaming.userid"); + private static final CompoundName streamingGroupname = new CompoundName("streaming.groupname"); + private static final CompoundName streamingSelection = new CompoundName("streaming.selection"); static final String STREAMING_STATISTICS = "streaming.statistics"; private final VisitorFactory visitorFactory; @@ -142,13 +142,13 @@ public class VdsStreamingSearcher extends VespaBackEndSearcher { private boolean shouldTraceQuery(Query query) { // Only trace for explicit bucket subset queries, as otherwise we'd get a trace entry for every superbucket in the system. return (queryIsLocationConstrained(query) && - ((query.getTraceLevel() > 0) || tracingOptions.getSamplingStrategy().shouldSample())); + ((query.getTrace().getLevel() > 0) || tracingOptions.getSamplingStrategy().shouldSample())); } private int inferEffectiveQueryTraceLevel(Query query) { - return ((query.getTraceLevel() == 0) && shouldTraceQuery(query)) // Honor query's explicit trace level if present. + return ((query.getTrace().getLevel() == 0) && shouldTraceQuery(query)) // Honor query's explicit trace level if present. ? tracingOptions.getTraceLevelOverride() - : query.getTraceLevel(); + : query.getTrace().getLevel(); } @Override @@ -308,7 +308,7 @@ public class VdsStreamingSearcher extends VespaBackEndSearcher { } private static void lazyTrace(Query query, int level, Object... args) { - if (query.isTraceable(level)) { + if (query.getTrace().isTraceable(level)) { StringBuilder s = new StringBuilder(); for (Object arg : args) { s.append(arg); diff --git a/container-search/src/main/java/com/yahoo/vespa/streamingvisitors/VdsVisitor.java b/container-search/src/main/java/com/yahoo/vespa/streamingvisitors/VdsVisitor.java index 22f77e6f19d..bd96f888b87 100644 --- a/container-search/src/main/java/com/yahoo/vespa/streamingvisitors/VdsVisitor.java +++ b/container-search/src/main/java/com/yahoo/vespa/streamingvisitors/VdsVisitor.java @@ -94,7 +94,7 @@ class VdsVisitor extends VisitorDataHandler implements Visitor { } else if (log.isLoggable(Level.FINE)) { implicitLevel = 7; } - return Math.max(query.getTraceLevel(), implicitLevel); + return Math.max(query.getTrace().getLevel(), implicitLevel); } private static String createSelectionString(String documentType, String selection) { diff --git a/container-search/src/test/java/com/yahoo/prelude/querytransform/test/NormalizingSearcherTestCase.java b/container-search/src/test/java/com/yahoo/prelude/querytransform/test/NormalizingSearcherTestCase.java index 0694c1a7e55..7ab50118c6d 100644 --- a/container-search/src/test/java/com/yahoo/prelude/querytransform/test/NormalizingSearcherTestCase.java +++ b/container-search/src/test/java/com/yahoo/prelude/querytransform/test/NormalizingSearcherTestCase.java @@ -84,7 +84,7 @@ public class NormalizingSearcherTestCase { @Test public void testPhraseQuery() { Query query = new Query("/search?query=" + enc("\"b\u00e9yonc\u00e8 beyonc\u00e9\"") + "&search=cluster1&restrict=type1"); - query.setTraceLevel(2); + query.getTrace().setLevel(2); createExecution().search(query); assertEquals("WEAKAND(100) \"beyonce beyonce\"", query.getModel().getQueryTree().getRoot().toString()); } diff --git a/container-search/src/test/java/com/yahoo/prelude/searcher/test/JSONDebugSearcherTestCase.java b/container-search/src/test/java/com/yahoo/prelude/searcher/test/JSONDebugSearcherTestCase.java index 2841ce5521a..6a230da4950 100644 --- a/container-search/src/test/java/com/yahoo/prelude/searcher/test/JSONDebugSearcherTestCase.java +++ b/container-search/src/test/java/com/yahoo/prelude/searcher/test/JSONDebugSearcherTestCase.java @@ -14,36 +14,35 @@ import com.yahoo.search.searchchain.Execution; import com.yahoo.search.searchchain.testutil.DocumentSourceSearcher; import com.yahoo.yolean.trace.TraceNode; import com.yahoo.yolean.trace.TraceVisitor; -import org.junit.After; -import org.junit.Before; import org.junit.Test; -import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; /** * Visit the trace and check JSON payload is stored there when requested. * - * @author <a href="mailto:steinar@yahoo-inc.com">Steinar Knutsen</a> + * @author Steinar Knutsen */ public class JSONDebugSearcherTestCase { private static final String NODUMPJSON = "?query=1&tracelevel=6"; private static final String DUMPJSON = "?query=1&dumpjson=jsonfield&tracelevel=6"; - private Chain<Searcher> searchChain; - private static class LookForJson extends TraceVisitor { - private static final String JSON_PAYLOAD = "{1: 2}"; - public boolean gotJson = false; - - @Override - public void visit(TraceNode node) { - if (node.payload() == null || node.payload().getClass() != String.class) { - return; - } - if (node.payload().toString().equals(JSONDebugSearcher.JSON_FIELD + JSON_PAYLOAD)) { - gotJson = true; - } - } + @Test + public void test() { + Chain<Searcher> searchChain = makeSearchChain("{1: 2}", new JSONDebugSearcher()); + Execution e = new Execution(searchChain, Execution.Context.createContextStub()); + e.search(new Query(NODUMPJSON)); + Trace t = e.trace(); + LookForJson visitor = new LookForJson(); + t.accept(visitor); + assertFalse(visitor.gotJson); + e = new Execution(searchChain, Execution.Context.createContextStub()); + e.search(new Query(DUMPJSON)); + t = e.trace(); + t.accept(visitor); + assertTrue(visitor.gotJson); } private Chain<Searcher> makeSearchChain(String content, Searcher dumper) { @@ -63,29 +62,20 @@ public class JSONDebugSearcherTestCase { docsource.addResult(q, r); } + private static class LookForJson extends TraceVisitor { - @Before - public void setUp() throws Exception { - searchChain = makeSearchChain("{1: 2}", new JSONDebugSearcher()); - } - - @After - public void tearDown() throws Exception { - } + private static final String JSON_PAYLOAD = "{1: 2}"; + public boolean gotJson = false; - @Test - public final void test() { - Execution e = new Execution(searchChain, Execution.Context.createContextStub()); - e.search(new Query(NODUMPJSON)); - Trace t = e.trace(); - LookForJson visitor = new LookForJson(); - t.accept(visitor); - assertEquals(false, visitor.gotJson); - e = new Execution(searchChain, Execution.Context.createContextStub()); - e.search(new Query(DUMPJSON)); - t = e.trace(); - t.accept(visitor); - assertEquals(true, visitor.gotJson); + @Override + public void visit(TraceNode node) { + if (node.payload() == null || node.payload().getClass() != String.class) { + return; + } + if (node.payload().toString().equals(JSONDebugSearcher.JSON_FIELD + JSON_PAYLOAD)) { + gotJson = true; + } + } } } diff --git a/container-search/src/test/java/com/yahoo/search/dispatch/InterleavedSearchInvokerTest.java b/container-search/src/test/java/com/yahoo/search/dispatch/InterleavedSearchInvokerTest.java index 347276d680d..e07d38fbf10 100644 --- a/container-search/src/test/java/com/yahoo/search/dispatch/InterleavedSearchInvokerTest.java +++ b/container-search/src/test/java/com/yahoo/search/dispatch/InterleavedSearchInvokerTest.java @@ -488,7 +488,7 @@ public class InterleavedSearchInvokerTest { public TestQuery() { super(); setTimeout(5000); - setTraceLevel(5); + getTrace().setLevel(5); } @Override diff --git a/container-search/src/test/java/com/yahoo/search/query/profile/config/test/XmlReadingTestCase.java b/container-search/src/test/java/com/yahoo/search/query/profile/config/test/XmlReadingTestCase.java index b1faac2036b..62bc89c8453 100644 --- a/container-search/src/test/java/com/yahoo/search/query/profile/config/test/XmlReadingTestCase.java +++ b/container-search/src/test/java/com/yahoo/search/query/profile/config/test/XmlReadingTestCase.java @@ -49,80 +49,82 @@ public class XmlReadingTestCase { @Test public void testValid() { - QueryProfileRegistry registry= + QueryProfileRegistry registry = new QueryProfileXMLReader().read("src/test/java/com/yahoo/search/query/profile/config/test/validxml"); - CompiledQueryProfileRegistry cRegistry= registry.compile(); + CompiledQueryProfileRegistry cRegistry = registry.compile(); QueryProfileType rootType = registry.getType("rootType"); - assertEquals(1,rootType.inherited().size()); - assertEquals("native",rootType.inherited().get(0).getId().getName()); + assertEquals(1, rootType.inherited().size()); + assertEquals("native", rootType.inherited().get(0).getId().getName()); assertTrue(rootType.isStrict()); assertTrue(rootType.getMatchAsPath()); - FieldDescription timeField=rootType.getField("time"); + FieldDescription timeField = rootType.getField("time"); assertTrue(timeField.isMandatory()); - assertEquals("long",timeField.getType().toInstanceDescription()); - FieldDescription userField=rootType.getField("user"); + assertEquals("long", timeField.getType().toInstanceDescription()); + FieldDescription userField = rootType.getField("user"); assertFalse(userField.isMandatory()); - assertEquals("reference to a query profile of type 'user'",userField.getType().toInstanceDescription()); + assertEquals("reference to a query profile of type 'user'", userField.getType().toInstanceDescription()); - QueryProfileType user=registry.getType("user"); - assertEquals(0,user.inherited().size()); + QueryProfileType user = registry.getType("user"); + assertEquals(0, user.inherited().size()); assertFalse(user.isStrict()); assertFalse(user.getMatchAsPath()); assertTrue(userField.isOverridable()); - FieldDescription ageField=user.getField("age"); + FieldDescription ageField = user.getField("age"); assertTrue(ageField.isMandatory()); - assertEquals("integer",ageField.getType().toInstanceDescription()); - FieldDescription robotField=user.getField("robot"); + assertEquals("integer", ageField.getType().toInstanceDescription()); + FieldDescription robotField = user.getField("robot"); assertFalse(robotField.isMandatory()); assertFalse(robotField.isOverridable()); - assertEquals("boolean",robotField.getType().toInstanceDescription()); + assertEquals("boolean", robotField.getType().toInstanceDescription()); - CompiledQueryProfile defaultProfile=cRegistry.getComponent("default"); + CompiledQueryProfile defaultProfile = cRegistry.getComponent("default"); assertNull(defaultProfile.getType()); - assertEquals("20",defaultProfile.get("hits")); + assertEquals("20", defaultProfile.get("hits")); assertFalse(defaultProfile.isOverridable(new CompoundName("hits"), null)); assertFalse(defaultProfile.isOverridable(new CompoundName("user.trusted"), null)); - assertEquals("false",defaultProfile.get("user.trusted")); + assertEquals("false", defaultProfile.get("user.trusted")); - CompiledQueryProfile referencingProfile=cRegistry.getComponent("referencingModelSettings"); + CompiledQueryProfile referencingProfile = cRegistry.getComponent("referencingModelSettings"); assertNull(referencingProfile.getType()); - assertEquals("some query",referencingProfile.get("model.queryString")); - assertEquals("aDefaultIndex",referencingProfile.get("model.defaultIndex")); + assertEquals("some query", referencingProfile.get("model.queryString")); + assertEquals("aDefaultIndex", referencingProfile.get("model.defaultIndex")); // Request parameters here should be ignored - HttpRequest request=HttpRequest.createTestRequest("?query=foo&user.trusted=true&default-index=title", Method.GET); - Query query=new Query(request, defaultProfile); - assertEquals("false",query.properties().get("user.trusted")); - assertEquals("default",query.getModel().getDefaultIndex()); - assertEquals("default",query.properties().get("default-index")); - - CompiledQueryProfile rootProfile=cRegistry.getComponent("root"); - assertEquals("rootType",rootProfile.getType().getId().getName()); - assertEquals(30,rootProfile.get("hits")); - assertEquals(3,rootProfile.get("traceLevel")); + HttpRequest request = HttpRequest.createTestRequest("?query=foo&user.trusted=true&default-index=title", Method.GET); + Query query = new Query(request, defaultProfile); + assertEquals("false", query.properties().get("user.trusted")); + assertEquals("default", query.getModel().getDefaultIndex()); + assertEquals("default", query.properties().get("default-index")); + + CompiledQueryProfile rootProfile = cRegistry.getComponent("root"); + assertEquals("rootType", rootProfile.getType().getId().getName()); + assertEquals(30, rootProfile.get("hits")); + //assertEquals(3, rootProfile.get("traceLevel")); assertTrue(rootProfile.isOverridable(new CompoundName("hits"), null)); + query = new Query(request, rootProfile); + assertEquals(3, query.getTrace().getLevel()); - QueryProfile someUser=registry.getComponent("someUser"); + QueryProfile someUser = registry.getComponent("someUser"); assertEquals("5",someUser.get("sub.test")); assertEquals(18,someUser.get("age")); // aliases - assertEquals(18,someUser.get("alder")); - assertEquals(18,someUser.get("anno")); - assertEquals(18,someUser.get("aLdER")); - assertEquals(18,someUser.get("ANNO")); + assertEquals(18, someUser.get("alder")); + assertEquals(18, someUser.get("anno")); + assertEquals(18, someUser.get("aLdER")); + assertEquals(18, someUser.get("ANNO")); assertNull(someUser.get("Age")); // Only aliases are case insensitive Map<String, String> context = new HashMap<>(); context.put("x", "x1"); assertEquals(37, someUser.get("alder", context, null)); - assertEquals(37,someUser.get("anno", context, null)); - assertEquals(37,someUser.get("aLdER", context, null)); - assertEquals(37,someUser.get("ANNO", context, null)); - assertEquals("male",someUser.get("gender", context, null)); - assertEquals("male",someUser.get("sex", context, null)); - assertEquals("male",someUser.get("Sex", context, null)); + assertEquals(37, someUser.get("anno", context, null)); + assertEquals(37, someUser.get("aLdER", context, null)); + assertEquals(37, someUser.get("ANNO", context, null)); + assertEquals("male", someUser.get("gender", context, null)); + assertEquals("male", someUser.get("sex", context, null)); + assertEquals("male", someUser.get("Sex", context, null)); assertNull(someUser.get("Gender", context, null)); // Only aliases are case insensitive } diff --git a/container-search/src/test/java/com/yahoo/search/query/profile/config/test/validxml/root.xml b/container-search/src/test/java/com/yahoo/search/query/profile/config/test/validxml/root.xml index 82ff7afb186..e7a5f132aaf 100644 --- a/container-search/src/test/java/com/yahoo/search/query/profile/config/test/validxml/root.xml +++ b/container-search/src/test/java/com/yahoo/search/query/profile/config/test/validxml/root.xml @@ -4,6 +4,6 @@ <query-profile id="root" type="rootType"> <field name="hits">30</field> - <field name="traceLevel">3</field> + <field name="trace.level">3</field> </query-profile> diff --git a/container-search/src/test/java/com/yahoo/search/query/profile/test/QueryProfileVariantsTestCase.java b/container-search/src/test/java/com/yahoo/search/query/profile/test/QueryProfileVariantsTestCase.java index 3542e1413eb..3cf2949f33c 100644 --- a/container-search/src/test/java/com/yahoo/search/query/profile/test/QueryProfileVariantsTestCase.java +++ b/container-search/src/test/java/com/yahoo/search/query/profile/test/QueryProfileVariantsTestCase.java @@ -1359,33 +1359,90 @@ public class QueryProfileVariantsTestCase { CompiledQueryProfileRegistry cRegistry = registry.compile(); CompiledQueryProfile cTest = cRegistry.findQueryProfile("test"); - assertValueForZone("default", ZoneInfo.defaultInfo(), cTest); + assertValueForZone("default", ZoneInfo.defaultInfo(), null, cTest); assertValueForZone("prod-region1-instance1", new ZoneInfo(new ApplicationId("tenant1", "application1", "instance1"), new Zone(Environment.prod, "region1")), + null, cTest); assertValueForZone("prod-instance2", new ZoneInfo(new ApplicationId("tenant2", "application2", "instance2"), new Zone(Environment.prod, "region1")), + null, cTest); assertValueForZone("prod-region3", new ZoneInfo(new ApplicationId("tenant3", "application3", "instance3"), new Zone(Environment.prod, "region3")), + null, cTest); assertValueForZone("dev", new ZoneInfo(new ApplicationId("tenant4", "application4", "instance4"), new Zone(Environment.dev, "region4")), + null, cTest); } - private void assertValueForZone(String expected, ZoneInfo zoneInfo, CompiledQueryProfile cTest) { - assertEquals(expected, - new Query.Builder().setQueryProfile(cTest).setZoneInfo(zoneInfo).build().properties().get("value")); + @Test + public void testZoneInfoInContextWithUnoverridability() { + QueryProfileRegistry registry = new QueryProfileRegistry(); + QueryProfile profile = new QueryProfile("test"); + profile.setDimensions(new String[] { "instance", "environment", "region" }); + profile.set("value", "default", registry); + profile.set("value", "prod-beta", + toMap("environment=prod", "instance=beta"), + registry); + profile.setOverridable("value", false, toMap("environment=prod", "instance=beta")); + registry.register(profile); + + CompiledQueryProfileRegistry cRegistry = registry.compile(); + CompiledQueryProfile cTest = cRegistry.findQueryProfile("test"); + + assertValueForZone("prod-beta", + new ZoneInfo(new ApplicationId("tenant1", "application1", "beta"), + new Zone(Environment.prod, "region1")), + "fromRequest", + cTest); + } + + + private void assertValueForZone(String expected, ZoneInfo zoneInfo, String requestValue, CompiledQueryProfile cTest) { + var builder = new Query.Builder().setQueryProfile(cTest).setZoneInfo(zoneInfo); + if (requestValue != null) + builder.setRequestMap(Map.of("value", requestValue)); + assertEquals(expected, builder.build().properties().get("value")); + } + + @Test + public void testZoneInfoInContextSettingNativeProperty() { + QueryProfileRegistry registry = new QueryProfileRegistry(); + QueryProfile profile = new QueryProfile("test"); + profile.setDimensions(new String[] { "instance", "environment", "region" }); + profile.set("timeout", "0.3", + toMap("environment=prod", "instance=beta"), + registry); + registry.register(profile); + + CompiledQueryProfileRegistry cRegistry = registry.compile(); + CompiledQueryProfile cTest = cRegistry.findQueryProfile("test"); + + assertTimeoutForZone(300, + new ZoneInfo(new ApplicationId("tenant1", "application1", "beta"), + new Zone(Environment.prod, "region1")), + null, + cTest); + } + + private void assertTimeoutForZone(int expected, ZoneInfo zoneInfo, String requestValue, CompiledQueryProfile cTest) { + var builder = new Query.Builder().setQueryProfile(cTest).setZoneInfo(zoneInfo); + if (requestValue != null) + builder.setRequestMap(Map.of("timeout", requestValue)); + assertEquals(expected, builder.build().getTimeout()); } private void assertGet(String expectedValue, String parameter, String[] dimensionValues, QueryProfile profile, CompiledQueryProfile cprofile) { - Map<String,String> context=toMap(profile,dimensionValues); - assertEquals("Looking up '" + parameter + "' for '" + Arrays.toString(dimensionValues) + "'",expectedValue,cprofile.get(parameter,context)); + Map<String, String> context = toMap(profile,dimensionValues); + assertEquals("Looking up '" + parameter + "' for '" + Arrays.toString(dimensionValues) + "'", + expectedValue, cprofile.get(parameter,context)); } public static Map<String,String> toMap(QueryProfile profile, String[] dimensionValues) { diff --git a/container-search/src/test/java/com/yahoo/search/searchchain/test/TraceTestCase.java b/container-search/src/test/java/com/yahoo/search/searchchain/test/TraceTestCase.java index 90da5a5b746..04b1fd85872 100644 --- a/container-search/src/test/java/com/yahoo/search/searchchain/test/TraceTestCase.java +++ b/container-search/src/test/java/com/yahoo/search/searchchain/test/TraceTestCase.java @@ -48,11 +48,33 @@ public class TraceTestCase { public void testTracingOnIncorrectAPIUseParallel() { assertTracing(false,true); } - + + @Test + public void testTraceWithQuery() { + testQueryInTrace(true, "trace.query=true"); + testQueryInTrace(false, "trace.query=false"); + testQueryInTrace(true, ""); + } + + private void testQueryInTrace(boolean expectQueryInTrace, String queryParameters) { + Query query = new Query("?query=foo&trace.level=1&" + queryParameters); + Chain<Searcher> chain = new Chain<>(new Tracer("tracer1", true)); + Execution execution = new Execution(chain, Execution.Context.createContextStub()); + Result result = execution.search(query); + Iterator<String> trace = collectTrace(query).iterator(); + assertEquals("(level start)", trace.next()); + assertEquals(" No query profile is used", trace.next()); + assertEquals(" (level start)", trace.next()); + if (expectQueryInTrace) + assertEquals(" During tracer1: 0: [WEAKAND(100) foo]", trace.next()); + else + assertEquals(" During tracer1: 0", trace.next()); + } + @Test public void testTraceInvocationsUnfillableHits() { final int traceLevel = 5; - Query query = new Query("?tracelevel=" + traceLevel); + Query query = new Query("?trace.level=" + traceLevel); Chain<Searcher> forkingChain = new Chain<>(new Tracer("tracer1"), new Tracer("tracer2"), new Backend("backend1", false)); @@ -118,7 +140,8 @@ public class TraceTestCase { private void assertTracing(boolean carryOverContext, boolean parallel) { Query query = new Query("?tracelevel=1"); - query.trace("Before execution",1); + assertEquals(1, query.getTrace().getLevel()); + query.trace("Before execution", 1); Chain<Searcher> forkingChain = new Chain<>(new Tracer("forker"), new Forker(carryOverContext, parallel, new Tracer("branch 1") , @@ -222,8 +245,8 @@ public class TraceTestCase { private static class TraceCollector extends TraceVisitor { - private List<String> trace = new ArrayList<>(); - private StringBuilder indent = new StringBuilder(); + private final List<String> trace = new ArrayList<>(); + private final StringBuilder indent = new StringBuilder(); @Override public void entering(TraceNode node) { @@ -249,30 +272,38 @@ public class TraceTestCase { private static class Tracer extends Searcher { - private String name; + private final String name; + private final boolean traceQuery; + private int counter = 0; public Tracer(String name) { + this(name, false); + } + + public Tracer(String name, boolean traceQuery) { super(new ComponentId(name)); this.name = name; + this.traceQuery = traceQuery; } @Override public Result search(Query query, Execution execution) { - query.trace("During " + name + ": " + (counter++),1); + query.trace("During " + name + ": " + (counter++), traceQuery, 1); return execution.search(query); } + } private static class Forker extends Searcher { - private List<Searcher> branches; + private final List<Searcher> branches; /** If true, this is using the api as recommended, if false, it is not */ - private boolean carryOverContext; + private final boolean carryOverContext; /** If true, simulate parallel execution by cloning the query */ - private boolean parallel; + private final boolean parallel; public Forker(boolean carryOverContext, boolean parallel, Searcher ... branches) { this.carryOverContext = carryOverContext; @@ -280,7 +311,6 @@ public class TraceTestCase { this.branches = Arrays.asList(branches); } - @SuppressWarnings("deprecation") @Override public Result search(Query query, Execution execution) { Result result = execution.search(query); @@ -319,6 +349,7 @@ public class TraceTestCase { result.hits().add(hit1); return result; } + } } diff --git a/container-search/src/test/java/com/yahoo/search/test/QueryTestCase.java b/container-search/src/test/java/com/yahoo/search/test/QueryTestCase.java index 5c5d9006d1a..7cd7e20b76f 100644 --- a/container-search/src/test/java/com/yahoo/search/test/QueryTestCase.java +++ b/container-search/src/test/java/com/yahoo/search/test/QueryTestCase.java @@ -552,7 +552,7 @@ public class QueryTestCase { @Test public void testTracing() { Query q = new Query("?query=foo&type=all&traceLevel=2"); - assertEquals(2, q.getTraceLevel()); + assertEquals(2, q.getTrace().getLevel()); q.trace(true, 1, "trace1"); q.trace(false,2, "trace2"); q.trace(true, 3, "Ignored"); @@ -571,7 +571,7 @@ public class QueryTestCase { @Test public void testNullTracing() { Query q = new Query("?query=foo&traceLevel=2"); - assertEquals(2, q.getTraceLevel()); + assertEquals(2, q.getTrace().getLevel()); q.trace(false,2, "trace2 ", null); Set<String> traces = new HashSet<>(); for (String trace : q.getContext(true).getTrace().traceNode().descendants(String.class)) @@ -582,8 +582,8 @@ public class QueryTestCase { @Test public void testExplain() { Query q = new Query("?query=foo&explainLevel=2"); - assertEquals(2, q.getExplainLevel()); - assertEquals(0, q.getTraceLevel()); + assertEquals(2, q.getTrace().getExplainLevel()); + assertEquals(0, q.getTrace().getLevel()); } @Test diff --git a/container-search/src/test/java/com/yahoo/vespa/streamingvisitors/VdsStreamingSearcherTestCase.java b/container-search/src/test/java/com/yahoo/vespa/streamingvisitors/VdsStreamingSearcherTestCase.java index 861be9a3ade..835235a593c 100644 --- a/container-search/src/test/java/com/yahoo/vespa/streamingvisitors/VdsStreamingSearcherTestCase.java +++ b/container-search/src/test/java/com/yahoo/vespa/streamingvisitors/VdsStreamingSearcherTestCase.java @@ -170,10 +170,10 @@ public class VdsStreamingSearcherTestCase { } else if (i == 1) { query.getPresentation().setSummary("summary"); } else if (i == 2) { - query.setTraceLevel(100); + query.getTrace().setLevel(100); } else if (i == 3) { query.getPresentation().setSummary("summary"); - query.setTraceLevel(100); + query.getTrace().setLevel(100); } queries[i] = query; } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTrigger.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTrigger.java index 6e2ae0da46d..9df305f7864 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTrigger.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTrigger.java @@ -158,7 +158,7 @@ public class DeploymentTrigger { * * Only one job per type is triggered each run for test jobs, since their environments have limited capacity. */ - public long triggerReadyJobs() { + public TriggerResult triggerReadyJobs() { List<Job> readyJobs = computeReadyJobs(); var prodJobs = new ArrayList<Job>(); @@ -182,23 +182,34 @@ public class DeploymentTrigger { .collect(groupingBy(Job::jobType)); // Trigger all prod jobs - sortedProdJobs.forEach(this::trigger); - long triggeredJobs = sortedProdJobs.size(); + long triggeredJobs = 0; + long failedJobs = 0; + for (Job job : sortedProdJobs) { + if (trigger(job)) ++triggeredJobs; + else ++failedJobs; + } // Trigger max one test job per type - for (var jobs : sortedTestJobsByType.values()) { - if (jobs.size() > 0) { - trigger(jobs.get(0)); - triggeredJobs++; - } - } - return triggeredJobs; + for (Collection<Job> jobs: sortedTestJobsByType.values()) + for (Job job : jobs) + if (trigger(job)) { ++triggeredJobs; break; } + else ++failedJobs; + + return new TriggerResult(triggeredJobs, failedJobs); } + public record TriggerResult(long triggered, long failed) { } /** Attempts to trigger the given job. */ - private void trigger(Job job) { - trigger(job, null); + private boolean trigger(Job job) { + try { + trigger(job, null); + return true; + } + catch (Exception e) { + log.log(Level.WARNING, "Failed triggering " + job.jobType() + " for " + job.instanceId, e); + return false; + } } /** Attempts to trigger the given job. */ diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java index 3d4a2f40303..c83527372b2 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java @@ -584,8 +584,8 @@ public class JobController { if (revision.compileVersion() .map(version -> controller.applications().versionCompatibility(id).refuse(versions.targetPlatform(), version)) .orElse(false)) - throw new IllegalArgumentException("Will not start a job with incompatible platform version (" + versions.targetPlatform() + ") " + - "and compile versions (" + revision.compileVersion().get() + ")"); + throw new IllegalArgumentException("Will not start " + type + " for " + id + " with incompatible platform version (" + + versions.targetPlatform() + ") " + "and compile versions (" + revision.compileVersion().get() + ")"); locked(id, type, __ -> { Optional<Run> last = last(id, type); diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ReadyJobsTrigger.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ReadyJobsTrigger.java index 26df8669fb1..5178918aa48 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ReadyJobsTrigger.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ReadyJobsTrigger.java @@ -2,6 +2,7 @@ package com.yahoo.vespa.hosted.controller.maintenance; import com.yahoo.vespa.hosted.controller.Controller; +import com.yahoo.vespa.hosted.controller.deployment.DeploymentTrigger.TriggerResult; import java.time.Duration; @@ -18,8 +19,8 @@ public class ReadyJobsTrigger extends ControllerMaintainer { @Override public double maintain() { - controller().applications().deploymentTrigger().triggerReadyJobs(); - return 1.0; + TriggerResult result = controller().applications().deploymentTrigger().triggerReadyJobs(); + return result.triggered() * 1.0f / (result.triggered() + result.failed()); } } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java index 53557bafcb0..8ce55fa5f1b 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java @@ -765,7 +765,7 @@ public class ControllerTest { fail("Should fail when specifying a major which is incompatible with compile version"); } catch (IllegalArgumentException e) { - assertEquals("Will not start a job with incompatible platform version (8) and compile versions (7)", e.getMessage()); + assertEquals("Will not start dev-us-east-1 for tenant.application with incompatible platform version (8) and compile versions (7)", e.getMessage()); } context.runJob(zone, new ApplicationPackageBuilder().compileVersion(version3).majorVersion(8).build()); diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTester.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTester.java index 78e7606d7c6..2b4a2baa17e 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTester.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTester.java @@ -146,7 +146,7 @@ public class DeploymentTester { int triggered; int triggeredTotal = 0; do { - triggered = (int) deploymentTrigger().triggerReadyJobs(); + triggered = (int) deploymentTrigger().triggerReadyJobs().triggered(); triggeredTotal += triggered; } while (triggered > 0); return triggeredTotal; diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java index d9f0f010104..c5d2dad20bc 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java @@ -842,7 +842,7 @@ public class ApplicationApiTest extends ControllerContainerTest { "{\"message\":\"Marked build '2' as non-deployable\"}"); // GET deployment job overview, after triggering system and staging test jobs. - assertEquals(2, tester.controller().applications().deploymentTrigger().triggerReadyJobs()); + assertEquals(2, tester.controller().applications().deploymentTrigger().triggerReadyJobs().triggered()); tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/instance/instance1/job", GET) .userIdentity(USER_ID), new File("jobs.json")); diff --git a/document/src/main/java/com/yahoo/document/ReferenceDataType.java b/document/src/main/java/com/yahoo/document/ReferenceDataType.java index 8cf6b665acf..ee1afa086d7 100644 --- a/document/src/main/java/com/yahoo/document/ReferenceDataType.java +++ b/document/src/main/java/com/yahoo/document/ReferenceDataType.java @@ -16,7 +16,7 @@ public class ReferenceDataType extends DataType { // Magic number for Identifiable, see document/util/identifiable.h public static final int classId = registerClass(Ids.document + 68, ReferenceDataType.class); - private StructuredDataType targetType; + private final StructuredDataType targetType; public ReferenceDataType(DocumentType targetType, int id) { this((StructuredDataType)targetType, id); diff --git a/document/src/main/java/com/yahoo/document/datatypes/ReferenceFieldValue.java b/document/src/main/java/com/yahoo/document/datatypes/ReferenceFieldValue.java index b76c3f2989f..ebc39b14eb1 100644 --- a/document/src/main/java/com/yahoo/document/datatypes/ReferenceFieldValue.java +++ b/document/src/main/java/com/yahoo/document/datatypes/ReferenceFieldValue.java @@ -49,6 +49,7 @@ public class ReferenceFieldValue extends FieldValue { /** * Creates a reference pointing to a particular document instance in the document * type referenced by <code>referenceType</code>. + * * @param referenceType reference target type * @param documentId document ID of the same document type as that given by <code>referenceType</code> * @throws IllegalArgumentException if documentId is not of the expected document type @@ -64,7 +65,7 @@ public class ReferenceFieldValue extends FieldValue { } private static void requireIdOfMatchingType(ReferenceDataType referenceType, DocumentId id) { - final String expectedTypeName = referenceType.getTargetType().getName(); + String expectedTypeName = referenceType.getTargetType().getName(); if (!id.getDocType().equals(expectedTypeName)) { throw new IllegalArgumentException(String.format( "Can't assign document ID '%s' (of type '%s') to reference of document type '%s'", @@ -87,9 +88,7 @@ public class ReferenceFieldValue extends FieldValue { } @Override - public void printXml(XmlStream xml) { - // TODO do we need/want this? - } + public void printXml(XmlStream xml) { } @Override public void clear() { diff --git a/flags/src/main/java/com/yahoo/vespa/flags/Flags.java b/flags/src/main/java/com/yahoo/vespa/flags/Flags.java index 46f224af1c7..52fcd226a28 100644 --- a/flags/src/main/java/com/yahoo/vespa/flags/Flags.java +++ b/flags/src/main/java/com/yahoo/vespa/flags/Flags.java @@ -376,6 +376,13 @@ public class Flags { "Takes effect on host admin restart", HOSTNAME); + public static final UnboundStringFlag LOG_FILE_COMPRESSION_ALGORITHM = defineStringFlag( + "log-file-compression-algorithm", "", + List.of("arnej"), "2022-06-14", "2024-12-31", + "Which algorithm to use for compressing log files. Valid values: empty string (default), gzip, zstd", + "Takes effect immediately", + ZONE_ID, APPLICATION_ID); + /** WARNING: public for testing: All flags should be defined in {@link Flags}. */ public static UnboundBooleanFlag defineFeatureFlag(String flagId, boolean defaultValue, List<String> owners, String createdAt, String expiresAt, String description, diff --git a/logserver/src/main/java/com/yahoo/logserver/handlers/archive/ArchiverHandler.java b/logserver/src/main/java/com/yahoo/logserver/handlers/archive/ArchiverHandler.java index 0b44e47f183..50df160d01f 100644 --- a/logserver/src/main/java/com/yahoo/logserver/handlers/archive/ArchiverHandler.java +++ b/logserver/src/main/java/com/yahoo/logserver/handlers/archive/ArchiverHandler.java @@ -86,9 +86,9 @@ public class ArchiverHandler extends AbstractLogHandler { * Creates an ArchiverHandler which puts files under * the given root directory. */ - public ArchiverHandler(String rootDir, int maxFileSize) { + public ArchiverHandler(String rootDir, int maxFileSize, String zip) { this(); - setRootDir(rootDir); + setRootDir(rootDir, zip); this.maxFileSize = maxFileSize; } @@ -189,7 +189,7 @@ public class ArchiverHandler extends AbstractLogHandler { } } - private void setRootDir(String rootDir) { + private void setRootDir(String rootDir, String zip) { // roundabout way of setting things, but this way we can // get around Java's ineptitude for file handling (relative paths in File are broken) absoluteRootDir = new File(rootDir).getAbsolutePath(); @@ -205,7 +205,7 @@ public class ArchiverHandler extends AbstractLogHandler { log.log(Level.FINE, () -> "Created root at " + absoluteRootDir); } } - filesArchived = new FilesArchived(root); + filesArchived = new FilesArchived(root, zip); } public String toString() { diff --git a/logserver/src/main/java/com/yahoo/logserver/handlers/archive/ArchiverPlugin.java b/logserver/src/main/java/com/yahoo/logserver/handlers/archive/ArchiverPlugin.java index deb3b1adcf4..dc6d70252c6 100644 --- a/logserver/src/main/java/com/yahoo/logserver/handlers/archive/ArchiverPlugin.java +++ b/logserver/src/main/java/com/yahoo/logserver/handlers/archive/ArchiverPlugin.java @@ -20,6 +20,8 @@ public class ArchiverPlugin implements Plugin { */ private static final String DEFAULT_MAXFILESIZE = "20971520"; + private static final String DEFAULT_COMPRESSION = "gzip"; + private final Server server = Server.getInstance(); private static final Logger log = Logger.getLogger(ArchiverPlugin.class.getName()); private ArchiverHandler archiver; @@ -52,9 +54,10 @@ public class ArchiverPlugin implements Plugin { String rootDir = config.get("dir", DEFAULT_DIR); int maxFileSize = config.getInt("maxfilesize", DEFAULT_MAXFILESIZE); String threadName = config.get("thread", getPluginName()); + String zip = config.get("compression", DEFAULT_COMPRESSION); // register log handler and flusher - archiver = new ArchiverHandler(rootDir, maxFileSize); + archiver = new ArchiverHandler(rootDir, maxFileSize, zip); server.registerLogHandler(archiver, threadName); server.registerFlusher(archiver); } diff --git a/logserver/src/main/java/com/yahoo/logserver/handlers/archive/FilesArchived.java b/logserver/src/main/java/com/yahoo/logserver/handlers/archive/FilesArchived.java index 54e47e15d8e..d1e9793ffaf 100644 --- a/logserver/src/main/java/com/yahoo/logserver/handlers/archive/FilesArchived.java +++ b/logserver/src/main/java/com/yahoo/logserver/handlers/archive/FilesArchived.java @@ -8,10 +8,26 @@ import java.io.FileOutputStream; import java.io.IOException; import java.util.ArrayList; import java.util.List; +import java.util.logging.Level; import java.util.logging.Logger; import java.util.regex.Pattern; import java.util.zip.GZIPOutputStream; +import com.yahoo.compress.ZstdOutputStream; +import com.yahoo.io.NativeIO; +import com.yahoo.log.LogFileDb; +import com.yahoo.protect.Process; +import com.yahoo.yolean.Exceptions; + +import java.io.BufferedOutputStream; +import java.io.FileDescriptor; +import java.io.FileNotFoundException; +import java.io.OutputStream; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.nio.file.StandardCopyOption; + /** * This class holds information about all (log) files contained @@ -28,6 +44,10 @@ public class FilesArchived { */ private final File root; + enum Compression {NONE, GZIP, ZSTD} + private final Compression compression; + private final NativeIO nativeIO = new NativeIO(); + private final Object mutex = new Object(); // known-existing files inside the archive directory @@ -60,8 +80,9 @@ public class FilesArchived { /** * Creates an instance of FilesArchive managing the given directory */ - public FilesArchived(File rootDir) { + public FilesArchived(File rootDir, String zip) { this.root = rootDir; + this.compression = ("zstd".equals(zip)) ? Compression.ZSTD : Compression.GZIP; rescan(); Thread thread = new Thread(this::run); thread.setDaemon(true); @@ -152,7 +173,24 @@ public class FilesArchived { return count > 0; } + private void compress(File oldFile) { + switch (compression) { + case ZSTD: + runCompressionZstd(nativeIO, oldFile); + break; + case GZIP: + compressGzip(oldFile); + break; + case NONE: + runCompressionNone(nativeIO, oldFile); + break; + default: + throw new IllegalArgumentException("Unknown compression " + compression); + } + } + + private void compressGzip(File oldFile) { File gzippedFile = new File(oldFile.getPath() + ".gz"); try (GZIPOutputStream compressor = new GZIPOutputStream(new FileOutputStream(gzippedFile), 0x100000); FileInputStream inputStream = new FileInputStream(oldFile)) @@ -173,6 +211,32 @@ public class FilesArchived { } } + private static void runCompressionZstd(NativeIO nativeIO, File oldFile) { + try { + Path compressedFile = Paths.get(oldFile.toString() + ".zst"); + int bufferSize = 2*1024*1024; + long mtime = oldFile.lastModified(); + try (FileOutputStream fileOut = AtomicFileOutputStream.create(compressedFile); + ZstdOutputStream out = new ZstdOutputStream(fileOut, bufferSize); + FileInputStream in = new FileInputStream(oldFile)) + { + pageFriendlyTransfer(nativeIO, out, fileOut.getFD(), in, bufferSize); + out.flush(); + } + compressedFile.toFile().setLastModified(mtime); + oldFile.delete(); + nativeIO.dropFileFromCache(compressedFile.toFile()); + } catch (IOException e) { + log.log(Level.WARNING, "Failed to compress log file with zstd: " + oldFile, e); + } finally { + nativeIO.dropFileFromCache(oldFile); + } + } + + private static void runCompressionNone(NativeIO nativeIO, File oldFile) { + nativeIO.dropFileFromCache(oldFile); + } + long sumFileSizes() { long sum = 0; for (LogFile lf : knownFiles) { @@ -210,7 +274,7 @@ public class FilesArchived { } static class LogFile { - public final File path; + public final File path; public final String prefix; public final int generation; public final boolean zsuff; @@ -245,6 +309,7 @@ public class FilesArchived { } private static boolean zSuffix(String name) { if (name.endsWith(".gz")) return true; + if (name.endsWith(".zst")) return true; // add other compression suffixes here return false; } @@ -259,4 +324,44 @@ public class FilesArchived { return "FilesArchived.LogFile{name="+path+" prefix="+prefix+" gen="+generation+" z="+zsuff+"}"; } } + + private static class AtomicFileOutputStream extends FileOutputStream { + private final Path path; + private final Path tmpPath; + private volatile boolean closed = false; + + private AtomicFileOutputStream(Path path, Path tmpPath) throws FileNotFoundException { + super(tmpPath.toFile()); + this.path = path; + this.tmpPath = tmpPath; + } + + @Override + public synchronized void close() throws IOException { + super.close(); + if (!closed) { + Files.move(tmpPath, path, StandardCopyOption.ATOMIC_MOVE); + closed = true; + } + } + + private static AtomicFileOutputStream create(Path path) throws FileNotFoundException { + return new AtomicFileOutputStream(path, path.resolveSibling("." + path.getFileName() + ".tmp")); + } + } + + private static void pageFriendlyTransfer(NativeIO nativeIO, OutputStream out, FileDescriptor outDescriptor, FileInputStream in, int bufferSize) throws IOException { + int read; + long totalBytesRead = 0; + byte[] buffer = new byte[bufferSize]; + while ((read = in.read(buffer)) >= 0) { + out.write(buffer, 0, read); + if (read > 0) { + nativeIO.dropPartialFileFromCache(in.getFD(), totalBytesRead, read, false); + nativeIO.dropPartialFileFromCache(outDescriptor, totalBytesRead, read, false); + } + totalBytesRead += read; + } + } + } diff --git a/logserver/src/test/java/com/yahoo/logserver/handlers/archive/ArchiverHandlerTestCase.java b/logserver/src/test/java/com/yahoo/logserver/handlers/archive/ArchiverHandlerTestCase.java index f3153f7bc14..bffbde67fa4 100644 --- a/logserver/src/test/java/com/yahoo/logserver/handlers/archive/ArchiverHandlerTestCase.java +++ b/logserver/src/test/java/com/yahoo/logserver/handlers/archive/ArchiverHandlerTestCase.java @@ -64,7 +64,7 @@ public class ArchiverHandlerTestCase { File tmpDir = temporaryFolder.newFolder(); ArchiverHandler a = new ArchiverHandler(tmpDir.getAbsolutePath(), - 1024); + 1024, "gzip"); long now = 1095159244095L; long midnight = 1095206400000L; assertEquals(2004091410, a.dateHash(now)); @@ -82,7 +82,7 @@ public class ArchiverHandlerTestCase { File tmpDir = temporaryFolder.newFolder(); try { ArchiverHandler a = new ArchiverHandler(tmpDir.getAbsolutePath(), - 1024); + 1024, "gzip"); LogMessage msg1 = LogMessage.parseNativeFormat("1139322725\thost\t1/1\tservice\tcomponent\tinfo\tpayload"); LogMessage msg2 = LogMessage.parseNativeFormat("1161172200\thost\t1/1\tservice\tcomponent\tinfo\tpayload"); assertEquals(tmpDir.getAbsolutePath() + "/2006/02/07/14", a.getPrefix(msg1)); @@ -103,7 +103,7 @@ public class ArchiverHandlerTestCase { File tmpDir = temporaryFolder.newFolder(); ArchiverHandler a = new ArchiverHandler(tmpDir.getAbsolutePath(), - 1024); + 1024, "gzip"); for (int i = 0; i < msg.length; i++) { a.handle(msg[i]); @@ -168,7 +168,8 @@ public class ArchiverHandlerTestCase { File tmpDir = temporaryFolder.newFolder(); ArchiverHandler a = new ArchiverHandler(tmpDir.getAbsolutePath(), - msg[1].toString().length() + 1); + msg[1].toString().length() + 1, + "gzip"); // log the same message 4 times for (int i = 0; i < 4; i++) { a.handle(msg[1]); @@ -205,7 +206,7 @@ public class ArchiverHandlerTestCase { public void testCacheEldestEntry() throws IOException { LogWriterLRUCache cache = new LogWriterLRUCache(5, (float) 0.75); String d = "target/tmp/logarchive"; - FilesArchived archive = new FilesArchived(new File(d)); + FilesArchived archive = new FilesArchived(new File(d), "gzip"); for (int i = 0; i < cache.maxEntries + 10; i++) { cache.put(i, new LogWriter(d+"/2018/12/31/17", 5, archive)); } diff --git a/logserver/src/test/java/com/yahoo/logserver/handlers/archive/FilesArchivedTestCase.java b/logserver/src/test/java/com/yahoo/logserver/handlers/archive/FilesArchivedTestCase.java index 6004df88cfe..babe4b1479d 100644 --- a/logserver/src/test/java/com/yahoo/logserver/handlers/archive/FilesArchivedTestCase.java +++ b/logserver/src/test/java/com/yahoo/logserver/handlers/archive/FilesArchivedTestCase.java @@ -55,7 +55,7 @@ public class FilesArchivedTestCase { makeLogfile("2018/12/31/16-0", 1); makeLogfile("2018/12/31/17-0", 0); dumpFiles("before archive maintenance"); - FilesArchived a = new FilesArchived(tmpDir); + FilesArchived a = new FilesArchived(tmpDir, "zstd"); dumpFiles("also before archive maintenance"); checkExist("foo/bar"); @@ -67,14 +67,14 @@ public class FilesArchivedTestCase { checkExist("2018/12/31/14-0"); checkExist("2018/12/31/16-0"); checkExist("2018/12/31/17-0"); - checkNoExist("2018/11/20/13-0.gz"); - checkNoExist("2018/11/21/13-0.gz"); - checkNoExist("2018/12/28/13-0.gz"); - checkNoExist("2018/12/29/13-0.gz"); - checkNoExist("2018/12/30/13-0.gz"); - checkNoExist("2018/12/31/14-0.gz"); - checkNoExist("2018/12/31/16-0.gz"); - checkNoExist("2018/12/31/17-0.gz"); + checkNoExist("2018/11/20/13-0.zst"); + checkNoExist("2018/11/21/13-0.zst"); + checkNoExist("2018/12/28/13-0.zst"); + checkNoExist("2018/12/29/13-0.zst"); + checkNoExist("2018/12/30/13-0.zst"); + checkNoExist("2018/12/31/14-0.zst"); + checkNoExist("2018/12/31/16-0.zst"); + checkNoExist("2018/12/31/17-0.zst"); a.maintenance(); @@ -82,22 +82,22 @@ public class FilesArchivedTestCase { checkExist("foo/bar"); checkExist("2018/12/31/17-0"); checkExist("2018/12/31/16-0"); - checkExist("2018/12/31/14-0.gz"); - checkExist("2018/12/28/13-0.gz"); - checkExist("2018/12/29/13-0.gz"); - checkExist("2018/12/30/13-0.gz"); + checkExist("2018/12/31/14-0.zst"); + checkExist("2018/12/28/13-0.zst"); + checkExist("2018/12/29/13-0.zst"); + checkExist("2018/12/30/13-0.zst"); - checkNoExist("2018/12/31/17-0.gz"); - checkNoExist("2018/12/31/16-0.gz"); + checkNoExist("2018/12/31/17-0.zst"); + checkNoExist("2018/12/31/16-0.zst"); checkNoExist("2018/12/31/14-0"); checkNoExist("2018/12/28/13-0"); checkNoExist("2018/12/29/13-0"); checkNoExist("2018/12/30/13-0"); checkNoExist("2018/11/20/13-0"); - checkNoExist("2018/11/20/13-0.gz"); + checkNoExist("2018/11/20/13-0.zst"); checkNoExist("2018/11/21/13-0"); - checkNoExist("2018/11/21/13-0.gz"); + checkNoExist("2018/11/21/13-0.zst"); makeLogfile("2018/12/31/16-0", 3); makeLogfile("2018/12/31/17-0", 3); @@ -110,8 +110,8 @@ public class FilesArchivedTestCase { checkExist("2018/12/31/17-2"); checkExist("2018/12/31/17-1"); - checkExist("2018/12/31/16-0.gz"); - checkExist("2018/12/31/17-0.gz"); + checkExist("2018/12/31/16-0.zst"); + checkExist("2018/12/31/17-0.zst"); checkNoExist("2018/12/31/16-0"); checkNoExist("2018/12/31/17-0"); diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/applications/Cluster.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/applications/Cluster.java index ad20f68ca33..14adc617497 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/applications/Cluster.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/applications/Cluster.java @@ -151,9 +151,7 @@ public class Cluster { } @Override - public String toString() { - return "cluster '" + id + "'"; - } + public String toString() { return id.toString(); } private void prune(List<ScalingEvent> scalingEvents) { while (scalingEvents.size() > maxScalingEvents) diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/Autoscaler.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/Autoscaler.java index cd127a6b17f..9623406767a 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/Autoscaler.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/Autoscaler.java @@ -72,20 +72,20 @@ public class Autoscaler { if (scaledIn(clusterModel.scalingDuration(), cluster)) return Advice.dontScale(Status.waiting, - "Won't autoscale " + cluster + " now: Less than " + clusterModel.scalingDuration() + + "Won't autoscale now: Less than " + clusterModel.scalingDuration() + " since last resource change"); if (clusterModel.nodeTimeseries().measurementsPerNode() < minimumMeasurementsPerNode(clusterModel.scalingDuration())) return Advice.none(Status.waiting, - "Collecting more data before making new scaling decisions for " + cluster + ": Need to measure for " + + "Collecting more data before making new scaling decisions: Need to measure for " + clusterModel.scalingDuration() + " since the last resource change completed, " + - clusterModel.nodeTimeseries().measurementsPerNode() + " measurements per node, " + - " need " + minimumMeasurementsPerNode(clusterModel.scalingDuration())); + clusterModel.nodeTimeseries().measurementsPerNode() + " measurements per node found," + + " need at least " + minimumMeasurementsPerNode(clusterModel.scalingDuration())); if (clusterModel.nodeTimeseries().nodesMeasured() != clusterNodes.size()) return Advice.none(Status.waiting, - "Collecting more data before making new scaling decisions for cluster " + cluster + ": " + - "Have measurements from " + clusterModel.nodeTimeseries().nodesMeasured() + + "Collecting more data before making new scaling decisions:" + + " Have measurements from " + clusterModel.nodeTimeseries().nodesMeasured() + " nodes, but require from " + clusterNodes.size()); var currentAllocation = new AllocatableClusterResources(clusterNodes.asList(), nodeRepository); diff --git a/screwdriver/publish-unpublished-rpms-to-jfrog-cloud.sh b/screwdriver/publish-unpublished-rpms-to-jfrog-cloud.sh index 3bcb4e101a5..ceddc9fe30b 100755 --- a/screwdriver/publish-unpublished-rpms-to-jfrog-cloud.sh +++ b/screwdriver/publish-unpublished-rpms-to-jfrog-cloud.sh @@ -1,13 +1,10 @@ #!/bin/bash # Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - set -euo pipefail readonly MYDIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" -yum install -y yum-utils &> /dev/null - # Copr repo file if [[ ! -f /etc/yum.repos.d/group_vespa-vespa-epel-7.repo ]]; then cat << 'EOF' > /etc/yum.repos.d/group_vespa-vespa-epel-7.repo @@ -36,7 +33,7 @@ fi readonly COPR_PACKAGES=$(mktemp) trap "rm -f $COPR_PACKAGES" EXIT -dnf list -q --disablerepo='*' --enablerepo=copr:copr.fedorainfracloud.org:group_vespa:vespa --showduplicates 'vespa*' | grep "Available Packages" -A 100000 | tail -n +2 | sed "s/\.x86_64\ */-/"| awk '{print $1}' > $COPR_PACKAGES +dnf list -q --disablerepo='*' --enablerepo=copr:copr.fedorainfracloud.org:group_vespa:vespa --showduplicates 'vespa*' | grep "Available Packages" -A 100000 | tail -n +2 | sed '/\.src\ */d' | sed 's/\.x86_64\ */-/' | awk '{print $1}' | grep -v '.src$' > $COPR_PACKAGES echo "Packages on Copr:" cat $COPR_PACKAGES @@ -45,7 +42,7 @@ echo for pv in $(cat $COPR_PACKAGES); do if ! dnf list -q --disablerepo='*' --enablerepo=vespa-release $pv &> /dev/null; then echo "$pv not found on JFrog Clould. Downloading..." - dnf download -q $pv + dnf download -q --disablerepo='*' --enablerepo=copr:copr.fedorainfracloud.org:group_vespa:vespa $pv echo "$pv downloaded." fi done diff --git a/searchlib/src/vespa/searchlib/attribute/attribute_blueprint_factory.cpp b/searchlib/src/vespa/searchlib/attribute/attribute_blueprint_factory.cpp index 3526a921645..4ec6c4f6565 100644 --- a/searchlib/src/vespa/searchlib/attribute/attribute_blueprint_factory.cpp +++ b/searchlib/src/vespa/searchlib/attribute/attribute_blueprint_factory.cpp @@ -556,11 +556,17 @@ public: setEstimate(HitEstimate(_dict_entry.posting_size, (_dict_entry.posting_size == 0))); } - SearchIterator::UP createLeafSearch(const TermFieldMatchDataArray &tfmda, bool) const override { + SearchIterator::UP createLeafSearch(const TermFieldMatchDataArray &tfmda, bool strict) const override { assert(tfmda.size() == 1); if (_dict_entry.posting_size == 0) { return std::make_unique<queryeval::EmptySearch>(); } + if (tfmda[0]->isNotNeeded()) { + auto bitvector_iterator = _attr.make_bitvector_iterator(_dict_entry.posting_idx, get_docid_limit(), *tfmda[0], strict); + if (bitvector_iterator) { + return bitvector_iterator; + } + } return std::make_unique<queryeval::DocumentWeightSearchIterator>(*tfmda[0], _attr, _dict_entry); } diff --git a/searchlib/src/vespa/searchlib/attribute/i_document_weight_attribute.h b/searchlib/src/vespa/searchlib/attribute/i_document_weight_attribute.h index baa65e42c03..be36bcd185a 100644 --- a/searchlib/src/vespa/searchlib/attribute/i_document_weight_attribute.h +++ b/searchlib/src/vespa/searchlib/attribute/i_document_weight_attribute.h @@ -5,6 +5,9 @@ #include "postinglisttraits.h" #include <functional> +namespace search::fef { class TermFieldMatchData; } +namespace search::queryeval { class SearchIterator; } + namespace search { using DocumentWeightIterator = attribute::PostingListTraits<int32_t>::const_iterator; @@ -40,6 +43,7 @@ struct IDocumentWeightAttribute virtual void collect_folded(vespalib::datastore::EntryRef enum_idx, vespalib::datastore::EntryRef dictionary_snapshot, const std::function<void(vespalib::datastore::EntryRef)>& callback) const = 0; virtual void create(vespalib::datastore::EntryRef idx, std::vector<DocumentWeightIterator> &dst) const = 0; virtual DocumentWeightIterator create(vespalib::datastore::EntryRef idx) const = 0; + virtual std::unique_ptr<queryeval::SearchIterator> make_bitvector_iterator(vespalib::datastore::EntryRef idx, uint32_t doc_id_limit, fef::TermFieldMatchData &match_data, bool strict) const = 0; virtual ~IDocumentWeightAttribute() = default; }; diff --git a/searchlib/src/vespa/searchlib/attribute/multinumericpostattribute.h b/searchlib/src/vespa/searchlib/attribute/multinumericpostattribute.h index 3bdab4cfc19..4bd8ad6e99f 100644 --- a/searchlib/src/vespa/searchlib/attribute/multinumericpostattribute.h +++ b/searchlib/src/vespa/searchlib/attribute/multinumericpostattribute.h @@ -40,6 +40,7 @@ private: void collect_folded(vespalib::datastore::EntryRef enum_idx, vespalib::datastore::EntryRef dictionary_snapshot, const std::function<void(vespalib::datastore::EntryRef)>& callback) const override; void create(vespalib::datastore::EntryRef idx, std::vector<DocumentWeightIterator> &dst) const override; DocumentWeightIterator create(vespalib::datastore::EntryRef idx) const override; + std::unique_ptr<queryeval::SearchIterator> make_bitvector_iterator(vespalib::datastore::EntryRef idx, uint32_t doc_id_limit, fef::TermFieldMatchData &match_data, bool strict) const override; }; DocumentWeightAttributeAdapter _document_weight_attribute_adapter; diff --git a/searchlib/src/vespa/searchlib/attribute/multinumericpostattribute.hpp b/searchlib/src/vespa/searchlib/attribute/multinumericpostattribute.hpp index 4854728ca37..9a8c9738bc0 100644 --- a/searchlib/src/vespa/searchlib/attribute/multinumericpostattribute.hpp +++ b/searchlib/src/vespa/searchlib/attribute/multinumericpostattribute.hpp @@ -138,6 +138,13 @@ MultiValueNumericPostingAttribute<B, M>::DocumentWeightAttributeAdapter::create( } template <typename B, typename M> +std::unique_ptr<queryeval::SearchIterator> +MultiValueNumericPostingAttribute<B, M>::DocumentWeightAttributeAdapter::make_bitvector_iterator(vespalib::datastore::EntryRef idx, uint32_t doc_id_limit, fef::TermFieldMatchData &match_data, bool strict) const +{ + return self.getPostingList().make_bitvector_iterator(idx, doc_id_limit, match_data, strict); +} + +template <typename B, typename M> const IDocumentWeightAttribute * MultiValueNumericPostingAttribute<B, M>::asDocumentWeightAttribute() const { diff --git a/searchlib/src/vespa/searchlib/attribute/multistringpostattribute.h b/searchlib/src/vespa/searchlib/attribute/multistringpostattribute.h index 031d90a88ed..4deb71e9759 100644 --- a/searchlib/src/vespa/searchlib/attribute/multistringpostattribute.h +++ b/searchlib/src/vespa/searchlib/attribute/multistringpostattribute.h @@ -38,6 +38,7 @@ private: void collect_folded(vespalib::datastore::EntryRef enum_idx, vespalib::datastore::EntryRef dictionary_snapshot, const std::function<void(vespalib::datastore::EntryRef)>& callback) const override; void create(vespalib::datastore::EntryRef idx, std::vector<DocumentWeightIterator> &dst) const override; DocumentWeightIterator create(vespalib::datastore::EntryRef idx) const override; + std::unique_ptr<queryeval::SearchIterator> make_bitvector_iterator(vespalib::datastore::EntryRef idx, uint32_t doc_id_limit, fef::TermFieldMatchData &match_data, bool strict) const override; }; DocumentWeightAttributeAdapter _document_weight_attribute_adapter; diff --git a/searchlib/src/vespa/searchlib/attribute/multistringpostattribute.hpp b/searchlib/src/vespa/searchlib/attribute/multistringpostattribute.hpp index 421e727761f..39754464ad3 100644 --- a/searchlib/src/vespa/searchlib/attribute/multistringpostattribute.hpp +++ b/searchlib/src/vespa/searchlib/attribute/multistringpostattribute.hpp @@ -159,6 +159,13 @@ MultiValueStringPostingAttributeT<B, M>::DocumentWeightAttributeAdapter::create( return self.getPostingList().beginFrozen(idx); } +template <typename B, typename M> +std::unique_ptr<queryeval::SearchIterator> +MultiValueStringPostingAttributeT<B, M>::DocumentWeightAttributeAdapter::make_bitvector_iterator(vespalib::datastore::EntryRef idx, uint32_t doc_id_limit, fef::TermFieldMatchData &match_data, bool strict) const +{ + return self.getPostingList().make_bitvector_iterator(idx, doc_id_limit, match_data, strict); +} + template <typename B, typename T> const IDocumentWeightAttribute * MultiValueStringPostingAttributeT<B, T>::asDocumentWeightAttribute() const diff --git a/searchlib/src/vespa/searchlib/attribute/postingstore.cpp b/searchlib/src/vespa/searchlib/attribute/postingstore.cpp index a942e70085a..1b296f7d03e 100644 --- a/searchlib/src/vespa/searchlib/attribute/postingstore.cpp +++ b/searchlib/src/vespa/searchlib/attribute/postingstore.cpp @@ -1,6 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "postingstore.h" +#include <vespa/searchlib/common/bitvectoriterator.h> #include <vespa/searchlib/common/growablebitvector.h> #include <vespa/searchcommon/attribute/config.h> #include <vespa/searchcommon/attribute/status.h> @@ -792,6 +793,21 @@ PostingStore<DataT>::consider_compact_worst_buffers(const CompactionStrategy& co return false; } +template <typename DataT> +std::unique_ptr<queryeval::SearchIterator> +PostingStore<DataT>::make_bitvector_iterator(RefType ref, uint32_t doc_id_limit, fef::TermFieldMatchData &match_data, bool strict) const +{ + if (!ref.valid()) { + return {}; + } + auto type_id = getTypeId(ref); + if (!isBitVector(type_id)) { + return {}; + } + const auto& bv = getBitVectorEntry(ref)->_bv->reader(); + return BitVectorIterator::create(&bv, std::min(bv.size(), doc_id_limit), match_data, strict, false); +} + template class PostingStore<BTreeNoLeafData>; template class PostingStore<int32_t>; diff --git a/searchlib/src/vespa/searchlib/attribute/postingstore.h b/searchlib/src/vespa/searchlib/attribute/postingstore.h index 949a355bc9d..6512c59fa79 100644 --- a/searchlib/src/vespa/searchlib/attribute/postingstore.h +++ b/searchlib/src/vespa/searchlib/attribute/postingstore.h @@ -12,6 +12,9 @@ namespace search { class GrowableBitVector; } +namespace search::fef { class TermFieldMatchData; } +namespace search::queryeval { class SearchIterator; } + namespace search::attribute { class Status; @@ -185,6 +188,8 @@ public: return _store.template getEntry<BitVectorEntry>(ref); } + std::unique_ptr<queryeval::SearchIterator> make_bitvector_iterator(RefType ref, uint32_t doc_id_limit, fef::TermFieldMatchData &match_data, bool strict) const; + static inline DataT bitVectorWeight(); vespalib::MemoryUsage getMemoryUsage() const; vespalib::MemoryUsage update_stat(const CompactionStrategy& compaction_strategy); diff --git a/vespa-osgi-testrunner/src/main/java/com/yahoo/vespa/testrunner/JunitRunner.java b/vespa-osgi-testrunner/src/main/java/com/yahoo/vespa/testrunner/JunitRunner.java index 75873e7b997..54f0941208d 100644 --- a/vespa-osgi-testrunner/src/main/java/com/yahoo/vespa/testrunner/JunitRunner.java +++ b/vespa-osgi-testrunner/src/main/java/com/yahoo/vespa/testrunner/JunitRunner.java @@ -103,7 +103,7 @@ public class JunitRunner extends AbstractComponent implements TestRunner { private TestReport launchJunit(Suite suite, byte[] testConfig) { List<Class<?>> testClasses = classLoader.apply(suite); - if (testClasses == null || testClasses.isEmpty()) + if (testClasses == null) return null; testRuntimeProvider.initialize(testConfig); @@ -117,14 +117,7 @@ public class JunitRunner extends AbstractComponent implements TestRunner { .map(DiscoverySelectors::selectClass) .collect(toList())) .build(); - ClassLoader old = Thread.currentThread().getContextClassLoader(); - try { - Thread.currentThread().setContextClassLoader(testClasses.get(0).getClassLoader()); - testExecutor.accept(discoveryRequest, new TestExecutionListener[] { testReportListener }); - } - finally { - Thread.currentThread().setContextClassLoader(old); - } + testExecutor.accept(discoveryRequest, new TestExecutionListener[] { testReportListener }); return testReportListener.report(); } diff --git a/vespajlib/src/main/java/com/yahoo/compress/ZstdCompressor.java b/vespajlib/src/main/java/com/yahoo/compress/ZstdCompressor.java index 004c1d2a017..29a58dbde47 100644 --- a/vespajlib/src/main/java/com/yahoo/compress/ZstdCompressor.java +++ b/vespajlib/src/main/java/com/yahoo/compress/ZstdCompressor.java @@ -28,7 +28,7 @@ public class ZstdCompressor { /** * Note: * Implementation assumes single frame (since {@link #getDecompressedLength(byte[], int, int)} only includes the first frame) - * The {@link #decompress(byte[], int, int, byte[], int, int)} overload will try to decompress all frame, causing the output buffer to overflow. + * The {@link #decompress(byte[], int, int, byte[], int, int)} overload will try to decompress all frames, causing the output buffer to overflow. */ public byte[] decompress(byte[] input, int inputOffset, int inputLength) { int decompressedLength = getDecompressedLength(input, inputOffset, inputLength); @@ -48,4 +48,5 @@ public class ZstdCompressor { public static int getDecompressedLength(byte[] input, int inputOffset, int inputLength) { return (int) io.airlift.compress.zstd.ZstdDecompressor.getDecompressedSize(input, inputOffset, inputLength); } + } diff --git a/vespajlib/src/main/java/com/yahoo/compress/ZstdOutputStream.java b/vespajlib/src/main/java/com/yahoo/compress/ZstdOutputStream.java index f439ee03ea6..2952195b224 100644 --- a/vespajlib/src/main/java/com/yahoo/compress/ZstdOutputStream.java +++ b/vespajlib/src/main/java/com/yahoo/compress/ZstdOutputStream.java @@ -38,7 +38,6 @@ public class ZstdOutputStream extends OutputStream { @Override public void write(byte[] b) throws IOException { - throwIfClosed(); write(b, 0, b.length); } diff --git a/vespalib/src/tests/shared_string_repo/CMakeLists.txt b/vespalib/src/tests/shared_string_repo/CMakeLists.txt index 9b788d9cbb7..6ae91b99411 100644 --- a/vespalib/src/tests/shared_string_repo/CMakeLists.txt +++ b/vespalib/src/tests/shared_string_repo/CMakeLists.txt @@ -5,4 +5,12 @@ vespa_add_executable(vespalib_shared_string_repo_test_app TEST DEPENDS vespalib ) -vespa_add_test(NAME vespalib_shared_string_repo_test_app COMMAND vespalib_shared_string_repo_test_app) +vespa_add_test( + NAME vespalib_shared_string_repo_test_app + COMMAND vespalib_shared_string_repo_test_app +) +vespa_add_test( + NAME vespalib_shared_string_repo_test_app_no_reclaim + COMMAND vespalib_shared_string_repo_test_app + ENVIRONMENT "VESPA_SHARED_STRING_REPO_NO_RECLAIM=true" +) diff --git a/vespalib/src/tests/shared_string_repo/shared_string_repo_test.cpp b/vespalib/src/tests/shared_string_repo/shared_string_repo_test.cpp index d128e5b8c49..acc710f8818 100644 --- a/vespalib/src/tests/shared_string_repo/shared_string_repo_test.cpp +++ b/vespalib/src/tests/shared_string_repo/shared_string_repo_test.cpp @@ -21,6 +21,14 @@ bool verbose = false; double budget = 0.10; size_t work_size = 4_Ki; +size_t active_enums() { + return SharedStringRepo::stats().active_entries; +} + +bool will_reclaim() { + return SharedStringRepo::will_reclaim(); +} + //----------------------------------------------------------------------------- std::vector<vespalib::string> make_strings(size_t cnt) { @@ -352,7 +360,7 @@ TEST("require that basic handle usage works") { Handle foo2("foo"); Handle bar2("bar"); - EXPECT_EQUAL(SharedStringRepo::stats().active_entries, 2u); + EXPECT_EQUAL(active_enums(), 2u); TEST_DO(verify_eq(empty, empty2)); TEST_DO(verify_eq(foo, foo2)); @@ -375,31 +383,37 @@ TEST("require that basic handle usage works") { } TEST("require that handles can be copied") { - Handle a("foo"); + size_t before = active_enums(); + Handle a("copied"); + EXPECT_EQUAL(active_enums(), before + 1); Handle b(a); Handle c; c = b; - EXPECT_EQUAL(SharedStringRepo::stats().active_entries, 1u); + EXPECT_EQUAL(active_enums(), before + 1); EXPECT_TRUE(a.id() == b.id()); EXPECT_TRUE(b.id() == c.id()); - EXPECT_EQUAL(c.as_string(), vespalib::string("foo")); + EXPECT_EQUAL(c.as_string(), vespalib::string("copied")); } TEST("require that handles can be moved") { - Handle a("foo"); + size_t before = active_enums(); + Handle a("moved"); + EXPECT_EQUAL(active_enums(), before + 1); Handle b(std::move(a)); Handle c; c = std::move(b); - EXPECT_EQUAL(SharedStringRepo::stats().active_entries, 1u); + EXPECT_EQUAL(active_enums(), before + 1); EXPECT_TRUE(a.id() == string_id()); EXPECT_TRUE(b.id() == string_id()); - EXPECT_EQUAL(c.as_string(), vespalib::string("foo")); + EXPECT_EQUAL(c.as_string(), vespalib::string("moved")); } TEST("require that handle/string can be obtained from string_id") { + size_t before = active_enums(); Handle a("str"); + EXPECT_EQUAL(active_enums(), before + 1); Handle b = Handle::handle_from_id(a.id()); - EXPECT_EQUAL(SharedStringRepo::stats().active_entries, 1u); + EXPECT_EQUAL(active_enums(), before + 1); EXPECT_EQUAL(Handle::string_from_id(b.id()), vespalib::string("str")); } @@ -419,19 +433,19 @@ TEST("require that handle can be self-assigned") { //----------------------------------------------------------------------------- void verify_direct(const vespalib::string &str, size_t value) { - size_t before = SharedStringRepo::stats().active_entries; + size_t before = active_enums(); Handle handle(str); EXPECT_EQUAL(handle.id().hash(), value + 1); EXPECT_EQUAL(handle.id().value(), value + 1); - EXPECT_EQUAL(SharedStringRepo::stats().active_entries, before); + EXPECT_EQUAL(active_enums(), before); EXPECT_EQUAL(handle.as_string(), str); } void verify_not_direct(const vespalib::string &str) { - size_t before = SharedStringRepo::stats().active_entries; + size_t before = active_enums(); Handle handle(str); EXPECT_EQUAL(handle.id().hash(), handle.id().value()); - EXPECT_EQUAL(SharedStringRepo::stats().active_entries, before + 1); + EXPECT_EQUAL(active_enums(), before + 1); EXPECT_EQUAL(handle.as_string(), str); } @@ -458,6 +472,7 @@ TEST("require that direct handles work as expected") { //----------------------------------------------------------------------------- TEST("require that basic multi-handle usage works") { + size_t before = active_enums(); Handles a; a.reserve(4); Handle foo("foo"); @@ -467,7 +482,12 @@ TEST("require that basic multi-handle usage works") { a.push_back(foo.id()); a.push_back(bar.id()); Handles b(std::move(a)); - EXPECT_EQUAL(SharedStringRepo::stats().active_entries, 2u); + if (will_reclaim()) { + EXPECT_EQUAL(before, 0u); + EXPECT_EQUAL(active_enums(), 2u); + } else { + EXPECT_EQUAL(active_enums(), before); + } EXPECT_EQUAL(a.view().size(), 0u); EXPECT_EQUAL(b.view().size(), 4u); EXPECT_TRUE(b.view()[0] == foo.id()); @@ -539,7 +559,15 @@ TEST("leak some handles on purpose") { #endif TEST("require that no handles have leaked during testing") { - EXPECT_EQUAL(SharedStringRepo::stats().active_entries, 0u); + if (will_reclaim()) { + EXPECT_EQUAL(active_enums(), 0u); + } else { + auto stats = SharedStringRepo::stats(); + fprintf(stderr, "enum stats after testing (no reclaim):\n"); + fprintf(stderr, " active enums: %zu\n", stats.active_entries); + fprintf(stderr, " id space usage: %g\n", stats.id_space_usage()); + fprintf(stderr, " memory usage: %zu\n", stats.memory_usage.usedBytes()); + } } //----------------------------------------------------------------------------- diff --git a/vespalib/src/vespa/vespalib/util/shared_string_repo.cpp b/vespalib/src/vespa/vespalib/util/shared_string_repo.cpp index e46d5474eb9..5a09d617616 100644 --- a/vespalib/src/vespa/vespalib/util/shared_string_repo.cpp +++ b/vespalib/src/vespa/vespalib/util/shared_string_repo.cpp @@ -7,6 +7,17 @@ LOG_SETUP(".vespalib.shared_string_repo"); namespace vespalib { +namespace { + +bool resolve_should_reclaim_flag() { + bool no_reclaim = (getenv("VESPA_SHARED_STRING_REPO_NO_RECLAIM") != nullptr); + return !no_reclaim; +} + +} + +const bool SharedStringRepo::should_reclaim = resolve_should_reclaim_flag(); + SharedStringRepo::Stats::Stats() : active_entries(0), total_entries(0), @@ -86,8 +97,10 @@ SharedStringRepo SharedStringRepo::_repo; SharedStringRepo::SharedStringRepo() = default; SharedStringRepo::~SharedStringRepo() { - for (size_t p = 0; p < _partitions.size(); ++p) { - _partitions[p].find_leaked_entries(p); + if (should_reclaim) { + for (size_t p = 0; p < _partitions.size(); ++p) { + _partitions[p].find_leaked_entries(p); + } } } @@ -116,8 +129,10 @@ SharedStringRepo::Handles::Handles(Handles &&rhs) SharedStringRepo::Handles::~Handles() { - for (string_id handle: _handles) { - _repo.reclaim(handle); + if (should_reclaim) { + for (string_id handle: _handles) { + _repo.reclaim(handle); + } } } diff --git a/vespalib/src/vespa/vespalib/util/shared_string_repo.h b/vespalib/src/vespa/vespalib/util/shared_string_repo.h index a55484537e2..260cac7428b 100644 --- a/vespalib/src/vespa/vespalib/util/shared_string_repo.h +++ b/vespalib/src/vespa/vespalib/util/shared_string_repo.h @@ -49,6 +49,7 @@ private: static constexpr uint32_t FAST_ID_MAX = 9999999; static constexpr uint32_t ID_BIAS = (FAST_ID_MAX + 2); static constexpr size_t PART_LIMIT = (std::numeric_limits<uint32_t>::max() - ID_BIAS) / NUM_PARTS; + static const bool should_reclaim; struct AltKey { vespalib::stringref str; @@ -141,10 +142,13 @@ private: Stats stats() const; uint32_t resolve(const AltKey &alt_key) { + bool count_refs = should_reclaim; std::lock_guard guard(_lock); auto pos = _hash.find(alt_key); if (pos != _hash.end()) { - _entries[pos->idx].add_ref(); + if (count_refs) { + _entries[pos->idx].add_ref(); + } return pos->idx; } else { uint32_t idx = make_entry(alt_key); @@ -232,7 +236,7 @@ private: } string_id copy(string_id id) { - if (id._id >= ID_BIAS) { + if ((id._id >= ID_BIAS) && should_reclaim) { uint32_t part = (id._id - ID_BIAS) & PART_MASK; uint32_t local_idx = (id._id - ID_BIAS) >> PART_BITS; _partitions[part].copy(local_idx); @@ -241,7 +245,7 @@ private: } void reclaim(string_id id) { - if (id._id >= ID_BIAS) { + if ((id._id >= ID_BIAS) && should_reclaim) { uint32_t part = (id._id - ID_BIAS) & PART_MASK; uint32_t local_idx = (id._id - ID_BIAS) >> PART_BITS; _partitions[part].reclaim(local_idx); @@ -251,6 +255,7 @@ private: static SharedStringRepo _repo; public: + static bool will_reclaim() { return should_reclaim; } static Stats stats(); // A single stand-alone string handle with ownership |